Vendor import of clang r114020 (from the release_28 branch):vendor/clang/clang-r114020

http://llvm.org/svn/llvm-project/cfe/branches/release_28@114020

Approved by:	rpaulo (mentor)

302 files changed, 39303 insertions, 29014 deletions

diff --git a/lib/AST/ASTConsumer.cpp b/lib/AST/ASTConsumer.cpp
index f37cbdea5480..04a084a06a44 100644
--- a/lib/AST/ASTConsumer.cpp
+++ b/lib/AST/ASTConsumer.cpp
@@ -17,3 +17,6 @@ using namespace clang;
 
 void ASTConsumer::HandleTopLevelDecl(DeclGroupRef D) {}
 
+void ASTConsumer::HandleInterestingDecl(DeclGroupRef D) {
+  HandleTopLevelDecl(D);
+}
diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
index d41051f5dcad..4591a0f3c55c 100644
--- a/lib/AST/ASTContext.cpp
+++ b/lib/AST/ASTContext.cpp
@@ -28,6 +28,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "CXXABI.h"
 
 using namespace clang;
 
@@ -134,11 +135,25 @@ ASTContext::getCanonicalTemplateTemplateParmDecl(
   return CanonTTP;
 }
 
+CXXABI *ASTContext::createCXXABI(const TargetInfo &T) {
+  if (!LangOpts.CPlusPlus) return 0;
+
+  switch (T.getCXXABI()) {
+  case CXXABI_ARM:
+    return CreateARMCXXABI(*this);
+  case CXXABI_Itanium:
+    return CreateItaniumCXXABI(*this);
+  case CXXABI_Microsoft:
+    return CreateMicrosoftCXXABI(*this);
+  }
+  return 0;
+}
+
 ASTContext::ASTContext(const LangOptions& LOpts, SourceManager &SM,
                        const TargetInfo &t,
                        IdentifierTable &idents, SelectorTable &sels,
                        Builtin::Context &builtins,
-                       bool FreeMem, unsigned size_reserve) :
+                       unsigned size_reserve) :
   TemplateSpecializationTypes(this_()),
   DependentTemplateSpecializationTypes(this_()),
   GlobalNestedNameSpecifier(0), IsInt128Installed(false),
@@ -146,7 +161,7 @@ ASTContext::ASTContext(const LangOptions& LOpts, SourceManager &SM,
   ObjCFastEnumerationStateTypeDecl(0), FILEDecl(0), jmp_bufDecl(0),
   sigjmp_bufDecl(0), BlockDescriptorType(0), BlockDescriptorExtendedType(0),
   NullTypeSourceInfo(QualType()),
-  SourceMgr(SM), LangOpts(LOpts), FreeMemory(FreeMem), Target(t),
+  SourceMgr(SM), LangOpts(LOpts), ABI(createCXXABI(t)), Target(t),
   Idents(idents), Selectors(sels),
   BuiltinInfo(builtins),
   DeclarationNames(*this),
@@ -155,7 +170,7 @@ ASTContext::ASTContext(const LangOptions& LOpts, SourceManager &SM,
   UniqueBlockByRefTypeID(0), UniqueBlockParmTypeID(0) {
   ObjCIdRedefinitionType = QualType();
   ObjCClassRedefinitionType = QualType();
-  ObjCSelRedefinitionType = QualType();
+  ObjCSelRedefinitionType = QualType();    
   if (size_reserve > 0) Types.reserve(size_reserve);
   TUDecl = TranslationUnitDecl::Create(*this);
   InitBuiltinTypes();
@@ -166,11 +181,9 @@ ASTContext::~ASTContext() {
   // FIXME: Is this the ideal solution?
   ReleaseDeclContextMaps();
 
-  if (!FreeMemory) {
-    // Call all of the deallocation functions.
-    for (unsigned I = 0, N = Deallocations.size(); I != N; ++I)
-      Deallocations[I].first(Deallocations[I].second);
-  }
+  // Call all of the deallocation functions.
+  for (unsigned I = 0, N = Deallocations.size(); I != N; ++I)
+    Deallocations[I].first(Deallocations[I].second);
   
   // Release all of the memory associated with overridden C++ methods.
   for (llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector>::iterator 
@@ -178,51 +191,26 @@ ASTContext::~ASTContext() {
        OM != OMEnd; ++OM)
     OM->second.Destroy();
   
-  if (FreeMemory) {
-    // Deallocate all the types.
-    while (!Types.empty()) {
-      Types.back()->Destroy(*this);
-      Types.pop_back();
-    }
-
-    for (llvm::FoldingSet<ExtQuals>::iterator
-         I = ExtQualNodes.begin(), E = ExtQualNodes.end(); I != E; ) {
-      // Increment in loop to prevent using deallocated memory.
-      Deallocate(&*I++);
-    }
-
-    for (llvm::DenseMap<const ObjCContainerDecl*,
-         const ASTRecordLayout*>::iterator
-         I = ObjCLayouts.begin(), E = ObjCLayouts.end(); I != E; ) {
-      // Increment in loop to prevent using deallocated memory.
-      if (ASTRecordLayout *R = const_cast<ASTRecordLayout*>((I++)->second))
-        R->Destroy(*this);
-    }
-  }
-
   // ASTRecordLayout objects in ASTRecordLayouts must always be destroyed
-  // even when using the BumpPtrAllocator because they can contain
-  // DenseMaps.
-  for (llvm::DenseMap<const RecordDecl*, const ASTRecordLayout*>::iterator
-       I = ASTRecordLayouts.begin(), E = ASTRecordLayouts.end(); I != E; ) {
+  // because they can contain DenseMaps.
+  for (llvm::DenseMap<const ObjCContainerDecl*,
+       const ASTRecordLayout*>::iterator
+       I = ObjCLayouts.begin(), E = ObjCLayouts.end(); I != E; )
     // Increment in loop to prevent using deallocated memory.
     if (ASTRecordLayout *R = const_cast<ASTRecordLayout*>((I++)->second))
       R->Destroy(*this);
-  }
 
-  // Destroy nested-name-specifiers.
-  for (llvm::FoldingSet<NestedNameSpecifier>::iterator
-         NNS = NestedNameSpecifiers.begin(),
-         NNSEnd = NestedNameSpecifiers.end();
-       NNS != NNSEnd; ) {
+  for (llvm::DenseMap<const RecordDecl*, const ASTRecordLayout*>::iterator
+       I = ASTRecordLayouts.begin(), E = ASTRecordLayouts.end(); I != E; ) {
     // Increment in loop to prevent using deallocated memory.
-    (*NNS++).Destroy(*this);
+    if (ASTRecordLayout *R = const_cast<ASTRecordLayout*>((I++)->second))
+      R->Destroy(*this);
   }
-
-  if (GlobalNestedNameSpecifier)
-    GlobalNestedNameSpecifier->Destroy(*this);
-
-  TUDecl->Destroy(*this);
+  
+  for (llvm::DenseMap<const Decl*, AttrVec*>::iterator A = DeclAttrs.begin(),
+                                                    AEnd = DeclAttrs.end();
+       A != AEnd; ++A)
+    A->second->~AttrVec();
 }
 
 void ASTContext::AddDeallocation(void (*Callback)(void*), void *Data) {
@@ -275,16 +263,12 @@ void ASTContext::PrintStats() const {
   fprintf(stderr, "  %u/%u implicit destructors created\n",
           NumImplicitDestructorsDeclared, NumImplicitDestructors);
   
-  if (!FreeMemory)
-    BumpAlloc.PrintStats();
-    
   if (ExternalSource.get()) {
     fprintf(stderr, "\n");
     ExternalSource->PrintStats();
   }
   
-  if (!FreeMemory)
-    BumpAlloc.PrintStats();
+  BumpAlloc.PrintStats();
 }
 
 
@@ -385,6 +369,26 @@ void ASTContext::InitBuiltinTypes() {
   InitBuiltinType(NullPtrTy,           BuiltinType::NullPtr);
 }
 
+AttrVec& ASTContext::getDeclAttrs(const Decl *D) {
+  AttrVec *&Result = DeclAttrs[D];
+  if (!Result) {
+    void *Mem = Allocate(sizeof(AttrVec));
+    Result = new (Mem) AttrVec;
+  }
+    
+  return *Result;
+}
+
+/// \brief Erase the attributes corresponding to the given declaration.
+void ASTContext::eraseDeclAttrs(const Decl *D) { 
+  llvm::DenseMap<const Decl*, AttrVec*>::iterator Pos = DeclAttrs.find(D);
+  if (Pos != DeclAttrs.end()) {
+    Pos->second->~AttrVec();
+    DeclAttrs.erase(Pos);
+  }
+}
+
+
 MemberSpecializationInfo *
 ASTContext::getInstantiatedFromStaticDataMember(const VarDecl *Var) {
   assert(Var->isStaticDataMember() && "Not a static data member");
@@ -499,20 +503,6 @@ void ASTContext::addOverriddenMethod(const CXXMethodDecl *Method,
   OverriddenMethods[Method].push_back(Overridden);
 }
 
-namespace {
-  class BeforeInTranslationUnit
-    : std::binary_function<SourceRange, SourceRange, bool> {
-    SourceManager *SourceMgr;
-
-  public:
-    explicit BeforeInTranslationUnit(SourceManager *SM) : SourceMgr(SM) { }
-
-    bool operator()(SourceRange X, SourceRange Y) {
-      return SourceMgr->isBeforeInTranslationUnit(X.getBegin(), Y.getBegin());
-    }
-  };
-}
-
 //===----------------------------------------------------------------------===//
 //                         Type Sizing and Analysis
 //===----------------------------------------------------------------------===//
@@ -538,8 +528,7 @@ const llvm::fltSemantics &ASTContext::getFloatTypeSemantics(QualType T) const {
 CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) {
   unsigned Align = Target.getCharWidth();
 
-  if (const AlignedAttr* AA = D->getAttr<AlignedAttr>())
-    Align = std::max(Align, AA->getMaxAlignment());
+  Align = std::max(Align, D->getMaxAlignment());
 
   if (const ValueDecl *VD = dyn_cast<ValueDecl>(D)) {
     QualType T = VD->getType();
@@ -716,6 +705,12 @@ ASTContext::getTypeInfo(const Type *T) {
       Width = Target.getPointerWidth(0); // C++ 3.9.1p11: sizeof(nullptr_t)
       Align = Target.getPointerAlign(0); //   == sizeof(void*)
       break;
+    case BuiltinType::ObjCId:
+    case BuiltinType::ObjCClass:
+    case BuiltinType::ObjCSel:
+      Width = Target.getPointerWidth(0); 
+      Align = Target.getPointerAlign(0);
+      break;
     }
     break;
   case Type::ObjCObjectPointer:
@@ -744,12 +739,10 @@ ASTContext::getTypeInfo(const Type *T) {
     break;
   }
   case Type::MemberPointer: {
-    QualType Pointee = cast<MemberPointerType>(T)->getPointeeType();
+    const MemberPointerType *MPT = cast<MemberPointerType>(T);
     std::pair<uint64_t, unsigned> PtrDiffInfo =
       getTypeInfo(getPointerDiffType());
-    Width = PtrDiffInfo.first;
-    if (Pointee->isFunctionType())
-      Width *= 2;
+    Width = PtrDiffInfo.first * ABI->getMemberPointerSize(MPT);
     Align = PtrDiffInfo.second;
     break;
   }
@@ -797,12 +790,10 @@ ASTContext::getTypeInfo(const Type *T) {
 
   case Type::Typedef: {
     const TypedefDecl *Typedef = cast<TypedefType>(T)->getDecl();
-    if (const AlignedAttr *Aligned = Typedef->getAttr<AlignedAttr>()) {
-      Align = std::max(Aligned->getMaxAlignment(),
-                       getTypeAlign(Typedef->getUnderlyingType().getTypePtr()));
-      Width = getTypeSize(Typedef->getUnderlyingType().getTypePtr());
-    } else
-      return getTypeInfo(Typedef->getUnderlyingType().getTypePtr());
+    std::pair<uint64_t, unsigned> Info
+      = getTypeInfo(Typedef->getUnderlyingType().getTypePtr());
+    Align = std::max(Typedef->getMaxAlignment(), Info.second);
+    Width = Info.first;
     break;
   }
 
@@ -868,60 +859,37 @@ unsigned ASTContext::getPreferredTypeAlign(const Type *T) {
   return ABIAlign;
 }
 
-static void CollectLocalObjCIvars(ASTContext *Ctx,
-                                  const ObjCInterfaceDecl *OI,
-                                  llvm::SmallVectorImpl<FieldDecl*> &Fields) {
-  for (ObjCInterfaceDecl::ivar_iterator I = OI->ivar_begin(),
-       E = OI->ivar_end(); I != E; ++I) {
-    ObjCIvarDecl *IVDecl = *I;
-    if (!IVDecl->isInvalidDecl())
-      Fields.push_back(cast<FieldDecl>(IVDecl));
-  }
-}
-
-void ASTContext::CollectObjCIvars(const ObjCInterfaceDecl *OI,
-                             llvm::SmallVectorImpl<FieldDecl*> &Fields) {
-  if (const ObjCInterfaceDecl *SuperClass = OI->getSuperClass())
-    CollectObjCIvars(SuperClass, Fields);
-  CollectLocalObjCIvars(this, OI, Fields);
-}
-
 /// ShallowCollectObjCIvars -
 /// Collect all ivars, including those synthesized, in the current class.
 ///
 void ASTContext::ShallowCollectObjCIvars(const ObjCInterfaceDecl *OI,
                                  llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
-  for (ObjCInterfaceDecl::ivar_iterator I = OI->ivar_begin(),
-         E = OI->ivar_end(); I != E; ++I) {
-     Ivars.push_back(*I);
-  }
-
-  CollectNonClassIvars(OI, Ivars);
-}
-
-/// CollectNonClassIvars -
-/// This routine collects all other ivars which are not declared in the class.
-/// This includes synthesized ivars (via @synthesize) and those in
-//  class's @implementation.
+  // FIXME. This need be removed but there are two many places which
+  // assume const-ness of ObjCInterfaceDecl
+  ObjCInterfaceDecl *IDecl = const_cast<ObjCInterfaceDecl *>(OI);
+  for (ObjCIvarDecl *Iv = IDecl->all_declared_ivar_begin(); Iv; 
+        Iv= Iv->getNextIvar())
+    Ivars.push_back(Iv);
+}
+
+/// DeepCollectObjCIvars -
+/// This routine first collects all declared, but not synthesized, ivars in
+/// super class and then collects all ivars, including those synthesized for
+/// current class. This routine is used for implementation of current class
+/// when all ivars, declared and synthesized are known.
 ///
-void ASTContext::CollectNonClassIvars(const ObjCInterfaceDecl *OI,
+void ASTContext::DeepCollectObjCIvars(const ObjCInterfaceDecl *OI,
+                                      bool leafClass,
                                 llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
-  // Find ivars declared in class extension.
-  for (const ObjCCategoryDecl *CDecl = OI->getFirstClassExtension(); CDecl;
-       CDecl = CDecl->getNextClassExtension()) {
-    for (ObjCCategoryDecl::ivar_iterator I = CDecl->ivar_begin(),
-         E = CDecl->ivar_end(); I != E; ++I) {
-      Ivars.push_back(*I);
-    }
-  }
-
-  // Also add any ivar defined in this class's implementation.  This
-  // includes synthesized ivars.
-  if (ObjCImplementationDecl *ImplDecl = OI->getImplementation()) {
-    for (ObjCImplementationDecl::ivar_iterator I = ImplDecl->ivar_begin(),
-         E = ImplDecl->ivar_end(); I != E; ++I)
+  if (const ObjCInterfaceDecl *SuperClass = OI->getSuperClass())
+    DeepCollectObjCIvars(SuperClass, false, Ivars);
+  if (!leafClass) {
+    for (ObjCInterfaceDecl::ivar_iterator I = OI->ivar_begin(),
+         E = OI->ivar_end(); I != E; ++I)
       Ivars.push_back(*I);
   }
+  else
+    ShallowCollectObjCIvars(OI, Ivars);
 }
 
 /// CollectInheritedProtocols - Collect all protocols in current class and
@@ -929,8 +897,10 @@ void ASTContext::CollectNonClassIvars(const ObjCInterfaceDecl *OI,
 void ASTContext::CollectInheritedProtocols(const Decl *CDecl,
                           llvm::SmallPtrSet<ObjCProtocolDecl*, 8> &Protocols) {
   if (const ObjCInterfaceDecl *OI = dyn_cast<ObjCInterfaceDecl>(CDecl)) {
-    for (ObjCInterfaceDecl::protocol_iterator P = OI->protocol_begin(),
-         PE = OI->protocol_end(); P != PE; ++P) {
+    // We can use protocol_iterator here instead of
+    // all_referenced_protocol_iterator since we are walking all categories.    
+    for (ObjCInterfaceDecl::all_protocol_iterator P = OI->all_referenced_protocol_begin(),
+         PE = OI->all_referenced_protocol_end(); P != PE; ++P) {
       ObjCProtocolDecl *Proto = (*P);
       Protocols.insert(Proto);
       for (ObjCProtocolDecl::protocol_iterator P = Proto->protocol_begin(),
@@ -950,7 +920,7 @@ void ASTContext::CollectInheritedProtocols(const Decl *CDecl,
         SD = SD->getSuperClass();
       }
   } else if (const ObjCCategoryDecl *OC = dyn_cast<ObjCCategoryDecl>(CDecl)) {
-    for (ObjCInterfaceDecl::protocol_iterator P = OC->protocol_begin(),
+    for (ObjCCategoryDecl::protocol_iterator P = OC->protocol_begin(),
          PE = OC->protocol_end(); P != PE; ++P) {
       ObjCProtocolDecl *Proto = (*P);
       Protocols.insert(Proto);
@@ -1154,6 +1124,15 @@ static QualType getExtFunctionType(ASTContext& Context, QualType T,
       return T;
 
     ResultType = Context.getBlockPointerType(ResultType);
+  } else if (const MemberPointerType *MemberPointer 
+                                            = T->getAs<MemberPointerType>()) {
+    QualType Pointee = MemberPointer->getPointeeType();
+    ResultType = getExtFunctionType(Context, Pointee, Info);
+    if (ResultType == Pointee)
+      return T;
+    
+    ResultType = Context.getMemberPointerType(ResultType, 
+                                              MemberPointer->getClass());
    } else if (const FunctionType *F = T->getAs<FunctionType>()) {
     if (F->getExtInfo() == Info)
       return T;
@@ -1570,10 +1549,7 @@ QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
   // If the element type isn't canonical, this won't be a canonical type either,
   // so fill in the canonical type field.
   QualType Canonical;
-  if (!vecType.isCanonical() || (AltiVecSpec == VectorType::AltiVec)) {
-    // pass VectorType::NotAltiVec for AltiVecSpec to make AltiVec canonical
-    // vector type (except 'vector bool ...' and 'vector Pixel') the same as
-    // the equivalent GCC vector types
+  if (!vecType.isCanonical()) {
     Canonical = getVectorType(getCanonicalType(vecType), NumElts,
       VectorType::NotAltiVec);
 
@@ -2567,21 +2543,31 @@ bool ASTContext::UnwrapSimilarPointerTypes(QualType &T1, QualType &T2) {
   return false;
 }
 
-DeclarationName ASTContext::getNameForTemplate(TemplateName Name) {
+DeclarationNameInfo ASTContext::getNameForTemplate(TemplateName Name,
+                                                   SourceLocation NameLoc) {
   if (TemplateDecl *TD = Name.getAsTemplateDecl())
-    return TD->getDeclName();
-  
+    // DNInfo work in progress: CHECKME: what about DNLoc?
+    return DeclarationNameInfo(TD->getDeclName(), NameLoc);
+
   if (DependentTemplateName *DTN = Name.getAsDependentTemplateName()) {
+    DeclarationName DName;
     if (DTN->isIdentifier()) {
-      return DeclarationNames.getIdentifier(DTN->getIdentifier());
+      DName = DeclarationNames.getIdentifier(DTN->getIdentifier());
+      return DeclarationNameInfo(DName, NameLoc);
     } else {
-      return DeclarationNames.getCXXOperatorName(DTN->getOperator());
+      DName = DeclarationNames.getCXXOperatorName(DTN->getOperator());
+      // DNInfo work in progress: FIXME: source locations?
+      DeclarationNameLoc DNLoc;
+      DNLoc.CXXOperatorName.BeginOpNameLoc = SourceLocation().getRawEncoding();
+      DNLoc.CXXOperatorName.EndOpNameLoc = SourceLocation().getRawEncoding();
+      return DeclarationNameInfo(DName, NameLoc, DNLoc);
     }
   }
 
   OverloadedTemplateStorage *Storage = Name.getAsOverloadedTemplate();
   assert(Storage);
-  return (*Storage->begin())->getDeclName();
+  // DNInfo work in progress: CHECKME: what about DNLoc?
+  return DeclarationNameInfo((*Storage->begin())->getDeclName(), NameLoc);
 }
 
 TemplateName ASTContext::getCanonicalTemplateName(TemplateName Name) {
@@ -3216,14 +3202,14 @@ bool ASTContext::BlockRequiresCopying(QualType Ty) {
   return false;
 }
 
-QualType ASTContext::BuildByRefType(const char *DeclName, QualType Ty) {
+QualType ASTContext::BuildByRefType(llvm::StringRef DeclName, QualType Ty) {
   //  type = struct __Block_byref_1_X {
   //    void *__isa;
   //    struct __Block_byref_1_X *__forwarding;
   //    unsigned int __flags;
   //    unsigned int __size;
-  //    void *__copy_helper;		// as needed
-  //    void *__destroy_help		// as needed
+  //    void *__copy_helper;            // as needed
+  //    void *__destroy_help            // as needed
   //    int X;
   //  } *
 
@@ -3249,7 +3235,7 @@ QualType ASTContext::BuildByRefType(const char *DeclName, QualType Ty) {
     Ty
   };
 
-  const char *FieldNames[] = {
+  llvm::StringRef FieldNames[] = {
     "__isa",
     "__forwarding",
     "__flags",
@@ -3326,7 +3312,7 @@ QualType ASTContext::getBlockParmType(
       const ValueDecl *D = BDRE->getDecl();
       FieldName = D->getIdentifier();
       if (BDRE->isByRef())
-        FieldType = BuildByRefType(D->getNameAsCString(), FieldType);
+        FieldType = BuildByRefType(D->getName(), FieldType);
     } else {
       // Padding.
       assert(isa<ConstantArrayType>(FieldType) &&
@@ -3885,15 +3871,14 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
     const IdentifierInfo *II = OI->getIdentifier();
     S += II->getName();
     S += '=';
-    llvm::SmallVector<FieldDecl*, 32> RecFields;
-    CollectObjCIvars(OI, RecFields);
-    for (unsigned i = 0, e = RecFields.size(); i != e; ++i) {
-      if (RecFields[i]->isBitField())
-        getObjCEncodingForTypeImpl(RecFields[i]->getType(), S, false, true,
-                                   RecFields[i]);
+    llvm::SmallVector<ObjCIvarDecl*, 32> Ivars;
+    DeepCollectObjCIvars(OI, true, Ivars);
+    for (unsigned i = 0, e = Ivars.size(); i != e; ++i) {
+      FieldDecl *Field = cast<FieldDecl>(Ivars[i]);
+      if (Field->isBitField())
+        getObjCEncodingForTypeImpl(Field->getType(), S, false, true, Field);
       else
-        getObjCEncodingForTypeImpl(RecFields[i]->getType(), S, false, true,
-                                   FD);
+        getObjCEncodingForTypeImpl(Field->getType(), S, false, true, FD);
     }
     S += '}';
     return;
@@ -4200,6 +4185,28 @@ static bool areCompatVectorTypes(const VectorType *LHS,
          LHS->getNumElements() == RHS->getNumElements();
 }
 
+bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
+                                          QualType SecondVec) {
+  assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
+  assert(SecondVec->isVectorType() && "SecondVec should be a vector type");
+
+  if (hasSameUnqualifiedType(FirstVec, SecondVec))
+    return true;
+
+  // AltiVec vectors types are identical to equivalent GCC vector types
+  const VectorType *First = FirstVec->getAs<VectorType>();
+  const VectorType *Second = SecondVec->getAs<VectorType>();
+  if ((((First->getAltiVecSpecific() == VectorType::AltiVec) &&
+        (Second->getAltiVecSpecific() == VectorType::NotAltiVec)) ||
+       ((First->getAltiVecSpecific() == VectorType::NotAltiVec) &&
+        (Second->getAltiVecSpecific() == VectorType::AltiVec))) &&
+      hasSameType(First->getElementType(), Second->getElementType()) &&
+      (First->getNumElements() == Second->getNumElements()))
+    return true;
+
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // ObjCQualifiedIdTypesAreCompatible - Compatibility testing for qualified id's.
 //===----------------------------------------------------------------------===//
@@ -4226,6 +4233,32 @@ bool ASTContext::QualifiedIdConformsQualifiedId(QualType lhs, QualType rhs) {
   return false;
 }
 
+/// ObjCQualifiedClassTypesAreCompatible - compare  Class<p,...> and
+/// Class<p1, ...>.
+bool ASTContext::ObjCQualifiedClassTypesAreCompatible(QualType lhs, 
+                                                      QualType rhs) {
+  const ObjCObjectPointerType *lhsQID = lhs->getAs<ObjCObjectPointerType>();
+  const ObjCObjectPointerType *rhsOPT = rhs->getAs<ObjCObjectPointerType>();
+  assert ((lhsQID && rhsOPT) && "ObjCQualifiedClassTypesAreCompatible");
+  
+  for (ObjCObjectPointerType::qual_iterator I = lhsQID->qual_begin(),
+       E = lhsQID->qual_end(); I != E; ++I) {
+    bool match = false;
+    ObjCProtocolDecl *lhsProto = *I;
+    for (ObjCObjectPointerType::qual_iterator J = rhsOPT->qual_begin(),
+         E = rhsOPT->qual_end(); J != E; ++J) {
+      ObjCProtocolDecl *rhsProto = *J;
+      if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto)) {
+        match = true;
+        break;
+      }
+    }
+    if (!match)
+      return false;
+  }
+  return true;
+}
+
 /// ObjCQualifiedIdTypesAreCompatible - We know that one of lhs/rhs is an
 /// ObjCQualifiedIDType.
 bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
@@ -4308,9 +4341,9 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
       if (ObjCInterfaceDecl *lhsID = lhsOPT->getInterfaceDecl()) {
         for (ObjCObjectPointerType::qual_iterator I = rhsQID->qual_begin(),
              E = rhsQID->qual_end(); I != E; ++I) {
-          // when comparing an id<P> on lhs with a static type on rhs,
-          // see if static class implements all of id's protocols, directly or
-          // through its super class and categories.
+          // when comparing an id<P> on rhs with a static type on lhs,
+          // static class must implement all of id's protocols directly or
+          // indirectly through its super class.
           if (lhsID->ClassImplementsProtocol(*I, true)) {
             match = true;
             break;
@@ -4365,7 +4398,11 @@ bool ASTContext::canAssignObjCInterfaces(const ObjCObjectPointerType *LHSOPT,
     return ObjCQualifiedIdTypesAreCompatible(QualType(LHSOPT,0),
                                              QualType(RHSOPT,0),
                                              false);
-
+  
+  if (LHS->isObjCQualifiedClass() && RHS->isObjCQualifiedClass())
+    return ObjCQualifiedClassTypesAreCompatible(QualType(LHSOPT,0),
+                                                QualType(RHSOPT,0));
+  
   // If we have 2 user-defined types, fall into that path.
   if (LHS->getInterface() && RHS->getInterface())
     return canAssignObjCInterfaces(LHS, RHS);
@@ -4541,15 +4578,22 @@ bool ASTContext::areComparableObjCPointerTypes(QualType LHS, QualType RHS) {
          canAssignObjCInterfaces(RHSOPT, LHSOPT);
 }
 
+bool ASTContext::canBindObjCObjectType(QualType To, QualType From) {
+  return canAssignObjCInterfaces(
+                getObjCObjectPointerType(To)->getAs<ObjCObjectPointerType>(),
+                getObjCObjectPointerType(From)->getAs<ObjCObjectPointerType>());
+}
+
 /// typesAreCompatible - C99 6.7.3p9: For two qualified types to be compatible,
 /// both shall have the identically qualified version of a compatible type.
 /// C99 6.2.7p1: Two types have compatible types if their types are the
 /// same. See 6.7.[2,3,5] for additional rules.
-bool ASTContext::typesAreCompatible(QualType LHS, QualType RHS) {
+bool ASTContext::typesAreCompatible(QualType LHS, QualType RHS,
+                                    bool CompareUnqualified) {
   if (getLangOptions().CPlusPlus)
     return hasSameType(LHS, RHS);
   
-  return !mergeTypes(LHS, RHS).isNull();
+  return !mergeTypes(LHS, RHS, false, CompareUnqualified).isNull();
 }
 
 bool ASTContext::typesAreBlockPointerCompatible(QualType LHS, QualType RHS) {
@@ -4557,7 +4601,8 @@ bool ASTContext::typesAreBlockPointerCompatible(QualType LHS, QualType RHS) {
 }
 
 QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs, 
-                                        bool OfBlockPointer) {
+                                        bool OfBlockPointer,
+                                        bool Unqualified) {
   const FunctionType *lbase = lhs->getAs<FunctionType>();
   const FunctionType *rbase = rhs->getAs<FunctionType>();
   const FunctionProtoType *lproto = dyn_cast<FunctionProtoType>(lbase);
@@ -4568,13 +4613,26 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
   // Check return type
   QualType retType;
   if (OfBlockPointer)
-    retType = mergeTypes(rbase->getResultType(), lbase->getResultType(), true);
+    retType = mergeTypes(rbase->getResultType(), lbase->getResultType(), true,
+                         Unqualified);
   else
-   retType = mergeTypes(lbase->getResultType(), rbase->getResultType());
+   retType = mergeTypes(lbase->getResultType(), rbase->getResultType(),
+                        false, Unqualified);
   if (retType.isNull()) return QualType();
-  if (getCanonicalType(retType) != getCanonicalType(lbase->getResultType()))
+  
+  if (Unqualified)
+    retType = retType.getUnqualifiedType();
+
+  CanQualType LRetType = getCanonicalType(lbase->getResultType());
+  CanQualType RRetType = getCanonicalType(rbase->getResultType());
+  if (Unqualified) {
+    LRetType = LRetType.getUnqualifiedType();
+    RRetType = RRetType.getUnqualifiedType();
+  }
+  
+  if (getCanonicalType(retType) != LRetType)
     allLTypes = false;
-  if (getCanonicalType(retType) != getCanonicalType(rbase->getResultType()))
+  if (getCanonicalType(retType) != RRetType)
     allRTypes = false;
   // FIXME: double check this
   // FIXME: should we error if lbase->getRegParmAttr() != 0 &&
@@ -4619,9 +4677,19 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     for (unsigned i = 0; i < lproto_nargs; i++) {
       QualType largtype = lproto->getArgType(i).getUnqualifiedType();
       QualType rargtype = rproto->getArgType(i).getUnqualifiedType();
-      QualType argtype = mergeTypes(largtype, rargtype, OfBlockPointer);
+      QualType argtype = mergeTypes(largtype, rargtype, OfBlockPointer,
+                                    Unqualified);
       if (argtype.isNull()) return QualType();
+      
+      if (Unqualified)
+        argtype = argtype.getUnqualifiedType();
+      
       types.push_back(argtype);
+      if (Unqualified) {
+        largtype = largtype.getUnqualifiedType();
+        rargtype = rargtype.getUnqualifiedType();
+      }
+      
       if (getCanonicalType(argtype) != getCanonicalType(largtype))
         allLTypes = false;
       if (getCanonicalType(argtype) != getCanonicalType(rargtype))
@@ -4677,7 +4745,8 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
 }
 
 QualType ASTContext::mergeTypes(QualType LHS, QualType RHS, 
-                                bool OfBlockPointer) {
+                                bool OfBlockPointer,
+                                bool Unqualified) {
   // C++ [expr]: If an expression initially has the type "reference to T", the
   // type is adjusted to "T" prior to any further analysis, the expression
   // designates the object or function denoted by the reference, and the
@@ -4685,6 +4754,11 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
   // the expression is a function call (possibly inside parentheses).
   assert(!LHS->getAs<ReferenceType>() && "LHS is a reference type?");
   assert(!RHS->getAs<ReferenceType>() && "RHS is a reference type?");
+
+  if (Unqualified) {
+    LHS = LHS.getUnqualifiedType();
+    RHS = RHS.getUnqualifiedType();
+  }
   
   QualType LHSCan = getCanonicalType(LHS),
            RHSCan = getCanonicalType(RHS);
@@ -4796,7 +4870,12 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
     // Merge two pointer types, while trying to preserve typedef info
     QualType LHSPointee = LHS->getAs<PointerType>()->getPointeeType();
     QualType RHSPointee = RHS->getAs<PointerType>()->getPointeeType();
-    QualType ResultType = mergeTypes(LHSPointee, RHSPointee);
+    if (Unqualified) {
+      LHSPointee = LHSPointee.getUnqualifiedType();
+      RHSPointee = RHSPointee.getUnqualifiedType();
+    }
+    QualType ResultType = mergeTypes(LHSPointee, RHSPointee, false, 
+                                     Unqualified);
     if (ResultType.isNull()) return QualType();
     if (getCanonicalType(LHSPointee) == getCanonicalType(ResultType))
       return LHS;
@@ -4809,7 +4888,12 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
     // Merge two block pointer types, while trying to preserve typedef info
     QualType LHSPointee = LHS->getAs<BlockPointerType>()->getPointeeType();
     QualType RHSPointee = RHS->getAs<BlockPointerType>()->getPointeeType();
-    QualType ResultType = mergeTypes(LHSPointee, RHSPointee, OfBlockPointer);
+    if (Unqualified) {
+      LHSPointee = LHSPointee.getUnqualifiedType();
+      RHSPointee = RHSPointee.getUnqualifiedType();
+    }
+    QualType ResultType = mergeTypes(LHSPointee, RHSPointee, OfBlockPointer,
+                                     Unqualified);
     if (ResultType.isNull()) return QualType();
     if (getCanonicalType(LHSPointee) == getCanonicalType(ResultType))
       return LHS;
@@ -4826,7 +4910,12 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
 
     QualType LHSElem = getAsArrayType(LHS)->getElementType();
     QualType RHSElem = getAsArrayType(RHS)->getElementType();
-    QualType ResultType = mergeTypes(LHSElem, RHSElem);
+    if (Unqualified) {
+      LHSElem = LHSElem.getUnqualifiedType();
+      RHSElem = RHSElem.getUnqualifiedType();
+    }
+    
+    QualType ResultType = mergeTypes(LHSElem, RHSElem, false, Unqualified);
     if (ResultType.isNull()) return QualType();
     if (LCAT && getCanonicalType(LHSElem) == getCanonicalType(ResultType))
       return LHS;
@@ -4860,7 +4949,7 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
                                   ArrayType::ArraySizeModifier(), 0);
   }
   case Type::FunctionNoProto:
-    return mergeFunctionTypes(LHS, RHS, OfBlockPointer);
+    return mergeFunctionTypes(LHS, RHS, OfBlockPointer, Unqualified);
   case Type::Record:
   case Type::Enum:
     return QualType();
@@ -5001,7 +5090,7 @@ unsigned ASTContext::getIntWidth(QualType T) {
 }
 
 QualType ASTContext::getCorrespondingUnsignedType(QualType T) {
-  assert(T->isSignedIntegerType() && "Unexpected type");
+  assert(T->hasSignedIntegerRepresentation() && "Unexpected type");
   
   // Turn <4 x signed int> -> <4 x unsigned int>
   if (const VectorType *VTy = T->getAs<VectorType>())
@@ -5381,8 +5470,8 @@ ASTContext::UsualArithmeticConversionsType(QualType lhs, QualType rhs) {
   // Finally, we have two differing integer types.
   // The rules for this case are in C99 6.3.1.8
   int compare = getIntegerTypeOrder(lhs, rhs);
-  bool lhsSigned = lhs->isSignedIntegerType(),
-       rhsSigned = rhs->isSignedIntegerType();
+  bool lhsSigned = lhs->hasSignedIntegerRepresentation(),
+       rhsSigned = rhs->hasSignedIntegerRepresentation();
   QualType destType;
   if (lhsSigned == rhsSigned) {
     // Same signedness; use the higher-ranked type
@@ -5405,3 +5494,173 @@ ASTContext::UsualArithmeticConversionsType(QualType lhs, QualType rhs) {
   }
   return destType;
 }
+
+GVALinkage ASTContext::GetGVALinkageForFunction(const FunctionDecl *FD) {
+  GVALinkage External = GVA_StrongExternal;
+
+  Linkage L = FD->getLinkage();
+  if (L == ExternalLinkage && getLangOptions().CPlusPlus &&
+      FD->getType()->getLinkage() == UniqueExternalLinkage)
+    L = UniqueExternalLinkage;
+  
+  switch (L) {
+  case NoLinkage:
+  case InternalLinkage:
+  case UniqueExternalLinkage:
+    return GVA_Internal;
+    
+  case ExternalLinkage:
+    switch (FD->getTemplateSpecializationKind()) {
+    case TSK_Undeclared:
+    case TSK_ExplicitSpecialization:
+      External = GVA_StrongExternal;
+      break;
+
+    case TSK_ExplicitInstantiationDefinition:
+      return GVA_ExplicitTemplateInstantiation;
+
+    case TSK_ExplicitInstantiationDeclaration:
+    case TSK_ImplicitInstantiation:
+      External = GVA_TemplateInstantiation;
+      break;
+    }
+  }
+
+  if (!FD->isInlined())
+    return External;
+    
+  if (!getLangOptions().CPlusPlus || FD->hasAttr<GNUInlineAttr>()) {
+    // GNU or C99 inline semantics. Determine whether this symbol should be
+    // externally visible.
+    if (FD->isInlineDefinitionExternallyVisible())
+      return External;
+
+    // C99 inline semantics, where the symbol is not externally visible.
+    return GVA_C99Inline;
+  }
+
+  // C++0x [temp.explicit]p9:
+  //   [ Note: The intent is that an inline function that is the subject of 
+  //   an explicit instantiation declaration will still be implicitly 
+  //   instantiated when used so that the body can be considered for 
+  //   inlining, but that no out-of-line copy of the inline function would be
+  //   generated in the translation unit. -- end note ]
+  if (FD->getTemplateSpecializationKind() 
+                                       == TSK_ExplicitInstantiationDeclaration)
+    return GVA_C99Inline;
+
+  return GVA_CXXInline;
+}
+
+GVALinkage ASTContext::GetGVALinkageForVariable(const VarDecl *VD) {
+  // If this is a static data member, compute the kind of template
+  // specialization. Otherwise, this variable is not part of a
+  // template.
+  TemplateSpecializationKind TSK = TSK_Undeclared;
+  if (VD->isStaticDataMember())
+    TSK = VD->getTemplateSpecializationKind();
+
+  Linkage L = VD->getLinkage();
+  if (L == ExternalLinkage && getLangOptions().CPlusPlus &&
+      VD->getType()->getLinkage() == UniqueExternalLinkage)
+    L = UniqueExternalLinkage;
+
+  switch (L) {
+  case NoLinkage:
+  case InternalLinkage:
+  case UniqueExternalLinkage:
+    return GVA_Internal;
+
+  case ExternalLinkage:
+    switch (TSK) {
+    case TSK_Undeclared:
+    case TSK_ExplicitSpecialization:
+      return GVA_StrongExternal;
+
+    case TSK_ExplicitInstantiationDeclaration:
+      llvm_unreachable("Variable should not be instantiated");
+      // Fall through to treat this like any other instantiation.
+        
+    case TSK_ExplicitInstantiationDefinition:
+      return GVA_ExplicitTemplateInstantiation;
+
+    case TSK_ImplicitInstantiation:
+      return GVA_TemplateInstantiation;      
+    }
+  }
+
+  return GVA_StrongExternal;
+}
+
+bool ASTContext::DeclMustBeEmitted(const Decl *D) {
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    if (!VD->isFileVarDecl())
+      return false;
+  } else if (!isa<FunctionDecl>(D))
+    return false;
+
+  // Weak references don't produce any output by themselves.
+  if (D->hasAttr<WeakRefAttr>())
+    return false;
+
+  // Aliases and used decls are required.
+  if (D->hasAttr<AliasAttr>() || D->hasAttr<UsedAttr>())
+    return true;
+
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    // Forward declarations aren't required.
+    if (!FD->isThisDeclarationADefinition())
+      return false;
+
+    // Constructors and destructors are required.
+    if (FD->hasAttr<ConstructorAttr>() || FD->hasAttr<DestructorAttr>())
+      return true;
+    
+    // The key function for a class is required.
+    if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) {
+      const CXXRecordDecl *RD = MD->getParent();
+      if (MD->isOutOfLine() && RD->isDynamicClass()) {
+        const CXXMethodDecl *KeyFunc = getKeyFunction(RD);
+        if (KeyFunc && KeyFunc->getCanonicalDecl() == MD->getCanonicalDecl())
+          return true;
+      }
+    }
+
+    GVALinkage Linkage = GetGVALinkageForFunction(FD);
+
+    // static, static inline, always_inline, and extern inline functions can
+    // always be deferred.  Normal inline functions can be deferred in C99/C++.
+    // Implicit template instantiations can also be deferred in C++.
+    if (Linkage == GVA_Internal  || Linkage == GVA_C99Inline ||
+        Linkage == GVA_CXXInline || Linkage == GVA_TemplateInstantiation)
+      return false;
+    return true;
+  }
+
+  const VarDecl *VD = cast<VarDecl>(D);
+  assert(VD->isFileVarDecl() && "Expected file scoped var");
+
+  if (VD->isThisDeclarationADefinition() == VarDecl::DeclarationOnly)
+    return false;
+
+  // Structs that have non-trivial constructors or destructors are required.
+
+  // FIXME: Handle references.
+  if (const RecordType *RT = VD->getType()->getAs<RecordType>()) {
+    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+      if (RD->hasDefinition() &&
+          (!RD->hasTrivialConstructor() || !RD->hasTrivialDestructor()))
+        return true;
+    }
+  }
+
+  GVALinkage L = GetGVALinkageForVariable(VD);
+  if (L == GVA_Internal || L == GVA_TemplateInstantiation) {
+    if (!(VD->getInit() && VD->getInit()->HasSideEffects(*this)))
+      return false;
+  }
+
+  return true;
+}
+
+CXXABI::~CXXABI() {}
diff --git a/lib/AST/ASTDiagnostic.cpp b/lib/AST/ASTDiagnostic.cpp
index 0d609bfa6d39..23f323d0286a 100644
--- a/lib/AST/ASTDiagnostic.cpp
+++ b/lib/AST/ASTDiagnostic.cpp
@@ -151,10 +151,13 @@ ConvertTypeToDiagnosticString(ASTContext &Context, QualType Ty,
     bool ShouldAKA = false;
     QualType DesugaredTy = Desugar(Context, Ty, ShouldAKA);
     if (ShouldAKA) {
-      S = "'"+S+"' (aka '";
-      S += DesugaredTy.getAsString(Context.PrintingPolicy);
-      S += "')";
-      return S;
+      std::string D = DesugaredTy.getAsString(Context.PrintingPolicy);
+      if (D != S) {
+        S = "'" + S + "' (aka '";
+        S += D;
+        S += "')";
+        return S;
+      }
     }
   }
 
diff --git a/lib/AST/ASTImporter.cpp b/lib/AST/ASTImporter.cpp
index 8d347d171665..2edd09c067e9 100644
--- a/lib/AST/ASTImporter.cpp
+++ b/lib/AST/ASTImporter.cpp
@@ -19,7 +19,6 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/StmtVisitor.h"
-#include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
@@ -82,6 +81,8 @@ namespace {
     bool ImportDeclParts(NamedDecl *D, DeclContext *&DC, 
                          DeclContext *&LexicalDC, DeclarationName &Name, 
                          SourceLocation &Loc);
+    void ImportDeclarationNameLoc(const DeclarationNameInfo &From,
+                                  DeclarationNameInfo& To);
     void ImportDeclContext(DeclContext *FromDC);
     bool IsStructuralMatch(RecordDecl *FromRecord, RecordDecl *ToRecord);
     bool IsStructuralMatch(EnumDecl *FromEnum, EnumDecl *ToRecord);
@@ -1385,6 +1386,40 @@ bool ASTNodeImporter::ImportDeclParts(NamedDecl *D, DeclContext *&DC,
   return false;
 }
 
+void
+ASTNodeImporter::ImportDeclarationNameLoc(const DeclarationNameInfo &From,
+                                          DeclarationNameInfo& To) {
+  // NOTE: To.Name and To.Loc are already imported.
+  // We only have to import To.LocInfo.
+  switch (To.getName().getNameKind()) {
+  case DeclarationName::Identifier:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXUsingDirective:
+    return;
+
+  case DeclarationName::CXXOperatorName: {
+    SourceRange Range = From.getCXXOperatorNameRange();
+    To.setCXXOperatorNameRange(Importer.Import(Range));
+    return;
+  }
+  case DeclarationName::CXXLiteralOperatorName: {
+    SourceLocation Loc = From.getCXXLiteralOperatorNameLoc();
+    To.setCXXLiteralOperatorNameLoc(Importer.Import(Loc));
+    return;
+  }
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName: {
+    TypeSourceInfo *FromTInfo = From.getNamedTypeInfo();
+    To.setNamedTypeInfo(Importer.Import(FromTInfo));
+    return;
+  }
+    assert(0 && "Unknown name kind.");
+  }
+}
+
 void ASTNodeImporter::ImportDeclContext(DeclContext *FromDC) {
   for (DeclContext::decl_iterator From = FromDC->decls_begin(),
                                FromEnd = FromDC->decls_end();
@@ -1752,7 +1787,7 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
                                  Base1->isVirtual(),
                                  Base1->isBaseOfClass(),
                                  Base1->getAccessSpecifierAsWritten(),
-                                 T));
+                                 Importer.Import(Base1->getTypeSourceInfo())));
       }
       if (!Bases.empty())
         D2CXX->setBases(Bases.data(), Bases.size());
@@ -1822,7 +1857,7 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   SourceLocation Loc;
   if (ImportDeclParts(D, DC, LexicalDC, Name, Loc))
     return 0;
-  
+
   // Try to find a function in our own ("to") context with the same name, same
   // type, and in the same context as the function we're importing.
   if (!LexicalDC->isFunctionOrMethod()) {
@@ -1871,6 +1906,10 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
     }    
   }
 
+  DeclarationNameInfo NameInfo(Name, Loc);
+  // Import additional name location/type info.
+  ImportDeclarationNameLoc(D->getNameInfo(), NameInfo);
+
   // Import the type.
   QualType T = Importer.Import(D->getType());
   if (T.isNull())
@@ -1893,26 +1932,26 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   if (CXXConstructorDecl *FromConstructor = dyn_cast<CXXConstructorDecl>(D)) {
     ToFunction = CXXConstructorDecl::Create(Importer.getToContext(),
                                             cast<CXXRecordDecl>(DC),
-                                            Loc, Name, T, TInfo, 
+                                            NameInfo, T, TInfo, 
                                             FromConstructor->isExplicit(),
                                             D->isInlineSpecified(), 
                                             D->isImplicit());
   } else if (isa<CXXDestructorDecl>(D)) {
     ToFunction = CXXDestructorDecl::Create(Importer.getToContext(),
                                            cast<CXXRecordDecl>(DC),
-                                           Loc, Name, T, 
+                                           NameInfo, T, 
                                            D->isInlineSpecified(),
                                            D->isImplicit());
   } else if (CXXConversionDecl *FromConversion
                                            = dyn_cast<CXXConversionDecl>(D)) {
     ToFunction = CXXConversionDecl::Create(Importer.getToContext(), 
                                            cast<CXXRecordDecl>(DC),
-                                           Loc, Name, T, TInfo,
+                                           NameInfo, T, TInfo,
                                            D->isInlineSpecified(),
                                            FromConversion->isExplicit());
   } else {
-    ToFunction = FunctionDecl::Create(Importer.getToContext(), DC, Loc, 
-                                      Name, T, TInfo, D->getStorageClass(), 
+    ToFunction = FunctionDecl::Create(Importer.getToContext(), DC,
+                                      NameInfo, T, TInfo, D->getStorageClass(),
                                       D->getStorageClassAsWritten(),
                                       D->isInlineSpecified(),
                                       D->hasWrittenPrototype());
@@ -2026,7 +2065,7 @@ Decl *ASTNodeImporter::VisitObjCIvarDecl(ObjCIvarDecl *D) {
                                               cast<ObjCContainerDecl>(DC),
                                               Loc, Name.getAsIdentifierInfo(),
                                               T, TInfo, D->getAccessControl(),
-                                              BitWidth);
+                                              BitWidth, D->getSynthesize());
   ToIvar->setLexicalDeclContext(LexicalDC);
   Importer.Imported(D, ToIvar);
   LexicalDC->addDecl(ToIvar);
@@ -2299,6 +2338,7 @@ Decl *ASTNodeImporter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
                              D->isInstanceMethod(),
                              D->isVariadic(),
                              D->isSynthesized(),
+                             D->isDefined(),
                              D->getImplementationControl());
 
   // FIXME: When we decide to merge method definitions, we'll need to
@@ -2513,6 +2553,8 @@ Decl *ASTNodeImporter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
     llvm::SmallVector<SourceLocation, 4> ProtocolLocs;
     ObjCInterfaceDecl::protocol_loc_iterator 
       FromProtoLoc = D->protocol_loc_begin();
+    
+    // FIXME: Should we be usng all_referenced_protocol_begin() here?
     for (ObjCInterfaceDecl::protocol_iterator FromProto = D->protocol_begin(),
                                            FromProtoEnd = D->protocol_end();
        FromProto != FromProtoEnd;
@@ -2778,8 +2820,9 @@ Expr *ASTNodeImporter::VisitIntegerLiteral(IntegerLiteral *E) {
   if (T.isNull())
     return 0;
 
-  return new (Importer.getToContext()) 
-    IntegerLiteral(E->getValue(), T, Importer.Import(E->getLocation()));
+  return IntegerLiteral::Create(Importer.getToContext(), 
+                                E->getValue(), T,
+                                Importer.Import(E->getLocation()));
 }
 
 Expr *ASTNodeImporter::VisitCharacterLiteral(CharacterLiteral *E) {
@@ -2886,6 +2929,13 @@ Expr *ASTNodeImporter::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
                                           Importer.Import(E->getOperatorLoc()));
 }
 
+bool ImportCastPath(CastExpr *E, CXXCastPath &Path) {
+  if (E->path_empty()) return false;
+
+  // TODO: import cast paths
+  return true;
+}
+
 Expr *ASTNodeImporter::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   QualType T = Importer.Import(E->getType());
   if (T.isNull())
@@ -2894,13 +2944,13 @@ Expr *ASTNodeImporter::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   Expr *SubExpr = Importer.Import(E->getSubExpr());
   if (!SubExpr)
     return 0;
-  
-  // FIXME: Initialize the base path.
-  assert(E->getBasePath().empty() && "FIXME: Must copy base path!");
-  CXXBaseSpecifierArray BasePath;
-  return new (Importer.getToContext()) ImplicitCastExpr(T, E->getCastKind(),
-                                                        SubExpr, BasePath,
-                                                        E->isLvalueCast());
+
+  CXXCastPath BasePath;
+  if (ImportCastPath(E, BasePath))
+    return 0;
+
+  return ImplicitCastExpr::Create(Importer.getToContext(), T, E->getCastKind(),
+                                  SubExpr, &BasePath, E->getValueKind());
 }
 
 Expr *ASTNodeImporter::VisitCStyleCastExpr(CStyleCastExpr *E) {
@@ -2916,13 +2966,14 @@ Expr *ASTNodeImporter::VisitCStyleCastExpr(CStyleCastExpr *E) {
   if (!TInfo && E->getTypeInfoAsWritten())
     return 0;
   
-  // FIXME: Initialize the base path.
-  assert(E->getBasePath().empty() && "FIXME: Must copy base path!");
-  CXXBaseSpecifierArray BasePath;
-  return new (Importer.getToContext()) CStyleCastExpr(T, E->getCastKind(),
-                                                      SubExpr, BasePath, TInfo,
-                                            Importer.Import(E->getLParenLoc()),
-                                            Importer.Import(E->getRParenLoc()));
+  CXXCastPath BasePath;
+  if (ImportCastPath(E, BasePath))
+    return 0;
+
+  return CStyleCastExpr::Create(Importer.getToContext(), T, E->getCastKind(),
+                                SubExpr, &BasePath, TInfo,
+                                Importer.Import(E->getLParenLoc()),
+                                Importer.Import(E->getRParenLoc()));
 }
 
 ASTImporter::ASTImporter(Diagnostic &Diags,
@@ -2964,8 +3015,7 @@ TypeSourceInfo *ASTImporter::Import(TypeSourceInfo *FromTSI) {
     return FromTSI;
 
   // FIXME: For now we just create a "trivial" type source info based
-  // on the type and a seingle location. Implement a real version of
-  // this.
+  // on the type and a single location. Implement a real version of this.
   QualType T = Import(FromTSI->getType());
   if (T.isNull())
     return 0;
diff --git a/lib/AST/AttrImpl.cpp b/lib/AST/AttrImpl.cpp
index b09ba895c019..3ca7d4d01d1c 100644
--- a/lib/AST/AttrImpl.cpp
+++ b/lib/AST/AttrImpl.cpp
@@ -13,198 +13,10 @@
 
 #include "clang/AST/Attr.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Type.h"
+#include "clang/AST/Expr.h"
 using namespace clang;
 
-void Attr::Destroy(ASTContext &C) {
-  if (Next) {
-    Next->Destroy(C);
-    Next = 0;
-  }
-  this->~Attr();
-  C.Deallocate((void*)this);
-}
+Attr::~Attr() { }
 
-AttrWithString::AttrWithString(attr::Kind AK, ASTContext &C, llvm::StringRef s)
-  : Attr(AK) {
-  assert(!s.empty());
-  StrLen = s.size();
-  Str = new (C) char[StrLen];
-  memcpy(const_cast<char*>(Str), s.data(), StrLen);
-}
-
-void AttrWithString::Destroy(ASTContext &C) {
-  C.Deallocate(const_cast<char*>(Str));  
-  Attr::Destroy(C);
-}
-
-void AttrWithString::ReplaceString(ASTContext &C, llvm::StringRef newS) {
-  if (newS.size() > StrLen) {
-    C.Deallocate(const_cast<char*>(Str));
-    Str = new (C) char[newS.size()];
-  }
-  StrLen = newS.size();
-  memcpy(const_cast<char*>(Str), newS.data(), StrLen);
-}
-
-void FormatAttr::setType(ASTContext &C, llvm::StringRef type) {
-  ReplaceString(C, type);
-}
-
-NonNullAttr::NonNullAttr(ASTContext &C, unsigned* arg_nums, unsigned size)
-  : Attr(attr::NonNull), ArgNums(0), Size(0) {
-  if (size == 0)
-    return;
-  assert(arg_nums);
-  ArgNums = new (C) unsigned[size];
-  Size = size;
-  memcpy(ArgNums, arg_nums, sizeof(*ArgNums)*size);
-}
-
-void NonNullAttr::Destroy(ASTContext &C) {
-  if (ArgNums)
-    C.Deallocate(ArgNums);
-  Attr::Destroy(C);
-}
-
-#define DEF_SIMPLE_ATTR_CLONE(ATTR)                                     \
-  Attr *ATTR##Attr::clone(ASTContext &C) const {                        \
-    return ::new (C) ATTR##Attr;                                        \
-  }
-
-// FIXME: Can we use variadic macro to define DEF_SIMPLE_ATTR_CLONE for
-// "non-simple" classes?
-
-DEF_SIMPLE_ATTR_CLONE(AlignMac68k)
-DEF_SIMPLE_ATTR_CLONE(AlwaysInline)
-DEF_SIMPLE_ATTR_CLONE(AnalyzerNoReturn)
-DEF_SIMPLE_ATTR_CLONE(BaseCheck)
-DEF_SIMPLE_ATTR_CLONE(CDecl)
-DEF_SIMPLE_ATTR_CLONE(CFReturnsNotRetained)
-DEF_SIMPLE_ATTR_CLONE(CFReturnsRetained)
-DEF_SIMPLE_ATTR_CLONE(Const)
-DEF_SIMPLE_ATTR_CLONE(DLLExport)
-DEF_SIMPLE_ATTR_CLONE(DLLImport)
-DEF_SIMPLE_ATTR_CLONE(Deprecated)
-DEF_SIMPLE_ATTR_CLONE(FastCall)
-DEF_SIMPLE_ATTR_CLONE(Final)
-DEF_SIMPLE_ATTR_CLONE(Hiding)
-DEF_SIMPLE_ATTR_CLONE(Malloc)
-DEF_SIMPLE_ATTR_CLONE(NSReturnsNotRetained)
-DEF_SIMPLE_ATTR_CLONE(NSReturnsRetained)
-DEF_SIMPLE_ATTR_CLONE(NoDebug)
-DEF_SIMPLE_ATTR_CLONE(NoInline)
-DEF_SIMPLE_ATTR_CLONE(NoInstrumentFunction)
-DEF_SIMPLE_ATTR_CLONE(NoReturn)
-DEF_SIMPLE_ATTR_CLONE(NoThrow)
-DEF_SIMPLE_ATTR_CLONE(ObjCException)
-DEF_SIMPLE_ATTR_CLONE(ObjCNSObject)
-DEF_SIMPLE_ATTR_CLONE(Override)
-DEF_SIMPLE_ATTR_CLONE(Packed)
-DEF_SIMPLE_ATTR_CLONE(Pure)
-DEF_SIMPLE_ATTR_CLONE(StdCall)
-DEF_SIMPLE_ATTR_CLONE(ThisCall)
-DEF_SIMPLE_ATTR_CLONE(TransparentUnion)
-DEF_SIMPLE_ATTR_CLONE(Unavailable)
-DEF_SIMPLE_ATTR_CLONE(Unused)
-DEF_SIMPLE_ATTR_CLONE(Used)
-DEF_SIMPLE_ATTR_CLONE(WarnUnusedResult)
-DEF_SIMPLE_ATTR_CLONE(Weak)
-DEF_SIMPLE_ATTR_CLONE(WeakImport)
-DEF_SIMPLE_ATTR_CLONE(WeakRef)
-DEF_SIMPLE_ATTR_CLONE(X86ForceAlignArgPointer)
-
-Attr* MaxFieldAlignmentAttr::clone(ASTContext &C) const {
-  return ::new (C) MaxFieldAlignmentAttr(Alignment);
-}
-
-Attr* AlignedAttr::clone(ASTContext &C) const {
-  return ::new (C) AlignedAttr(Alignment);
-}
-
-Attr* AnnotateAttr::clone(ASTContext &C) const {
-  return ::new (C) AnnotateAttr(C, getAnnotation());
-}
-
-Attr *AsmLabelAttr::clone(ASTContext &C) const {
-  return ::new (C) AsmLabelAttr(C, getLabel());
-}
-
-Attr *AliasAttr::clone(ASTContext &C) const {
-  return ::new (C) AliasAttr(C, getAliasee());
-}
-
-Attr *ConstructorAttr::clone(ASTContext &C) const {
-  return ::new (C) ConstructorAttr(priority);
-}
-
-Attr *DestructorAttr::clone(ASTContext &C) const {
-  return ::new (C) DestructorAttr(priority);
-}
-
-Attr *IBOutletAttr::clone(ASTContext &C) const {
-  return ::new (C) IBOutletAttr;
-}
-
-Attr *IBOutletCollectionAttr::clone(ASTContext &C) const {
-  return ::new (C) IBOutletCollectionAttr(D);
-}
-
-Attr *IBActionAttr::clone(ASTContext &C) const {
-  return ::new (C) IBActionAttr;
-}
-
-Attr *GNUInlineAttr::clone(ASTContext &C) const {
-  return ::new (C) GNUInlineAttr;
-}
-
-Attr *SectionAttr::clone(ASTContext &C) const {
-  return ::new (C) SectionAttr(C, getName());
-}
-
-Attr *NonNullAttr::clone(ASTContext &C) const {
-  return ::new (C) NonNullAttr(C, ArgNums, Size);
-}
-
-Attr *FormatAttr::clone(ASTContext &C) const {
-  return ::new (C) FormatAttr(C, getType(), formatIdx, firstArg);
-}
-
-Attr *FormatArgAttr::clone(ASTContext &C) const {
-  return ::new (C) FormatArgAttr(formatIdx);
-}
-
-Attr *SentinelAttr::clone(ASTContext &C) const {
-  return ::new (C) SentinelAttr(sentinel, NullPos);
-}
-
-Attr *VisibilityAttr::clone(ASTContext &C) const {
-  return ::new (C) VisibilityAttr(VisibilityType);
-}
-
-Attr *OverloadableAttr::clone(ASTContext &C) const {
-  return ::new (C) OverloadableAttr;
-}
-
-Attr *BlocksAttr::clone(ASTContext &C) const {
-  return ::new (C) BlocksAttr(BlocksAttrType);
-}
-
-Attr *CleanupAttr::clone(ASTContext &C) const {
-  return ::new (C) CleanupAttr(FD);
-}
-
-Attr *RegparmAttr::clone(ASTContext &C) const {
-  return ::new (C) RegparmAttr(NumParams);
-}
-
-Attr *ReqdWorkGroupSizeAttr::clone(ASTContext &C) const {
-  return ::new (C) ReqdWorkGroupSizeAttr(X, Y, Z);
-}
-
-Attr *InitPriorityAttr::clone(ASTContext &C) const {
-  return ::new (C) InitPriorityAttr(Priority);
-}
-
-Attr *MSP430InterruptAttr::clone(ASTContext &C) const {
-  return ::new (C) MSP430InterruptAttr(Number);
-}
+#include "clang/AST/AttrImpl.inc"
diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt
index 407ed95f3ee1..82a81ec42411 100644
--- a/lib/AST/CMakeLists.txt
+++ b/lib/AST/CMakeLists.txt
@@ -23,6 +23,8 @@ add_clang_library(clangAST
   ExprCXX.cpp
   FullExpr.cpp
   InheritViz.cpp
+  ItaniumCXXABI.cpp
+  MicrosoftCXXABI.cpp
   NestedNameSpecifier.cpp
   ParentMap.cpp
   RecordLayout.cpp
@@ -41,4 +43,4 @@ add_clang_library(clangAST
   )
 
 add_dependencies(clangAST ClangARMNeon ClangAttrClasses ClangAttrList 
-                 ClangDiagnosticAST ClangDeclNodes ClangStmtNodes)
+                 ClangAttrImpl ClangDiagnosticAST ClangDeclNodes ClangStmtNodes)
diff --git a/lib/AST/CXXABI.h b/lib/AST/CXXABI.h
new file mode 100644
index 000000000000..4b38d7afb6a4
--- /dev/null
+++ b/lib/AST/CXXABI.h
@@ -0,0 +1,39 @@
+//===----- CXXABI.h - Interface to C++ ABIs ---------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides an abstract class for C++ AST support. Concrete
+// subclasses of this implement AST support for specific C++ ABIs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_CXXABI_H
+#define LLVM_CLANG_AST_CXXABI_H
+
+namespace clang {
+
+class ASTContext;
+class MemberPointerType;
+
+/// Implements C++ ABI-specific semantic analysis functions.
+class CXXABI {
+public:
+  virtual ~CXXABI();
+
+  /// Returns the size of a member pointer in multiples of the target
+  /// pointer size.
+  virtual unsigned getMemberPointerSize(const MemberPointerType *MPT) const = 0;
+};
+
+/// Creates an instance of a C++ ABI class.
+CXXABI *CreateARMCXXABI(ASTContext &Ctx);
+CXXABI *CreateItaniumCXXABI(ASTContext &Ctx);
+CXXABI *CreateMicrosoftCXXABI(ASTContext &Ctx);
+}
+
+#endif
diff --git a/lib/AST/Decl.cpp b/lib/AST/Decl.cpp
index 149938fc5c86..b7be02d74533 100644
--- a/lib/AST/Decl.cpp
+++ b/lib/AST/Decl.cpp
@@ -97,8 +97,14 @@ static Linkage getLinkageForTemplateArgumentList(const TemplateArgument *Args,
   return L;
 }
 
+static Linkage
+getLinkageForTemplateArgumentList(const TemplateArgumentList &TArgs) {
+  return getLinkageForTemplateArgumentList(TArgs.getFlatArgumentList(), 
+                                           TArgs.flat_size());
+}
+
 static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
-  assert(D->getDeclContext()->getLookupContext()->isFileContext() &&
+  assert(D->getDeclContext()->getRedeclContext()->isFileContext() &&
          "Not a name having namespace scope");
   ASTContext &Context = D->getASTContext();
 
@@ -110,7 +116,7 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
   // (This bullet corresponds to C99 6.2.2p3.)
   if (const VarDecl *Var = dyn_cast<VarDecl>(D)) {
     // Explicitly declared static.
-    if (Var->getStorageClass() == VarDecl::Static)
+    if (Var->getStorageClass() == SC_Static)
       return InternalLinkage;
 
     // - an object or reference that is explicitly declared const
@@ -119,8 +125,8 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
     // (there is no equivalent in C99)
     if (Context.getLangOptions().CPlusPlus &&
         Var->getType().isConstant(Context) && 
-        Var->getStorageClass() != VarDecl::Extern &&
-        Var->getStorageClass() != VarDecl::PrivateExtern) {
+        Var->getStorageClass() != SC_Extern &&
+        Var->getStorageClass() != SC_PrivateExtern) {
       bool FoundExtern = false;
       for (const VarDecl *PrevVar = Var->getPreviousDeclaration();
            PrevVar && !FoundExtern; 
@@ -143,7 +149,7 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
       Function = cast<FunctionDecl>(D);
 
     // Explicitly declared static.
-    if (Function->getStorageClass() == FunctionDecl::Static)
+    if (Function->getStorageClass() == SC_Static)
       return InternalLinkage;
   } else if (const FieldDecl *Field = dyn_cast<FieldDecl>(D)) {
     //   - a data member of an anonymous union.
@@ -159,8 +165,8 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
   //     - an object or reference, unless it has internal linkage; or
   if (const VarDecl *Var = dyn_cast<VarDecl>(D)) {
     if (!Context.getLangOptions().CPlusPlus &&
-        (Var->getStorageClass() == VarDecl::Extern ||
-         Var->getStorageClass() == VarDecl::PrivateExtern)) {
+        (Var->getStorageClass() == SC_Extern ||
+         Var->getStorageClass() == SC_PrivateExtern)) {
       // C99 6.2.2p4:
       //   For an identifier declared with the storage-class specifier
       //   extern in a scope in which a prior declaration of that
@@ -194,9 +200,9 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
     //   as if it were declared with the storage-class specifier
     //   extern.
     if (!Context.getLangOptions().CPlusPlus &&
-        (Function->getStorageClass() == FunctionDecl::Extern ||
-         Function->getStorageClass() == FunctionDecl::PrivateExtern ||
-         Function->getStorageClass() == FunctionDecl::None)) {
+        (Function->getStorageClass() == SC_Extern ||
+         Function->getStorageClass() == SC_PrivateExtern ||
+         Function->getStorageClass() == SC_None)) {
       // C99 6.2.2p4:
       //   For an identifier declared with the storage-class specifier
       //   extern in a scope in which a prior declaration of that
@@ -219,10 +225,7 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
                                = Function->getTemplateSpecializationInfo()) {
       Linkage L = SpecInfo->getTemplate()->getLinkage();
       const TemplateArgumentList &TemplateArgs = *SpecInfo->TemplateArguments;
-      L = minLinkage(L, 
-                     getLinkageForTemplateArgumentList(
-                                          TemplateArgs.getFlatArgumentList(), 
-                                          TemplateArgs.flat_size()));
+      L = minLinkage(L, getLinkageForTemplateArgumentList(TemplateArgs));
       return L;
     }
 
@@ -245,9 +248,7 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
       if (const ClassTemplateSpecializationDecl *Spec
             = dyn_cast<ClassTemplateSpecializationDecl>(Tag)) {
         const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
-        Linkage L = getLinkageForTemplateArgumentList(
-                                          TemplateArgs.getFlatArgumentList(),
-                                                 TemplateArgs.flat_size());
+        Linkage L = getLinkageForTemplateArgumentList(TemplateArgs);
         return minLinkage(L, Spec->getSpecializedTemplate()->getLinkage());
       }
 
@@ -279,6 +280,47 @@ static Linkage getLinkageForNamespaceScopeDecl(const NamedDecl *D) {
   return NoLinkage;
 }
 
+static Linkage getLinkageForClassMember(const NamedDecl *D) {
+  if (!(isa<CXXMethodDecl>(D) ||
+        isa<VarDecl>(D) ||
+        (isa<TagDecl>(D) &&
+         (D->getDeclName() || cast<TagDecl>(D)->getTypedefForAnonDecl()))))
+    return NoLinkage;
+
+  // Class members only have linkage if their class has external linkage.
+  Linkage L = cast<RecordDecl>(D->getDeclContext())->getLinkage();
+  if (!isExternalLinkage(L)) return NoLinkage;
+
+  // If the class already has unique-external linkage, we can't improve.
+  if (L == UniqueExternalLinkage) return UniqueExternalLinkage;
+
+  // If this is a method template specialization, use the linkage for
+  // the template parameters and arguments.
+  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D)) {
+    if (FunctionTemplateSpecializationInfo *SpecInfo
+           = MD->getTemplateSpecializationInfo()) {
+      Linkage ArgLinkage =
+        getLinkageForTemplateArgumentList(*SpecInfo->TemplateArguments);
+      Linkage ParamLinkage =
+        getLinkageForTemplateParameterList(
+                           SpecInfo->getTemplate()->getTemplateParameters());
+      return minLinkage(ArgLinkage, ParamLinkage);
+    }
+
+  // Similarly for member class template specializations.
+  } else if (const ClassTemplateSpecializationDecl *Spec
+               = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
+    Linkage ArgLinkage =
+      getLinkageForTemplateArgumentList(Spec->getTemplateArgs());
+    Linkage ParamLinkage =
+      getLinkageForTemplateParameterList(
+                    Spec->getSpecializedTemplate()->getTemplateParameters());
+    return minLinkage(ArgLinkage, ParamLinkage);
+  }
+
+  return ExternalLinkage;
+}
+
 Linkage NamedDecl::getLinkage() const {
 
   // Objective-C: treat all Objective-C declarations as having external
@@ -303,7 +345,7 @@ Linkage NamedDecl::getLinkage() const {
   }
 
   // Handle linkage for namespace-scope names.
-  if (getDeclContext()->getLookupContext()->isFileContext())
+  if (getDeclContext()->getRedeclContext()->isFileContext())
     if (Linkage L = getLinkageForNamespaceScopeDecl(this))
       return L;
   
@@ -314,14 +356,8 @@ Linkage NamedDecl::getLinkage() const {
   //   that the class or enumeration has the typedef name for linkage
   //   purposes (7.1.3), has external linkage if the name of the class
   //   has external linkage.
-  if (getDeclContext()->isRecord() &&
-      (isa<CXXMethodDecl>(this) || isa<VarDecl>(this) ||
-       (isa<TagDecl>(this) &&
-        (getDeclName() || cast<TagDecl>(this)->getTypedefForAnonDecl())))) {
-    Linkage L = cast<RecordDecl>(getDeclContext())->getLinkage();
-    if (isExternalLinkage(L))
-      return L;
-  }
+  if (getDeclContext()->isRecord())
+    return getLinkageForClassMember(this);
 
   // C++ [basic.link]p6:
   //   The name of a function declared in block scope and the name of
@@ -347,8 +383,8 @@ Linkage NamedDecl::getLinkage() const {
     }
 
     if (const VarDecl *Var = dyn_cast<VarDecl>(this))
-      if (Var->getStorageClass() == VarDecl::Extern ||
-          Var->getStorageClass() == VarDecl::PrivateExtern) {
+      if (Var->getStorageClass() == SC_Extern ||
+          Var->getStorageClass() == SC_PrivateExtern) {
         if (Var->getPreviousDeclaration())
           if (Linkage L = Var->getPreviousDeclaration()->getLinkage())
             return L;
@@ -531,13 +567,6 @@ static SourceLocation getTemplateOrInnerLocStart(const DeclT *decl) {
     return decl->getInnerLocStart();
 }
 
-DeclaratorDecl::~DeclaratorDecl() {}
-void DeclaratorDecl::Destroy(ASTContext &C) {
-  if (hasExtInfo())
-    C.Deallocate(getExtInfo());
-  ValueDecl::Destroy(C);
-}
-
 SourceLocation DeclaratorDecl::getTypeSpecStartLoc() const {
   TypeSourceInfo *TSI = getTypeSourceInfo();
   if (TSI) return TSI->getTypeLoc().getBeginLoc();
@@ -602,24 +631,18 @@ QualifierInfo::setTemplateParameterListsInfo(ASTContext &Context,
   }
 }
 
-void QualifierInfo::Destroy(ASTContext &Context) {
-  // FIXME: Deallocate template parameter lists themselves!
-  if (TemplParamLists)
-    Context.Deallocate(TemplParamLists);
-}
-
 //===----------------------------------------------------------------------===//
 // VarDecl Implementation
 //===----------------------------------------------------------------------===//
 
 const char *VarDecl::getStorageClassSpecifierString(StorageClass SC) {
   switch (SC) {
-  case VarDecl::None:          break;
-  case VarDecl::Auto:          return "auto"; break;
-  case VarDecl::Extern:        return "extern"; break;
-  case VarDecl::PrivateExtern: return "__private_extern__"; break;
-  case VarDecl::Register:      return "register"; break;
-  case VarDecl::Static:        return "static"; break;
+  case SC_None:          break;
+  case SC_Auto:          return "auto"; break;
+  case SC_Extern:        return "extern"; break;
+  case SC_PrivateExtern: return "__private_extern__"; break;
+  case SC_Register:      return "register"; break;
+  case SC_Static:        return "static"; break;
   }
 
   assert(0 && "Invalid storage class");
@@ -632,22 +655,6 @@ VarDecl *VarDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
   return new (C) VarDecl(Var, DC, L, Id, T, TInfo, S, SCAsWritten);
 }
 
-void VarDecl::Destroy(ASTContext& C) {
-  Expr *Init = getInit();
-  if (Init) {
-    Init->Destroy(C);
-    if (EvaluatedStmt *Eval = this->Init.dyn_cast<EvaluatedStmt *>()) {
-      Eval->~EvaluatedStmt();
-      C.Deallocate(Eval);
-    }
-  }
-  this->~VarDecl();
-  DeclaratorDecl::Destroy(C);
-}
-
-VarDecl::~VarDecl() {
-}
-
 SourceLocation VarDecl::getInnerLocStart() const {
   SourceLocation Start = getTypeSpecStartLoc();
   if (Start.isInvalid())
@@ -665,14 +672,14 @@ bool VarDecl::isExternC() const {
   ASTContext &Context = getASTContext();
   if (!Context.getLangOptions().CPlusPlus)
     return (getDeclContext()->isTranslationUnit() &&
-            getStorageClass() != Static) ||
+            getStorageClass() != SC_Static) ||
       (getDeclContext()->isFunctionOrMethod() && hasExternalStorage());
 
   for (const DeclContext *DC = getDeclContext(); !DC->isTranslationUnit();
        DC = DC->getParent()) {
     if (const LinkageSpecDecl *Linkage = dyn_cast<LinkageSpecDecl>(DC))  {
       if (Linkage->getLanguage() == LinkageSpecDecl::lang_c)
-        return getStorageClass() != Static;
+        return getStorageClass() != SC_Static;
 
       break;
     }
@@ -717,8 +724,8 @@ VarDecl::DefinitionKind VarDecl::isThisDeclarationADefinition() const {
   if (hasExternalStorage())
     return DeclarationOnly;
   
-  if (getStorageClassAsWritten() == Extern ||
-       getStorageClassAsWritten() == PrivateExtern) {
+  if (getStorageClassAsWritten() == SC_Extern ||
+       getStorageClassAsWritten() == SC_PrivateExtern) {
     for (const VarDecl *PrevVar = getPreviousDeclaration();
          PrevVar; PrevVar = PrevVar->getPreviousDeclaration()) {
       if (PrevVar->getLinkage() == InternalLinkage && PrevVar->hasInit())
@@ -912,28 +919,6 @@ SourceRange ParmVarDecl::getDefaultArgRange() const {
 // FunctionDecl Implementation
 //===----------------------------------------------------------------------===//
 
-void FunctionDecl::Destroy(ASTContext& C) {
-  if (Body && Body.isOffset())
-    Body.get(C.getExternalSource())->Destroy(C);
-
-  for (param_iterator I=param_begin(), E=param_end(); I!=E; ++I)
-    (*I)->Destroy(C);
-
-  FunctionTemplateSpecializationInfo *FTSInfo
-    = TemplateOrSpecialization.dyn_cast<FunctionTemplateSpecializationInfo*>();
-  if (FTSInfo)
-    C.Deallocate(FTSInfo);
-  
-  MemberSpecializationInfo *MSInfo
-    = TemplateOrSpecialization.dyn_cast<MemberSpecializationInfo*>();
-  if (MSInfo)
-    C.Deallocate(MSInfo);
-  
-  C.Deallocate(ParamInfo);
-
-  DeclaratorDecl::Destroy(C);
-}
-
 void FunctionDecl::getNameForDiagnostic(std::string &S,
                                         const PrintingPolicy &Policy,
                                         bool Qualified) const {
@@ -984,7 +969,7 @@ void FunctionDecl::setBody(Stmt *B) {
 bool FunctionDecl::isMain() const {
   ASTContext &Context = getASTContext();
   return !Context.getLangOptions().Freestanding &&
-    getDeclContext()->getLookupContext()->isTranslationUnit() &&
+    getDeclContext()->getRedeclContext()->isTranslationUnit() &&
     getIdentifier() && getIdentifier()->isStr("main");
 }
 
@@ -993,17 +978,20 @@ bool FunctionDecl::isExternC() const {
   // In C, any non-static, non-overloadable function has external
   // linkage.
   if (!Context.getLangOptions().CPlusPlus)
-    return getStorageClass() != Static && !getAttr<OverloadableAttr>();
+    return getStorageClass() != SC_Static && !getAttr<OverloadableAttr>();
 
   for (const DeclContext *DC = getDeclContext(); !DC->isTranslationUnit();
        DC = DC->getParent()) {
     if (const LinkageSpecDecl *Linkage = dyn_cast<LinkageSpecDecl>(DC))  {
       if (Linkage->getLanguage() == LinkageSpecDecl::lang_c)
-        return getStorageClass() != Static &&
+        return getStorageClass() != SC_Static &&
                !getAttr<OverloadableAttr>();
 
       break;
     }
+    
+    if (DC->isRecord())
+      break;
   }
 
   return false;
@@ -1013,7 +1001,7 @@ bool FunctionDecl::isGlobal() const {
   if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(this))
     return Method->isStatic();
 
-  if (getStorageClass() == Static)
+  if (getStorageClass() == SC_Static)
     return false;
 
   for (const DeclContext *DC = getDeclContext();
@@ -1072,7 +1060,7 @@ unsigned FunctionDecl::getBuiltinID() const {
   // function or whether it just has the same name.
 
   // If this is a static function, it's not a builtin.
-  if (getStorageClass() == Static)
+  if (getStorageClass() == SC_Static)
     return 0;
 
   // If this function is at translation-unit scope and we're not in
@@ -1206,7 +1194,7 @@ bool FunctionDecl::isInlineDefinitionExternallyVisible() const {
     for (redecl_iterator Redecl = redecls_begin(), RedeclEnd = redecls_end();
          Redecl != RedeclEnd;
          ++Redecl) {
-      if (Redecl->isInlineSpecified() && Redecl->getStorageClass() != Extern)
+      if (Redecl->isInlineSpecified() && Redecl->getStorageClass() != SC_Extern)
         return true;
     }
     
@@ -1225,7 +1213,7 @@ bool FunctionDecl::isInlineDefinitionExternallyVisible() const {
     if (!Redecl->getLexicalDeclContext()->isTranslationUnit())
       continue;
     
-    if (!Redecl->isInlineSpecified() || Redecl->getStorageClass() == Extern) 
+    if (!Redecl->isInlineSpecified() || Redecl->getStorageClass() == SC_Extern) 
       return true; // Not an inline definition
   }
   
@@ -1400,14 +1388,15 @@ FunctionDecl::setFunctionTemplateSpecialization(FunctionTemplateDecl *Template,
   if (InsertPos)
     Template->getSpecializations().InsertNode(Info, InsertPos);
   else {
-    // Try to insert the new node. If there is an existing node, remove it 
-    // first.
+    // Try to insert the new node. If there is an existing node, leave it, the
+    // set will contain the canonical decls while
+    // FunctionTemplateDecl::findSpecialization will return
+    // the most recent redeclarations.
     FunctionTemplateSpecializationInfo *Existing
       = Template->getSpecializations().GetOrInsertNode(Info);
-    if (Existing) {
-      Template->getSpecializations().RemoveNode(Existing);
-      Template->getSpecializations().GetOrInsertNode(Info);
-    }
+    (void)Existing;
+    assert((!Existing || Existing->Function->isCanonicalDecl()) &&
+           "Set is supposed to only contain canonical decls");
   }
 }
 
@@ -1564,12 +1553,6 @@ bool FieldDecl::isAnonymousStructOrUnion() const {
 // TagDecl Implementation
 //===----------------------------------------------------------------------===//
 
-void TagDecl::Destroy(ASTContext &C) {
-  if (hasExtInfo())
-    C.Deallocate(getExtInfo());
-  TypeDecl::Destroy(C);
-}
-
 SourceLocation TagDecl::getOuterLocStart() const {
   return getTemplateOrInnerLocStart(this);
 }
@@ -1590,14 +1573,7 @@ void TagDecl::setTypedefForAnonDecl(TypedefDecl *TDD) {
 }
 
 void TagDecl::startDefinition() {
-  if (TagType *TagT = const_cast<TagType *>(TypeForDecl->getAs<TagType>())) {
-    TagT->decl.setPointer(this);
-    TagT->decl.setInt(1);
-  } else if (InjectedClassNameType *Injected
-               = const_cast<InjectedClassNameType *>(
-                                 TypeForDecl->getAs<InjectedClassNameType>())) {
-    Injected->Decl = cast<CXXRecordDecl>(this);
-  }
+  IsBeingDefined = true;
 
   if (isa<CXXRecordDecl>(this)) {
     CXXRecordDecl *D = cast<CXXRecordDecl>(this);
@@ -1614,17 +1590,7 @@ void TagDecl::completeDefinition() {
          "definition completed but not started");
 
   IsDefinition = true;
-  if (TagType *TagT = const_cast<TagType *>(TypeForDecl->getAs<TagType>())) {
-    assert(TagT->decl.getPointer() == this &&
-           "Attempt to redefine a tag definition?");
-    TagT->decl.setInt(0);
-  } else if (InjectedClassNameType *Injected
-               = const_cast<InjectedClassNameType *>(
-                                TypeForDecl->getAs<InjectedClassNameType>())) {
-    assert(Injected->Decl == this &&
-           "Attempt to redefine a class template definition?");
-    (void)Injected;
-  }
+  IsBeingDefined = false;
 }
 
 TagDecl* TagDecl::getDefinition() const {
@@ -1675,10 +1641,6 @@ EnumDecl *EnumDecl::Create(ASTContext &C, EmptyShell Empty) {
   return new (C) EnumDecl(0, SourceLocation(), 0, 0, SourceLocation());
 }
 
-void EnumDecl::Destroy(ASTContext& C) {
-  TagDecl::Destroy(C);
-}
-
 void EnumDecl::completeDefinition(QualType NewType,
                                   QualType NewPromotionType,
                                   unsigned NumPositiveBits,
@@ -1719,13 +1681,6 @@ RecordDecl *RecordDecl::Create(ASTContext &C, EmptyShell Empty) {
                             SourceLocation());
 }
 
-RecordDecl::~RecordDecl() {
-}
-
-void RecordDecl::Destroy(ASTContext& C) {
-  TagDecl::Destroy(C);
-}
-
 bool RecordDecl::isInjectedClassName() const {
   return isImplicit() && getDeclName() && getDeclContext()->isRecord() &&
     cast<RecordDecl>(getDeclContext())->getDeclName() == getDeclName();
@@ -1753,20 +1708,6 @@ ValueDecl *RecordDecl::getAnonymousStructOrUnionObject() {
 // BlockDecl Implementation
 //===----------------------------------------------------------------------===//
 
-BlockDecl::~BlockDecl() {
-}
-
-void BlockDecl::Destroy(ASTContext& C) {
-  if (Body)
-    Body->Destroy(C);
-
-  for (param_iterator I=param_begin(), E=param_end(); I!=E; ++I)
-    (*I)->Destroy(C);
-
-  C.Deallocate(ParamInfo);
-  Decl::Destroy(C);
-}
-
 void BlockDecl::setParams(ParmVarDecl **NewParamInfo,
                           unsigned NParms) {
   assert(ParamInfo == 0 && "Already has param info!");
@@ -1798,27 +1739,17 @@ NamespaceDecl *NamespaceDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) NamespaceDecl(DC, L, Id);
 }
 
-void NamespaceDecl::Destroy(ASTContext& C) {
-  // NamespaceDecl uses "NextDeclarator" to chain namespace declarations
-  // together. They are all top-level Decls.
-
-  this->~NamespaceDecl();
-  Decl::Destroy(C);
-}
-
-
 ImplicitParamDecl *ImplicitParamDecl::Create(ASTContext &C, DeclContext *DC,
     SourceLocation L, IdentifierInfo *Id, QualType T) {
   return new (C) ImplicitParamDecl(ImplicitParam, DC, L, Id, T);
 }
 
 FunctionDecl *FunctionDecl::Create(ASTContext &C, DeclContext *DC,
-                                   SourceLocation L,
-                                   DeclarationName N, QualType T,
-                                   TypeSourceInfo *TInfo,
+                                   const DeclarationNameInfo &NameInfo,
+                                   QualType T, TypeSourceInfo *TInfo,
                                    StorageClass S, StorageClass SCAsWritten,
                                    bool isInline, bool hasWrittenPrototype) {
-  FunctionDecl *New = new (C) FunctionDecl(Function, DC, L, N, T, TInfo,
+  FunctionDecl *New = new (C) FunctionDecl(Function, DC, NameInfo, T, TInfo,
                                            S, SCAsWritten, isInline);
   New->HasWrittenPrototype = hasWrittenPrototype;
   return New;
@@ -1835,9 +1766,11 @@ EnumConstantDecl *EnumConstantDecl::Create(ASTContext &C, EnumDecl *CD,
   return new (C) EnumConstantDecl(CD, L, Id, T, E, V);
 }
 
-void EnumConstantDecl::Destroy(ASTContext& C) {
-  if (Init) Init->Destroy(C);
-  ValueDecl::Destroy(C);
+SourceRange EnumConstantDecl::getSourceRange() const {
+  SourceLocation End = getLocation();
+  if (Init)
+    End = Init->getLocEnd();
+  return SourceRange(getLocation(), End);
 }
 
 TypedefDecl *TypedefDecl::Create(ASTContext &C, DeclContext *DC,
@@ -1846,9 +1779,6 @@ TypedefDecl *TypedefDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) TypedefDecl(DC, L, Id, TInfo);
 }
 
-// Anchor TypedefDecl's vtable here.
-TypedefDecl::~TypedefDecl() {}
-
 FileScopeAsmDecl *FileScopeAsmDecl::Create(ASTContext &C, DeclContext *DC,
                                            SourceLocation L,
                                            StringLiteral *Str) {
diff --git a/lib/AST/DeclBase.cpp b/lib/AST/DeclBase.cpp
index d4f997de7669..0b958fe82b09 100644
--- a/lib/AST/DeclBase.cpp
+++ b/lib/AST/DeclBase.cpp
@@ -157,9 +157,7 @@ void PrettyStackTraceDecl::print(llvm::raw_ostream &OS) const {
 //===----------------------------------------------------------------------===//
 
 // Out-of-line virtual method providing a home for Decl.
-Decl::~Decl() {
-  assert(!HasAttrs && "attributes should have been freed by Destroy");
-}
+Decl::~Decl() { }
 
 void Decl::setDeclContext(DeclContext *DC) {
   if (isOutOfSemaDC())
@@ -314,35 +312,25 @@ unsigned Decl::getIdentifierNamespaceForKind(Kind DeclKind) {
   return 0;
 }
 
-void Decl::initAttrs(Attr *attrs) {
+void Decl::setAttrs(const AttrVec &attrs) {
   assert(!HasAttrs && "Decl already contains attrs.");
 
-  Attr *&AttrBlank = getASTContext().getDeclAttrs(this);
-  assert(AttrBlank == 0 && "HasAttrs was wrong?");
+  AttrVec &AttrBlank = getASTContext().getDeclAttrs(this);
+  assert(AttrBlank.empty() && "HasAttrs was wrong?");
 
   AttrBlank = attrs;
   HasAttrs = true;
 }
 
-void Decl::addAttr(Attr *NewAttr) {
-  Attr *&ExistingAttr = getASTContext().getDeclAttrs(this);
-
-  assert(NewAttr->getNext() == 0 && "Chain of attributes will be truncated!");
-  NewAttr->setNext(ExistingAttr);
-  ExistingAttr = NewAttr;
-
-  HasAttrs = true;
-}
-
-void Decl::invalidateAttrs() {
+void Decl::dropAttrs() {
   if (!HasAttrs) return;
 
   HasAttrs = false;
   getASTContext().eraseDeclAttrs(this);
 }
 
-const Attr *Decl::getAttrsImpl() const {
-  assert(HasAttrs && "getAttrs() should verify this!");
+const AttrVec &Decl::getAttrs() const {
+  assert(HasAttrs && "No attrs to get!");
   return getASTContext().getDeclAttrs(this);
 }
 
@@ -372,40 +360,6 @@ void Decl::swapAttrs(Decl *RHS) {
   RHS->HasAttrs = true;
 }
 
-void Decl::Destroy(ASTContext &C) {
-  // Free attributes for this decl.
-  if (HasAttrs) {
-    C.getDeclAttrs(this)->Destroy(C);
-    invalidateAttrs();
-    HasAttrs = false;
-  }
-
-#if 0
-  // FIXME: Once ownership is fully understood, we can enable this code
-  if (DeclContext *DC = dyn_cast<DeclContext>(this))
-    DC->decls_begin()->Destroy(C);
-
-  // Observe the unrolled recursion.  By setting N->NextDeclInContext = 0x0
-  // within the loop, only the Destroy method for the first Decl
-  // will deallocate all of the Decls in a chain.
-
-  Decl* N = getNextDeclInContext();
-
-  while (N) {
-    Decl* Tmp = N->getNextDeclInContext();
-    N->NextDeclInContext = 0;
-    N->Destroy(C);
-    N = Tmp;
-  }
-
-  if (isOutOfSemaDC())
-    delete (C) getMultipleDC();
-  
-  this->~Decl();
-  C.Deallocate((void *)this);
-#endif  
-}
-
 Decl *Decl::castFromDeclContext (const DeclContext *D) {
   Decl::Kind DK = D->getDeclKind();
   switch(DK) {
@@ -506,17 +460,7 @@ bool DeclContext::classof(const Decl *D) {
   }
 }
 
-DeclContext::~DeclContext() {
-  // FIXME: Currently ~ASTContext will delete the StoredDeclsMaps because
-  // ~DeclContext() is not guaranteed to be called when ASTContext uses
-  // a BumpPtrAllocator.
-  // delete LookupPtr;
-}
-
-void DeclContext::DestroyDecls(ASTContext &C) {
-  for (decl_iterator D = decls_begin(); D != decls_end(); )
-    (*D++)->Destroy(C);
-}
+DeclContext::~DeclContext() { }
 
 /// \brief Find the parent context of this context that will be
 /// used for unqualified name lookup.
@@ -527,13 +471,18 @@ void DeclContext::DestroyDecls(ASTContext &C) {
 DeclContext *DeclContext::getLookupParent() {
   // FIXME: Find a better way to identify friends
   if (isa<FunctionDecl>(this))
-    if (getParent()->getLookupContext()->isFileContext() &&
-        getLexicalParent()->getLookupContext()->isRecord())
+    if (getParent()->getRedeclContext()->isFileContext() &&
+        getLexicalParent()->getRedeclContext()->isRecord())
       return getLexicalParent();
   
   return getParent();
 }
 
+bool DeclContext::isInlineNamespace() const {
+  return isNamespace() &&
+         cast<NamespaceDecl>(this)->isInline();
+}
+
 bool DeclContext::isDependentContext() const {
   if (isFileContext())
     return false;
@@ -565,13 +514,11 @@ bool DeclContext::isTransparentContext() const {
     return true;
   else if (DeclKind >= Decl::firstRecord && DeclKind <= Decl::lastRecord)
     return cast<RecordDecl>(this)->isAnonymousStructOrUnion();
-  else if (DeclKind == Decl::Namespace)
-    return false; // FIXME: Check for C++0x inline namespaces
 
   return false;
 }
 
-bool DeclContext::Encloses(DeclContext *DC) {
+bool DeclContext::Encloses(const DeclContext *DC) const {
   if (getPrimaryContext() != this)
     return getPrimaryContext()->Encloses(DC);
 
@@ -652,6 +599,9 @@ DeclContext::LoadLexicalDeclsFromExternalStorage() const {
   ExternalASTSource *Source = getParentASTContext().getExternalSource();
   assert(hasExternalLexicalStorage() && Source && "No external storage?");
 
+  // Notify that we have a DeclContext that is initializing.
+  ExternalASTSource::Deserializing ADeclContext(Source);
+
   llvm::SmallVector<Decl*, 64> Decls;
   if (Source->FindExternalLexicalDecls(this, Decls))
     return;
@@ -701,19 +651,6 @@ ExternalASTSource::SetNoExternalVisibleDeclsForName(const DeclContext *DC,
 
 DeclContext::lookup_result
 ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
-                                          const VisibleDeclaration &VD) {
-  ASTContext &Context = DC->getParentASTContext();
-  StoredDeclsMap *Map;
-  if (!(Map = DC->LookupPtr))
-    Map = DC->CreateStoredDeclsMap(Context);
-
-  StoredDeclsList &List = (*Map)[VD.Name];
-  List.setFromDeclIDs(VD.Declarations);
-  return List.getLookupResult(Context);
-}
-
-DeclContext::lookup_result
-ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
                                                   DeclarationName Name,
                                     llvm::SmallVectorImpl<NamedDecl*> &Decls) {
   ASTContext &Context = DC->getParentASTContext();;
@@ -730,35 +667,34 @@ ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
       List.AddSubsequentDecl(Decls[I]);
   }
 
-  return List.getLookupResult(Context);
+  return List.getLookupResult();
 }
 
-void ExternalASTSource::SetExternalVisibleDecls(const DeclContext *DC,
-                    const llvm::SmallVectorImpl<VisibleDeclaration> &Decls) {
-  // There is no longer any visible storage in this context.
-  DC->ExternalVisibleStorage = false;
-
-  assert(!DC->LookupPtr && "Have a lookup map before de-serialization?");
-  StoredDeclsMap *Map = DC->CreateStoredDeclsMap(DC->getParentASTContext());
-  for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
-    (*Map)[Decls[I].Name].setFromDeclIDs(Decls[I].Declarations);
+void ExternalASTSource::MaterializeVisibleDeclsForName(const DeclContext *DC,
+                                                       DeclarationName Name,
+                                     llvm::SmallVectorImpl<NamedDecl*> &Decls) {
+  assert(DC->LookupPtr);
+  StoredDeclsMap &Map = *DC->LookupPtr;
+
+  // If there's an entry in the table the visible decls for this name have
+  // already been deserialized.
+  if (Map.find(Name) == Map.end()) {
+    StoredDeclsList &List = Map[Name];
+    for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
+      if (List.isNull())
+        List.setOnlyValue(Decls[I]);
+      else
+        List.AddSubsequentDecl(Decls[I]);
+    }
   }
 }
 
-void ExternalASTSource::SetExternalVisibleDecls(const DeclContext *DC,
-                            const llvm::SmallVectorImpl<NamedDecl*> &Decls) {
-  // There is no longer any visible storage in this context.
-  DC->ExternalVisibleStorage = false;
+DeclContext::decl_iterator DeclContext::noload_decls_begin() const {
+  return decl_iterator(FirstDecl);
+}
 
-  assert(!DC->LookupPtr && "Have a lookup map before de-serialization?");
-  StoredDeclsMap &Map = *DC->CreateStoredDeclsMap(DC->getParentASTContext());
-  for (unsigned I = 0, N = Decls.size(); I != N; ++I) {
-    StoredDeclsList &List = Map[Decls[I]->getDeclName()];
-    if (List.isNull())
-      List.setOnlyValue(Decls[I]);
-    else
-      List.AddSubsequentDecl(Decls[I]);
-  }
+DeclContext::decl_iterator DeclContext::noload_decls_end() const {
+  return decl_iterator();
 }
 
 DeclContext::decl_iterator DeclContext::decls_begin() const {
@@ -866,10 +802,10 @@ void DeclContext::buildLookup(DeclContext *DCtx) {
              I != IEnd; ++I)
           makeDeclVisibleInContextImpl(I->getInterface());
       
-      // If this declaration is itself a transparent declaration context,
-      // add its members (recursively).
+      // If this declaration is itself a transparent declaration context or
+      // inline namespace, add its members (recursively).
       if (DeclContext *InnerCtx = dyn_cast<DeclContext>(*D))
-        if (InnerCtx->isTransparentContext())
+        if (InnerCtx->isTransparentContext() || InnerCtx->isInlineNamespace())
           buildLookup(InnerCtx->getPrimaryContext());
     }
   }
@@ -886,7 +822,7 @@ DeclContext::lookup(DeclarationName Name) {
     if (LookupPtr) {
       StoredDeclsMap::iterator I = LookupPtr->find(Name);
       if (I != LookupPtr->end())
-        return I->second.getLookupResult(getParentASTContext());
+        return I->second.getLookupResult();
     }
 
     ExternalASTSource *Source = getParentASTContext().getExternalSource();
@@ -906,7 +842,7 @@ DeclContext::lookup(DeclarationName Name) {
   StoredDeclsMap::iterator Pos = LookupPtr->find(Name);
   if (Pos == LookupPtr->end())
     return lookup_result(lookup_iterator(0), lookup_iterator(0));
-  return Pos->second.getLookupResult(getParentASTContext());
+  return Pos->second.getLookupResult();
 }
 
 DeclContext::lookup_const_result
@@ -914,7 +850,7 @@ DeclContext::lookup(DeclarationName Name) const {
   return const_cast<DeclContext*>(this)->lookup(Name);
 }
 
-DeclContext *DeclContext::getLookupContext() {
+DeclContext *DeclContext::getRedeclContext() {
   DeclContext *Ctx = this;
   // Skip through transparent contexts.
   while (Ctx->isTransparentContext())
@@ -925,11 +861,29 @@ DeclContext *DeclContext::getLookupContext() {
 DeclContext *DeclContext::getEnclosingNamespaceContext() {
   DeclContext *Ctx = this;
   // Skip through non-namespace, non-translation-unit contexts.
-  while (!Ctx->isFileContext() || Ctx->isTransparentContext())
+  while (!Ctx->isFileContext())
     Ctx = Ctx->getParent();
   return Ctx->getPrimaryContext();
 }
 
+bool DeclContext::InEnclosingNamespaceSetOf(const DeclContext *O) const {
+  // For non-file contexts, this is equivalent to Equals.
+  if (!isFileContext())
+    return O->Equals(this);
+
+  do {
+    if (O->Equals(this))
+      return true;
+
+    const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(O);
+    if (!NS || !NS->isInline())
+      break;
+    O = NS->getParent();
+  } while (O);
+
+  return false;
+}
+
 void DeclContext::makeDeclVisibleInContext(NamedDecl *D, bool Recoverable) {
   // FIXME: This feels like a hack. Should DeclarationName support
   // template-ids, or is there a better way to keep specializations
@@ -953,9 +907,9 @@ void DeclContext::makeDeclVisibleInContext(NamedDecl *D, bool Recoverable) {
   if (LookupPtr || !Recoverable || hasExternalVisibleStorage())
     makeDeclVisibleInContextImpl(D);
 
-  // If we are a transparent context, insert into our parent context,
-  // too. This operation is recursive.
-  if (isTransparentContext())
+  // If we are a transparent context or inline namespace, insert into our
+  // parent context, too. This operation is recursive.
+  if (isTransparentContext() || isInlineNamespace())
     getParent()->makeDeclVisibleInContext(D, Recoverable);
 }
 
@@ -970,18 +924,21 @@ void DeclContext::makeDeclVisibleInContextImpl(NamedDecl *D) {
   if (isa<ClassTemplateSpecializationDecl>(D))
     return;
 
-  // If there is an external AST source, load any declarations it knows about
-  // with this declaration's name.
-  if (ExternalASTSource *Source = getParentASTContext().getExternalSource())
-    if (hasExternalVisibleStorage())
-      Source->FindExternalVisibleDeclsByName(this, D->getDeclName());
-
   ASTContext *C = 0;
   if (!LookupPtr) {
     C = &getParentASTContext();
     CreateStoredDeclsMap(*C);
   }
 
+  // If there is an external AST source, load any declarations it knows about
+  // with this declaration's name.
+  // If the lookup table contains an entry about this name it means that we
+  // have already checked the external source.
+  if (ExternalASTSource *Source = getParentASTContext().getExternalSource())
+    if (hasExternalVisibleStorage() &&
+        LookupPtr->find(D->getDeclName()) == LookupPtr->end())
+      Source->FindExternalVisibleDeclsByName(this, D->getDeclName());
+
   // Insert this declaration into the map.
   StoredDeclsList &DeclNameEntries = (*LookupPtr)[D->getDeclName()];
   if (DeclNameEntries.isNull()) {
@@ -992,16 +949,22 @@ void DeclContext::makeDeclVisibleInContextImpl(NamedDecl *D) {
   // If it is possible that this is a redeclaration, check to see if there is
   // already a decl for which declarationReplaces returns true.  If there is
   // one, just replace it and return.
-  if (!C)
-    C = &getParentASTContext();
-  
-  if (DeclNameEntries.HandleRedeclaration(*C, D))
+  if (DeclNameEntries.HandleRedeclaration(D))
     return;
 
   // Put this declaration into the appropriate slot.
   DeclNameEntries.AddSubsequentDecl(D);
 }
 
+void DeclContext::MaterializeVisibleDeclsFromExternalStorage() {
+  ExternalASTSource *Source = getParentASTContext().getExternalSource();
+  assert(hasExternalVisibleStorage() && Source && "No external storage?");
+
+  if (!LookupPtr)
+    CreateStoredDeclsMap(getParentASTContext());
+  Source->MaterializeVisibleDecls(this);
+}
+
 /// Returns iterator range [First, Last) of UsingDirectiveDecls stored within
 /// this context.
 DeclContext::udir_iterator_range
@@ -1011,43 +974,6 @@ DeclContext::getUsingDirectives() const {
                              reinterpret_cast<udir_iterator>(Result.second));
 }
 
-void StoredDeclsList::materializeDecls(ASTContext &Context) {
-  if (isNull())
-    return;
-
-  switch ((DataKind)(Data & 0x03)) {
-  case DK_Decl:
-  case DK_Decl_Vector:
-    break;
-
-  case DK_DeclID: {
-    // Resolve this declaration ID to an actual declaration by
-    // querying the external AST source.
-    unsigned DeclID = Data >> 2;
-
-    ExternalASTSource *Source = Context.getExternalSource();
-    assert(Source && "No external AST source available!");
-
-    Data = reinterpret_cast<uintptr_t>(Source->GetExternalDecl(DeclID));
-    break;
-  }
-
-  case DK_ID_Vector: {
-    // We have a vector of declaration IDs. Resolve all of them to
-    // actual declarations.
-    VectorTy &Vector = *getAsVector();
-    ExternalASTSource *Source = Context.getExternalSource();
-    assert(Source && "No external AST source available!");
-
-    for (unsigned I = 0, N = Vector.size(); I != N; ++I)
-      Vector[I] = reinterpret_cast<uintptr_t>(Source->GetExternalDecl(Vector[I]));
-
-    Data = (Data & ~0x03) | DK_Decl_Vector;
-    break;
-  }
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Creation and Destruction of StoredDeclsMaps.                               //
 //===----------------------------------------------------------------------===//
@@ -1073,7 +999,6 @@ void ASTContext::ReleaseDeclContextMaps() {
   // It's okay to delete DependentStoredDeclsMaps via a StoredDeclsMap
   // pointer because the subclass doesn't add anything that needs to
   // be deleted.
-  
   StoredDeclsMap::DestroyAll(LastSDM.getPointer(), LastSDM.getInt());
 }
 
diff --git a/lib/AST/DeclCXX.cpp b/lib/AST/DeclCXX.cpp
index dd0fe08979f0..f2f0694826c6 100644
--- a/lib/AST/DeclCXX.cpp
+++ b/lib/AST/DeclCXX.cpp
@@ -65,18 +65,6 @@ CXXRecordDecl *CXXRecordDecl::Create(ASTContext &C, EmptyShell Empty) {
                                SourceLocation());
 }
 
-CXXRecordDecl::~CXXRecordDecl() {
-}
-
-void CXXRecordDecl::Destroy(ASTContext &C) {
-  if (data().Definition == this) {
-    C.Deallocate(data().Bases);
-    C.Deallocate(data().VBases);
-    C.Deallocate(&data());
-  }
-  this->RecordDecl::Destroy(C);
-}
-
 void
 CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
                         unsigned NumBases) {
@@ -133,19 +121,19 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
   data().VBases = new (C) CXXBaseSpecifier[VBases.size()];
   data().NumVBases = VBases.size();
   for (int I = 0, E = VBases.size(); I != E; ++I) {
-    QualType VBaseType = VBases[I]->getType();
-    
+    TypeSourceInfo *VBaseTypeInfo = VBases[I]->getTypeSourceInfo();
+
     // Skip dependent types; we can't do any checking on them now.
-    if (VBaseType->isDependentType())
+    if (VBaseTypeInfo->getType()->isDependentType())
       continue;
 
-    CXXRecordDecl *VBaseClassDecl
-      = cast<CXXRecordDecl>(VBaseType->getAs<RecordType>()->getDecl());
+    CXXRecordDecl *VBaseClassDecl = cast<CXXRecordDecl>(
+      VBaseTypeInfo->getType()->getAs<RecordType>()->getDecl());
 
     data().VBases[I] =
       CXXBaseSpecifier(VBaseClassDecl->getSourceRange(), true,
                        VBaseClassDecl->getTagKind() == TTK_Class,
-                       VBases[I]->getAccessSpecifier(), VBaseType);
+                       VBases[I]->getAccessSpecifier(), VBaseTypeInfo);
   }
 }
 
@@ -621,7 +609,8 @@ CXXDestructorDecl *CXXRecordDecl::getDestructor() const {
 
   DeclContext::lookup_const_iterator I, E;
   llvm::tie(I, E) = lookup(Name);
-  assert(I != E && "Did not find a destructor!");
+  if (I == E)
+    return 0;
 
   CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(*I);
   assert(++I == E && "Found more than one destructor!");
@@ -631,10 +620,10 @@ CXXDestructorDecl *CXXRecordDecl::getDestructor() const {
 
 CXXMethodDecl *
 CXXMethodDecl::Create(ASTContext &C, CXXRecordDecl *RD,
-                      SourceLocation L, DeclarationName N,
+                      const DeclarationNameInfo &NameInfo,
                       QualType T, TypeSourceInfo *TInfo,
                       bool isStatic, StorageClass SCAsWritten, bool isInline) {
-  return new (C) CXXMethodDecl(CXXMethod, RD, L, N, T, TInfo,
+  return new (C) CXXMethodDecl(CXXMethod, RD, NameInfo, T, TInfo,
                                isStatic, SCAsWritten, isInline);
 }
 
@@ -796,13 +785,6 @@ CXXBaseOrMemberInitializer::Create(ASTContext &Context,
                                               L, Init, R, Indices, NumIndices);
 }
 
-void CXXBaseOrMemberInitializer::Destroy(ASTContext &Context) {
-  if (Init)
-    Init->Destroy(Context);
-  // FIXME: Destroy indices
-  this->~CXXBaseOrMemberInitializer();
-}
-
 TypeLoc CXXBaseOrMemberInitializer::getBaseClassLoc() const {
   if (isBaseInitializer())
     return BaseOrMember.get<TypeSourceInfo*>()->getTypeLoc();
@@ -837,20 +819,21 @@ SourceRange CXXBaseOrMemberInitializer::getSourceRange() const {
 
 CXXConstructorDecl *
 CXXConstructorDecl::Create(ASTContext &C, EmptyShell Empty) {
-  return new (C) CXXConstructorDecl(0, SourceLocation(), DeclarationName(),
+  return new (C) CXXConstructorDecl(0, DeclarationNameInfo(),
                                     QualType(), 0, false, false, false);
 }
 
 CXXConstructorDecl *
 CXXConstructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
-                           SourceLocation L, DeclarationName N,
+                           const DeclarationNameInfo &NameInfo,
                            QualType T, TypeSourceInfo *TInfo,
                            bool isExplicit,
                            bool isInline,
                            bool isImplicitlyDeclared) {
-  assert(N.getNameKind() == DeclarationName::CXXConstructorName &&
+  assert(NameInfo.getName().getNameKind()
+         == DeclarationName::CXXConstructorName &&
          "Name must refer to a constructor");
-  return new (C) CXXConstructorDecl(RD, L, N, T, TInfo, isExplicit,
+  return new (C) CXXConstructorDecl(RD, NameInfo, T, TInfo, isExplicit,
                                     isInline, isImplicitlyDeclared);
 }
 
@@ -945,40 +928,38 @@ bool CXXConstructorDecl::isCopyConstructorLikeSpecialization() const {
 
 CXXDestructorDecl *
 CXXDestructorDecl::Create(ASTContext &C, EmptyShell Empty) {
-  return new (C) CXXDestructorDecl(0, SourceLocation(), DeclarationName(),
+  return new (C) CXXDestructorDecl(0, DeclarationNameInfo(),
                                    QualType(), false, false);
 }
 
 CXXDestructorDecl *
 CXXDestructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
-                          SourceLocation L, DeclarationName N,
+                          const DeclarationNameInfo &NameInfo,
                           QualType T, bool isInline,
                           bool isImplicitlyDeclared) {
-  assert(N.getNameKind() == DeclarationName::CXXDestructorName &&
+  assert(NameInfo.getName().getNameKind()
+         == DeclarationName::CXXDestructorName &&
          "Name must refer to a destructor");
-  return new (C) CXXDestructorDecl(RD, L, N, T, isInline, isImplicitlyDeclared);
-}
-
-void
-CXXConstructorDecl::Destroy(ASTContext& C) {
-  C.Deallocate(BaseOrMemberInitializers);
-  CXXMethodDecl::Destroy(C);
+  return new (C) CXXDestructorDecl(RD, NameInfo, T, isInline,
+                                   isImplicitlyDeclared);
 }
 
 CXXConversionDecl *
 CXXConversionDecl::Create(ASTContext &C, EmptyShell Empty) {
-  return new (C) CXXConversionDecl(0, SourceLocation(), DeclarationName(),
+  return new (C) CXXConversionDecl(0, DeclarationNameInfo(),
                                    QualType(), 0, false, false);
 }
 
 CXXConversionDecl *
 CXXConversionDecl::Create(ASTContext &C, CXXRecordDecl *RD,
-                          SourceLocation L, DeclarationName N,
+                          const DeclarationNameInfo &NameInfo,
                           QualType T, TypeSourceInfo *TInfo,
                           bool isInline, bool isExplicit) {
-  assert(N.getNameKind() == DeclarationName::CXXConversionFunctionName &&
+  assert(NameInfo.getName().getNameKind()
+         == DeclarationName::CXXConversionFunctionName &&
          "Name must refer to a conversion function");
-  return new (C) CXXConversionDecl(RD, L, N, T, TInfo, isInline, isExplicit);
+  return new (C) CXXConversionDecl(RD, NameInfo, T, TInfo,
+                                   isInline, isExplicit);
 }
 
 LinkageSpecDecl *LinkageSpecDecl::Create(ASTContext &C,
@@ -1009,14 +990,8 @@ NamespaceDecl *UsingDirectiveDecl::getNominatedNamespace() {
   return cast_or_null<NamespaceDecl>(NominatedNamespace);
 }
 
-void UsingDirectiveDecl::setNominatedNamespace(NamedDecl* ND) {
-  assert((isa<NamespaceDecl>(ND) || isa<NamespaceAliasDecl>(ND)) &&
-    "expected a NamespaceDecl or NamespaceAliasDecl");
-  NominatedNamespace = ND;
-}
-
 NamespaceAliasDecl *NamespaceAliasDecl::Create(ASTContext &C, DeclContext *DC,
-                                               SourceLocation L,
+                                               SourceLocation UsingLoc,
                                                SourceLocation AliasLoc,
                                                IdentifierInfo *Alias,
                                                SourceRange QualifierRange,
@@ -1025,15 +1000,16 @@ NamespaceAliasDecl *NamespaceAliasDecl::Create(ASTContext &C, DeclContext *DC,
                                                NamedDecl *Namespace) {
   if (NamespaceDecl *NS = dyn_cast_or_null<NamespaceDecl>(Namespace))
     Namespace = NS->getOriginalNamespace();
-  return new (C) NamespaceAliasDecl(DC, L, AliasLoc, Alias, QualifierRange,
+  return new (C) NamespaceAliasDecl(DC, UsingLoc, AliasLoc, Alias, QualifierRange,
                                     Qualifier, IdentLoc, Namespace);
 }
 
 UsingDecl *UsingDecl::Create(ASTContext &C, DeclContext *DC,
-      SourceLocation L, SourceRange NNR, SourceLocation UL,
-      NestedNameSpecifier* TargetNNS, DeclarationName Name,
-      bool IsTypeNameArg) {
-  return new (C) UsingDecl(DC, L, NNR, UL, TargetNNS, Name, IsTypeNameArg);
+                             SourceRange NNR, SourceLocation UL,
+                             NestedNameSpecifier* TargetNNS,
+                             const DeclarationNameInfo &NameInfo,
+                             bool IsTypeNameArg) {
+  return new (C) UsingDecl(DC, NNR, UL, TargetNNS, NameInfo, IsTypeNameArg);
 }
 
 UnresolvedUsingValueDecl *
@@ -1041,11 +1017,9 @@ UnresolvedUsingValueDecl::Create(ASTContext &C, DeclContext *DC,
                                  SourceLocation UsingLoc,
                                  SourceRange TargetNNR,
                                  NestedNameSpecifier *TargetNNS,
-                                 SourceLocation TargetNameLoc,
-                                 DeclarationName TargetName) {
+                                 const DeclarationNameInfo &NameInfo) {
   return new (C) UnresolvedUsingValueDecl(DC, C.DependentTy, UsingLoc,
-                                          TargetNNR, TargetNNS,
-                                          TargetNameLoc, TargetName);
+                                          TargetNNR, TargetNNS, NameInfo);
 }
 
 UnresolvedUsingTypenameDecl *
@@ -1068,15 +1042,6 @@ StaticAssertDecl *StaticAssertDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) StaticAssertDecl(DC, L, AssertExpr, Message);
 }
 
-void StaticAssertDecl::Destroy(ASTContext& C) {
-  AssertExpr->Destroy(C);
-  Message->Destroy(C);
-  Decl::Destroy(C);
-}
-
-StaticAssertDecl::~StaticAssertDecl() {
-}
-
 static const char *getAccessName(AccessSpecifier AS) {
   switch (AS) {
     default:
@@ -1096,5 +1061,3 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
                                            AccessSpecifier AS) {
   return DB << getAccessName(AS);
 }
-
-
diff --git a/lib/AST/DeclGroup.cpp b/lib/AST/DeclGroup.cpp
index 434bf00d354e..036acc2d77a5 100644
--- a/lib/AST/DeclGroup.cpp
+++ b/lib/AST/DeclGroup.cpp
@@ -30,9 +30,3 @@ DeclGroup::DeclGroup(unsigned numdecls, Decl** decls) : NumDecls(numdecls) {
   assert(decls);
   memcpy(this+1, decls, numdecls * sizeof(*decls));
 }
-
-void DeclGroup::Destroy(ASTContext& C) {
-  // Decls are destroyed by the DeclContext.
-  this->~DeclGroup();
-  C.Deallocate((void*) this);
-}
diff --git a/lib/AST/DeclObjC.cpp b/lib/AST/DeclObjC.cpp
index adb0e7d08345..d952cc36ef1d 100644
--- a/lib/AST/DeclObjC.cpp
+++ b/lib/AST/DeclObjC.cpp
@@ -21,14 +21,8 @@ using namespace clang;
 // ObjCListBase
 //===----------------------------------------------------------------------===//
 
-void ObjCListBase::Destroy(ASTContext &Ctx) {
-  Ctx.Deallocate(List);
-  NumElts = 0;
-  List = 0;
-}
-
 void ObjCListBase::set(void *const* InList, unsigned Elts, ASTContext &Ctx) {
-  assert(List == 0 && "Elements already set!");
+  List = 0;
   if (Elts == 0) return;  // Setting to an empty list is a noop.
 
 
@@ -47,12 +41,6 @@ void ObjCProtocolList::set(ObjCProtocolDecl* const* InList, unsigned Elts,
   set(InList, Elts, Ctx);
 }
 
-void ObjCProtocolList::Destroy(ASTContext &Ctx) {
-  Ctx.Deallocate(Locations);
-  Locations = 0;
-  ObjCList<ObjCProtocolDecl>::Destroy(Ctx);
-}
-
 //===----------------------------------------------------------------------===//
 // ObjCInterfaceDecl
 //===----------------------------------------------------------------------===//
@@ -132,8 +120,9 @@ ObjCContainerDecl::FindPropertyDeclaration(IdentifierInfo *PropertyId) const {
             return P;
 
       // Look through protocols.
-      for (ObjCInterfaceDecl::protocol_iterator
-            I = OID->protocol_begin(), E = OID->protocol_end(); I != E; ++I)
+      for (ObjCInterfaceDecl::all_protocol_iterator
+            I = OID->all_referenced_protocol_begin(),
+            E = OID->all_referenced_protocol_end(); I != E; ++I)
         if (ObjCPropertyDecl *P = (*I)->FindPropertyDeclaration(PropertyId))
           return P;
 
@@ -169,8 +158,9 @@ ObjCInterfaceDecl::FindPropertyVisibleInPrimaryClass(
     return PD;
 
   // Look through protocols.
-  for (ObjCInterfaceDecl::protocol_iterator
-        I = protocol_begin(), E = protocol_end(); I != E; ++I)
+  for (ObjCInterfaceDecl::all_protocol_iterator
+        I = all_referenced_protocol_begin(),
+        E = all_referenced_protocol_end(); I != E; ++I)
     if (ObjCPropertyDecl *P = (*I)->FindPropertyDeclaration(PropertyId))
       return P;
 
@@ -179,23 +169,23 @@ ObjCInterfaceDecl::FindPropertyVisibleInPrimaryClass(
 
 void ObjCInterfaceDecl::mergeClassExtensionProtocolList(
                               ObjCProtocolDecl *const* ExtList, unsigned ExtNum,
-                              const SourceLocation *Locs,
                               ASTContext &C)
 {
-  if (ReferencedProtocols.empty()) {
-    ReferencedProtocols.set(ExtList, ExtNum, Locs, C);
+  if (AllReferencedProtocols.empty() && ReferencedProtocols.empty()) {
+    AllReferencedProtocols.set(ExtList, ExtNum, C);
     return;
   }
+  
   // Check for duplicate protocol in class's protocol list.
-  // This is (O)2. But it is extremely rare and number of protocols in
+  // This is O(n*m). But it is extremely rare and number of protocols in
   // class or its extension are very few.
   llvm::SmallVector<ObjCProtocolDecl*, 8> ProtocolRefs;
-  llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
   for (unsigned i = 0; i < ExtNum; i++) {
     bool protocolExists = false;
     ObjCProtocolDecl *ProtoInExtension = ExtList[i];
-    for (protocol_iterator p = protocol_begin(), e = protocol_end();
-         p != e; p++) {
+    for (all_protocol_iterator
+          p = all_referenced_protocol_begin(),
+          e = all_referenced_protocol_end(); p != e; ++p) {
       ObjCProtocolDecl *Proto = (*p);
       if (C.ProtocolCompatibleWithProtocol(ProtoInExtension, Proto)) {
         protocolExists = true;
@@ -204,23 +194,20 @@ void ObjCInterfaceDecl::mergeClassExtensionProtocolList(
     }
     // Do we want to warn on a protocol in extension class which
     // already exist in the class? Probably not.
-    if (!protocolExists) {
+    if (!protocolExists)
       ProtocolRefs.push_back(ProtoInExtension);
-      ProtocolLocs.push_back(Locs[i]);
-    }
   }
+
   if (ProtocolRefs.empty())
     return;
+
   // Merge ProtocolRefs into class's protocol list;
-  protocol_loc_iterator pl = protocol_loc_begin();
-  for (protocol_iterator p = protocol_begin(), e = protocol_end();
-       p != e; ++p, ++pl) {
+  for (all_protocol_iterator p = all_referenced_protocol_begin(), 
+        e = all_referenced_protocol_end(); p != e; ++p) {
     ProtocolRefs.push_back(*p);
-    ProtocolLocs.push_back(*pl);
   }
-  ReferencedProtocols.Destroy(C);
-  unsigned NumProtoRefs = ProtocolRefs.size();
-  setProtocolList(ProtocolRefs.data(), NumProtoRefs, ProtocolLocs.data(), C);
+
+  AllReferencedProtocols.set(ProtocolRefs.data(), ProtocolRefs.size(), C);
 }
 
 /// getFirstClassExtension - Find first class extension of the given class.
@@ -339,27 +326,17 @@ ObjCMethodDecl *ObjCMethodDecl::Create(ASTContext &C,
                                        bool isInstance,
                                        bool isVariadic,
                                        bool isSynthesized,
+                                       bool isDefined,
                                        ImplementationControl impControl,
                                        unsigned numSelectorArgs) {
   return new (C) ObjCMethodDecl(beginLoc, endLoc,
                                 SelInfo, T, ResultTInfo, contextDecl,
                                 isInstance,
-                                isVariadic, isSynthesized, impControl,
+                                isVariadic, isSynthesized, isDefined,
+                                impControl,
                                 numSelectorArgs);
 }
 
-void ObjCMethodDecl::Destroy(ASTContext &C) {
-  if (Body) Body->Destroy(C);
-  if (SelfDecl) SelfDecl->Destroy(C);
-
-  for (param_iterator I=param_begin(), E=param_end(); I!=E; ++I)
-    if (*I) (*I)->Destroy(C);
-
-  ParamInfo.Destroy(C);
-
-  Decl::Destroy(C);
-}
-
 /// \brief A definition will return its interface declaration.
 /// An interface declaration will return its definition.
 /// Otherwise it will return itself.
@@ -465,22 +442,11 @@ ObjCInterfaceDecl(DeclContext *DC, SourceLocation atLoc, IdentifierInfo *Id,
                   SourceLocation CLoc, bool FD, bool isInternal)
   : ObjCContainerDecl(ObjCInterface, DC, atLoc, Id),
     TypeForDecl(0), SuperClass(0),
-    CategoryList(0), ForwardDecl(FD), InternalInterface(isInternal),
+    CategoryList(0), IvarList(0), 
+    ForwardDecl(FD), InternalInterface(isInternal),
     ClassLoc(CLoc) {
 }
 
-void ObjCInterfaceDecl::Destroy(ASTContext &C) {
-  for (ivar_iterator I = ivar_begin(), E = ivar_end(); I != E; ++I)
-    if (*I) (*I)->Destroy(C);
-
-  // FIXME: CategoryList?
-
-  // FIXME: Because there is no clear ownership
-  //  role between ObjCInterfaceDecls and the ObjCPropertyDecls that they
-  //  reference, we destroy ObjCPropertyDecls in ~TranslationUnit.
-  Decl::Destroy(C);
-}
-
 ObjCImplementationDecl *ObjCInterfaceDecl::getImplementation() const {
   return getASTContext().getObjCImplementation(
                                           const_cast<ObjCInterfaceDecl*>(this));
@@ -490,6 +456,49 @@ void ObjCInterfaceDecl::setImplementation(ObjCImplementationDecl *ImplD) {
   getASTContext().setObjCImplementation(this, ImplD);
 }
 
+/// all_declared_ivar_begin - return first ivar declared in this class,
+/// its extensions and its implementation. Lazily build the list on first
+/// access.
+ObjCIvarDecl *ObjCInterfaceDecl::all_declared_ivar_begin() {
+  if (IvarList)
+    return IvarList;
+  
+  ObjCIvarDecl *curIvar = 0;
+  if (!ivar_empty()) {
+    ObjCInterfaceDecl::ivar_iterator I = ivar_begin(), E = ivar_end();
+    IvarList = (*I); ++I;
+    for (curIvar = IvarList; I != E; curIvar = *I, ++I)
+      curIvar->setNextIvar(*I);
+  }
+  
+  for (const ObjCCategoryDecl *CDecl = getFirstClassExtension(); CDecl;
+       CDecl = CDecl->getNextClassExtension()) {
+    if (!CDecl->ivar_empty()) {
+      ObjCCategoryDecl::ivar_iterator I = CDecl->ivar_begin(),
+                                          E = CDecl->ivar_end();
+      if (!IvarList) {
+        IvarList = (*I); ++I;
+        curIvar = IvarList;
+      }
+      for ( ;I != E; curIvar = *I, ++I)
+        curIvar->setNextIvar(*I);
+    }
+  }
+  
+  if (ObjCImplementationDecl *ImplDecl = getImplementation()) {
+    if (!ImplDecl->ivar_empty()) {
+      ObjCImplementationDecl::ivar_iterator I = ImplDecl->ivar_begin(),
+                                            E = ImplDecl->ivar_end();
+      if (!IvarList) {
+        IvarList = (*I); ++I;
+        curIvar = IvarList;
+      }
+      for ( ;I != E; curIvar = *I, ++I)
+        curIvar->setNextIvar(*I);
+    }
+  }
+  return IvarList;
+}
 
 /// FindCategoryDeclaration - Finds category declaration in the list of
 /// categories for this class and returns it. Name of the category is passed
@@ -575,7 +584,8 @@ bool ObjCInterfaceDecl::ClassImplementsProtocol(ObjCProtocolDecl *lProto,
 ObjCIvarDecl *ObjCIvarDecl::Create(ASTContext &C, ObjCContainerDecl *DC,
                                    SourceLocation L, IdentifierInfo *Id,
                                    QualType T, TypeSourceInfo *TInfo,
-                                   AccessControl ac, Expr *BW) {
+                                   AccessControl ac, Expr *BW,
+                                   bool synthesized) {
   if (DC) {
     // Ivar's can only appear in interfaces, implementations (via synthesized
     // properties), and class extensions (via direct declaration, or synthesized
@@ -590,9 +600,26 @@ ObjCIvarDecl *ObjCIvarDecl::Create(ASTContext &C, ObjCContainerDecl *DC,
     assert((isa<ObjCInterfaceDecl>(DC) || isa<ObjCImplementationDecl>(DC) ||
             isa<ObjCCategoryDecl>(DC)) &&
            "Invalid ivar decl context!");
+    // Once a new ivar is created in any of class/class-extension/implementation
+    // decl contexts, the previously built IvarList must be rebuilt.
+    ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(DC);
+    if (!ID) {
+      if (ObjCImplementationDecl *IM = dyn_cast<ObjCImplementationDecl>(DC)) {
+        ID = IM->getClassInterface();
+        if (BW)
+          IM->setHasSynthBitfield(true);
+      }
+      else {
+        ObjCCategoryDecl *CD = cast<ObjCCategoryDecl>(DC);
+        ID = CD->getClassInterface();
+        if (BW)
+          CD->setHasSynthBitfield(true);
+      }
+    }
+    ID->setIvarList(0);
   }
 
-  return new (C) ObjCIvarDecl(DC, L, Id, T, TInfo, ac, BW);
+  return new (C) ObjCIvarDecl(DC, L, Id, T, TInfo, ac, BW, synthesized);
 }
 
 const ObjCInterfaceDecl *ObjCIvarDecl::getContainingInterface() const {
@@ -630,11 +657,6 @@ ObjCAtDefsFieldDecl
   return new (C) ObjCAtDefsFieldDecl(DC, L, Id, T, BW);
 }
 
-void ObjCAtDefsFieldDecl::Destroy(ASTContext& C) {
-  this->~ObjCAtDefsFieldDecl();
-  C.Deallocate((void *)this);
-}
-
 //===----------------------------------------------------------------------===//
 // ObjCProtocolDecl
 //===----------------------------------------------------------------------===//
@@ -645,11 +667,6 @@ ObjCProtocolDecl *ObjCProtocolDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) ObjCProtocolDecl(DC, L, Id);
 }
 
-void ObjCProtocolDecl::Destroy(ASTContext &C) {
-  ReferencedProtocols.Destroy(C);
-  ObjCContainerDecl::Destroy(C);
-}
-
 ObjCProtocolDecl *ObjCProtocolDecl::lookupProtocolNamed(IdentifierInfo *Name) {
   ObjCProtocolDecl *PDecl = this;
 
@@ -709,23 +726,6 @@ ObjCClassDecl *ObjCClassDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) ObjCClassDecl(DC, L, Elts, Locs, nElts, C);
 }
 
-void ObjCClassDecl::Destroy(ASTContext &C) {
-  // ObjCInterfaceDecls registered with a DeclContext will get destroyed
-  // when the DeclContext is destroyed.  For those created only by a forward
-  // declaration, the first @class that created the ObjCInterfaceDecl gets
-  // to destroy it.
-  // FIXME: Note that this ownership role is very brittle; a better
-  // polict is surely need in the future.
-  for (iterator I = begin(), E = end(); I !=E ; ++I) {
-    ObjCInterfaceDecl *ID = I->getInterface();
-    if (ID->isForwardDecl() && ID->getLocStart() == getLocStart())
-      ID->Destroy(C);
-  }
-  
-  C.Deallocate(ForwardDecls);
-  Decl::Destroy(C);
-}
-
 SourceRange ObjCClassDecl::getSourceRange() const {
   // FIXME: We should include the semicolon
   assert(NumDecls);
@@ -754,11 +754,6 @@ ObjCForwardProtocolDecl::Create(ASTContext &C, DeclContext *DC,
   return new (C) ObjCForwardProtocolDecl(DC, L, Elts, NumElts, Locs, C);
 }
 
-void ObjCForwardProtocolDecl::Destroy(ASTContext &C) {
-  ReferencedProtocols.Destroy(C);
-  Decl::Destroy(C);
-}
-
 //===----------------------------------------------------------------------===//
 // ObjCCategoryDecl
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/DeclPrinter.cpp b/lib/AST/DeclPrinter.cpp
index 765772dd13f9..f18d2f0215dd 100644
--- a/lib/AST/DeclPrinter.cpp
+++ b/lib/AST/DeclPrinter.cpp
@@ -198,6 +198,12 @@ void DeclPrinter::VisitDeclContext(DeclContext *DC, bool Indent) {
   llvm::SmallVector<Decl*, 2> Decls;
   for (DeclContext::decl_iterator D = DC->decls_begin(), DEnd = DC->decls_end();
        D != DEnd; ++D) {
+
+    // Don't print ObjCIvarDecls, as they are printed when visiting the
+    // containing ObjCInterfaceDecl.
+    if (isa<ObjCIvarDecl>(*D))
+      continue;
+
     if (!Policy.Dump) {
       // Skip over implicit declarations in pretty-printing mode.
       if (D->isImplicit()) continue;
@@ -329,10 +335,11 @@ void DeclPrinter::VisitEnumConstantDecl(EnumConstantDecl *D) {
 void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
   if (!Policy.SuppressSpecifiers) {
     switch (D->getStorageClass()) {
-    case FunctionDecl::None: break;
-    case FunctionDecl::Extern: Out << "extern "; break;
-    case FunctionDecl::Static: Out << "static "; break;
-    case FunctionDecl::PrivateExtern: Out << "__private_extern__ "; break;
+    case SC_None: break;
+    case SC_Extern: Out << "extern "; break;
+    case SC_Static: Out << "static "; break;
+    case SC_PrivateExtern: Out << "__private_extern__ "; break;
+    case SC_Auto: case SC_Register: llvm_unreachable("invalid for functions");
     }
 
     if (D->isInlineSpecified())           Out << "inline ";
@@ -341,7 +348,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 
   PrintingPolicy SubPolicy(Policy);
   SubPolicy.SuppressSpecifiers = false;
-  std::string Proto = D->getNameAsString();
+  std::string Proto = D->getNameInfo().getAsString();
   if (isa<FunctionType>(D->getType().getTypePtr())) {
     const FunctionType *AFT = D->getType()->getAs<FunctionType>();
 
@@ -499,7 +506,7 @@ void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
 }
 
 void DeclPrinter::VisitVarDecl(VarDecl *D) {
-  if (!Policy.SuppressSpecifiers && D->getStorageClass() != VarDecl::None)
+  if (!Policy.SuppressSpecifiers && D->getStorageClass() != SC_None)
     Out << VarDecl::getStorageClassSpecifierString(D->getStorageClass()) << " ";
 
   if (!Policy.SuppressSpecifiers && D->isThreadSpecified())
diff --git a/lib/AST/DeclTemplate.cpp b/lib/AST/DeclTemplate.cpp
index 9e1d79d79d6b..e69338a7730d 100644
--- a/lib/AST/DeclTemplate.cpp
+++ b/lib/AST/DeclTemplate.cpp
@@ -84,10 +84,59 @@ unsigned TemplateParameterList::getDepth() const {
 }
 
 //===----------------------------------------------------------------------===//
-// TemplateDecl Implementation
+// RedeclarableTemplateDecl Implementation
 //===----------------------------------------------------------------------===//
 
-TemplateDecl::~TemplateDecl() {
+RedeclarableTemplateDecl::CommonBase *RedeclarableTemplateDecl::getCommonPtr() {
+  // Find the first declaration of this function template.
+  RedeclarableTemplateDecl *First = getCanonicalDecl();
+
+  if (First->CommonOrPrev.isNull()) {
+    CommonBase *CommonPtr = First->newCommon();
+    First->CommonOrPrev = CommonPtr;
+    CommonPtr->Latest = First;
+  }
+  return First->CommonOrPrev.get<CommonBase*>();
+}
+
+
+RedeclarableTemplateDecl *RedeclarableTemplateDecl::getCanonicalDeclImpl() {
+  RedeclarableTemplateDecl *Tmpl = this;
+  while (Tmpl->getPreviousDeclaration())
+    Tmpl = Tmpl->getPreviousDeclaration();
+  return Tmpl;
+}
+
+void RedeclarableTemplateDecl::setPreviousDeclarationImpl(
+                                               RedeclarableTemplateDecl *Prev) {
+  if (Prev) {
+    CommonBase *Common = Prev->getCommonPtr();
+    Prev = Common->Latest;
+    Common->Latest = this;
+    CommonOrPrev = Prev;
+  } else {
+    assert(CommonOrPrev.is<CommonBase*>() && "Cannot reset TemplateDecl Prev");
+  }
+}
+
+RedeclarableTemplateDecl *RedeclarableTemplateDecl::getNextRedeclaration() {
+  if (CommonOrPrev.is<RedeclarableTemplateDecl*>())
+    return CommonOrPrev.get<RedeclarableTemplateDecl*>();
+  CommonBase *Common = CommonOrPrev.get<CommonBase*>();
+  return Common ? Common->Latest : this;
+}
+
+template <class EntryType>
+typename RedeclarableTemplateDecl::SpecEntryTraits<EntryType>::DeclType*
+RedeclarableTemplateDecl::findSpecializationImpl(
+                                 llvm::FoldingSet<EntryType> &Specs,
+                                 const TemplateArgument *Args, unsigned NumArgs,
+                                 void *&InsertPos) {
+  typedef SpecEntryTraits<EntryType> SETraits;
+  llvm::FoldingSetNodeID ID;
+  EntryType::Profile(ID,Args,NumArgs, getASTContext());
+  EntryType *Entry = Specs.FindNodeOrInsertPos(ID, InsertPos);
+  return Entry ? SETraits::getMostRecentDeclaration(Entry) : 0;
 }
 
 //===----------------------------------------------------------------------===//
@@ -107,37 +156,16 @@ FunctionTemplateDecl *FunctionTemplateDecl::Create(ASTContext &C,
   return new (C) FunctionTemplateDecl(DC, L, Name, Params, Decl);
 }
 
-void FunctionTemplateDecl::Destroy(ASTContext &C) {
-  if (Common *CommonPtr = CommonOrPrev.dyn_cast<Common*>()) {
-    for (llvm::FoldingSet<FunctionTemplateSpecializationInfo>::iterator
-              Spec = CommonPtr->Specializations.begin(),
-           SpecEnd = CommonPtr->Specializations.end();
-         Spec != SpecEnd; ++Spec)
-      C.Deallocate(&*Spec);
-  }
-
-  Decl::Destroy(C);
+RedeclarableTemplateDecl::CommonBase *FunctionTemplateDecl::newCommon() {
+  Common *CommonPtr = new (getASTContext()) Common;
+  getASTContext().AddDeallocation(DeallocateCommon, CommonPtr);
+  return CommonPtr;
 }
 
-FunctionTemplateDecl *FunctionTemplateDecl::getCanonicalDecl() {
-  FunctionTemplateDecl *FunTmpl = this;
-  while (FunTmpl->getPreviousDeclaration())
-    FunTmpl = FunTmpl->getPreviousDeclaration();
-  return FunTmpl;
-}
-
-FunctionTemplateDecl::Common *FunctionTemplateDecl::getCommonPtr() {
-  // Find the first declaration of this function template.
-  FunctionTemplateDecl *First = this;
-  while (First->getPreviousDeclaration())
-    First = First->getPreviousDeclaration();
-
-  if (First->CommonOrPrev.isNull()) {
-    Common *CommonPtr = new (getASTContext()) Common;
-    getASTContext().AddDeallocation(DeallocateCommon, CommonPtr);
-    First->CommonOrPrev = CommonPtr;
-  }
-  return First->CommonOrPrev.get<Common*>();
+FunctionDecl *
+FunctionTemplateDecl::findSpecialization(const TemplateArgument *Args,
+                                         unsigned NumArgs, void *&InsertPos) {
+  return findSpecializationImpl(getSpecializations(), Args, NumArgs, InsertPos);
 }
 
 //===----------------------------------------------------------------------===//
@@ -148,13 +176,6 @@ void ClassTemplateDecl::DeallocateCommon(void *Ptr) {
   static_cast<Common *>(Ptr)->~Common();
 }
 
-ClassTemplateDecl *ClassTemplateDecl::getCanonicalDecl() {
-  ClassTemplateDecl *Template = this;
-  while (Template->getPreviousDeclaration())
-    Template = Template->getPreviousDeclaration();
-  return Template;
-}
-
 ClassTemplateDecl *ClassTemplateDecl::Create(ASTContext &C,
                                              DeclContext *DC,
                                              SourceLocation L,
@@ -167,8 +188,24 @@ ClassTemplateDecl *ClassTemplateDecl::Create(ASTContext &C,
   return New;
 }
 
-void ClassTemplateDecl::Destroy(ASTContext& C) {
-  Decl::Destroy(C);
+RedeclarableTemplateDecl::CommonBase *ClassTemplateDecl::newCommon() {
+  Common *CommonPtr = new (getASTContext()) Common;
+  getASTContext().AddDeallocation(DeallocateCommon, CommonPtr);
+  return CommonPtr;
+}
+
+ClassTemplateSpecializationDecl *
+ClassTemplateDecl::findSpecialization(const TemplateArgument *Args,
+                                      unsigned NumArgs, void *&InsertPos) {
+  return findSpecializationImpl(getSpecializations(), Args, NumArgs, InsertPos);
+}
+
+ClassTemplatePartialSpecializationDecl *
+ClassTemplateDecl::findPartialSpecialization(const TemplateArgument *Args,
+                                             unsigned NumArgs,
+                                             void *&InsertPos) {
+  return findSpecializationImpl(getPartialSpecializations(), Args, NumArgs,
+                                InsertPos);
 }
 
 void ClassTemplateDecl::getPartialSpecializations(
@@ -181,7 +218,7 @@ void ClassTemplateDecl::getPartialSpecializations(
        P = PartialSpecs.begin(), PEnd = PartialSpecs.end();
        P != PEnd; ++P) {
     assert(!PS[P->getSequenceNumber()]);
-    PS[P->getSequenceNumber()] = &*P;
+    PS[P->getSequenceNumber()] = P->getMostRecentDeclaration();
   }
 }
 
@@ -194,7 +231,22 @@ ClassTemplateDecl::findPartialSpecialization(QualType T) {
                           PEnd = getPartialSpecializations().end();
        P != PEnd; ++P) {
     if (Context.hasSameType(P->getInjectedSpecializationType(), T))
-      return &*P;
+      return P->getMostRecentDeclaration();
+  }
+
+  return 0;
+}
+
+ClassTemplatePartialSpecializationDecl *
+ClassTemplateDecl::findPartialSpecInstantiatedFromMember(
+                                    ClassTemplatePartialSpecializationDecl *D) {
+  Decl *DCanon = D->getCanonicalDecl();
+  for (llvm::FoldingSet<ClassTemplatePartialSpecializationDecl>::iterator
+            P = getPartialSpecializations().begin(),
+         PEnd = getPartialSpecializations().end();
+       P != PEnd; ++P) {
+    if (P->getInstantiatedFromMember()->getCanonicalDecl() == DCanon)
+      return P->getMostRecentDeclaration();
   }
 
   return 0;
@@ -239,20 +291,6 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
   return CommonPtr->InjectedClassNameType;
 }
 
-ClassTemplateDecl::Common *ClassTemplateDecl::getCommonPtr() {
-  // Find the first declaration of this function template.
-  ClassTemplateDecl *First = this;
-  while (First->getPreviousDeclaration())
-    First = First->getPreviousDeclaration();
-
-  if (First->CommonOrPrev.isNull()) {
-    Common *CommonPtr = new (getASTContext()) Common;
-    getASTContext().AddDeallocation(DeallocateCommon, CommonPtr);
-    First->CommonOrPrev = CommonPtr;
-  }
-  return First->CommonOrPrev.get<Common*>();
-}
-
 //===----------------------------------------------------------------------===//
 // TemplateTypeParm Allocation/Deallocation Method Implementations
 //===----------------------------------------------------------------------===//
@@ -431,15 +469,6 @@ StructuredArguments.setPointer(NewArgs);
 StructuredArguments.setInt(0); // Doesn't own the pointer.
 }
 
-void TemplateArgumentList::Destroy(ASTContext &C) {
-  if (FlatArguments.getInt())
-    C.Deallocate((void*)FlatArguments.getPointer());
-  if (StructuredArguments.getInt())
-    C.Deallocate((void*)StructuredArguments.getPointer());
-}
-
-TemplateArgumentList::~TemplateArgumentList() {}
-
 //===----------------------------------------------------------------------===//
 // ClassTemplateSpecializationDecl Implementation
 //===----------------------------------------------------------------------===//
@@ -487,16 +516,6 @@ ClassTemplateSpecializationDecl::Create(ASTContext &Context, EmptyShell Empty) {
     new (Context)ClassTemplateSpecializationDecl(ClassTemplateSpecialization);
 }
 
-void ClassTemplateSpecializationDecl::Destroy(ASTContext &C) {
-  delete ExplicitInfo;
-
-  if (SpecializedPartialSpecialization *PartialSpec
-        = SpecializedTemplate.dyn_cast<SpecializedPartialSpecialization*>())
-    C.Deallocate(PartialSpec);
-
-  CXXRecordDecl::Destroy(C);
-}
-
 void
 ClassTemplateSpecializationDecl::getNameForDiagnostic(std::string &S,
                                                   const PrintingPolicy &Policy,
@@ -584,3 +603,8 @@ FriendTemplateDecl *FriendTemplateDecl::Create(ASTContext &Context,
     = new (Context) FriendTemplateDecl(DC, L, NParams, Params, Friend, FLoc);
   return Result;
 }
+
+FriendTemplateDecl *FriendTemplateDecl::Create(ASTContext &Context,
+                                               EmptyShell Empty) {
+  return new (Context) FriendTemplateDecl(Empty);
+}
diff --git a/lib/AST/DeclarationName.cpp b/lib/AST/DeclarationName.cpp
index 343d403e76ad..860a0b276a79 100644
--- a/lib/AST/DeclarationName.cpp
+++ b/lib/AST/DeclarationName.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/DenseMap.h"
@@ -404,26 +405,6 @@ DeclarationNameTable::~DeclarationNameTable() {
     = static_cast<llvm::FoldingSet<CXXLiteralOperatorIdName>*>
         (CXXLiteralOperatorNames);
 
-  if (Ctx.FreeMemory) {
-    llvm::FoldingSetIterator<CXXSpecialName>
-      SI = SpecialNames->begin(), SE = SpecialNames->end();
-
-    while (SI != SE) {
-      CXXSpecialName *n = &*SI++;
-      Ctx.Deallocate(n);
-    }
-
-    llvm::FoldingSetIterator<CXXLiteralOperatorIdName>
-      LI = LiteralNames->begin(), LE = LiteralNames->end();
-
-    while (LI != LE) {
-      CXXLiteralOperatorIdName *n = &*LI++;
-      Ctx.Deallocate(n);
-    }
-
-    Ctx.Deallocate(CXXOperatorNames);
-  }
-
   delete SpecialNames;
   delete LiteralNames;
 }
@@ -505,3 +486,98 @@ getHashValue(clang::DeclarationName N) {
   return DenseMapInfo<void*>::getHashValue(N.getAsOpaquePtr());
 }
 
+DeclarationNameLoc::DeclarationNameLoc(DeclarationName Name) {
+  switch (Name.getNameKind()) {
+  case DeclarationName::Identifier:
+    break;
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    NamedType.TInfo = 0;
+    break;
+  case DeclarationName::CXXOperatorName:
+    CXXOperatorName.BeginOpNameLoc = SourceLocation().getRawEncoding();
+    CXXOperatorName.EndOpNameLoc = SourceLocation().getRawEncoding();
+    break;
+  case DeclarationName::CXXLiteralOperatorName:
+    CXXLiteralOperatorName.OpNameLoc = SourceLocation().getRawEncoding();
+    break;
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+    // FIXME: ?
+    break;
+  case DeclarationName::CXXUsingDirective:
+    break;
+  }
+}
+
+std::string DeclarationNameInfo::getAsString() const {
+  std::string Result;
+  llvm::raw_string_ostream OS(Result);
+  printName(OS);
+  return OS.str();
+}
+
+void DeclarationNameInfo::printName(llvm::raw_ostream &OS) const {
+  switch (Name.getNameKind()) {
+  case DeclarationName::Identifier:
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXOperatorName:
+  case DeclarationName::CXXLiteralOperatorName:
+  case DeclarationName::CXXUsingDirective:
+    Name.printName(OS);
+    return;
+
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    if (TypeSourceInfo *TInfo = LocInfo.NamedType.TInfo) {
+      if (Name.getNameKind() == DeclarationName::CXXDestructorName)
+        OS << '~';
+      else if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName)
+        OS << "operator ";
+      OS << TInfo->getType().getAsString();
+    }
+    else
+      Name.printName(OS);
+    return;
+  }
+  assert(false && "Unexpected declaration name kind");
+}
+
+SourceLocation DeclarationNameInfo::getEndLoc() const {
+  switch (Name.getNameKind()) {
+  case DeclarationName::Identifier:
+    return NameLoc;
+
+  case DeclarationName::CXXOperatorName: {
+    unsigned raw = LocInfo.CXXOperatorName.EndOpNameLoc;
+    return SourceLocation::getFromRawEncoding(raw);
+  }
+
+  case DeclarationName::CXXLiteralOperatorName: {
+    unsigned raw = LocInfo.CXXLiteralOperatorName.OpNameLoc;
+    return SourceLocation::getFromRawEncoding(raw);
+  }
+
+  case DeclarationName::CXXConstructorName:
+  case DeclarationName::CXXDestructorName:
+  case DeclarationName::CXXConversionFunctionName:
+    if (TypeSourceInfo *TInfo = LocInfo.NamedType.TInfo)
+      return TInfo->getTypeLoc().getEndLoc();
+    else
+      return NameLoc;
+
+    // DNInfo work in progress: FIXME.
+  case DeclarationName::ObjCZeroArgSelector:
+  case DeclarationName::ObjCOneArgSelector:
+  case DeclarationName::ObjCMultiArgSelector:
+  case DeclarationName::CXXUsingDirective:
+    return NameLoc;
+  }
+  assert(false && "Unexpected declaration name kind");
+  return SourceLocation();
+}
diff --git a/lib/AST/Expr.cpp b/lib/AST/Expr.cpp
index 6524a312dc8b..5feef1c80332 100644
--- a/lib/AST/Expr.cpp
+++ b/lib/AST/Expr.cpp
@@ -44,8 +44,8 @@ bool Expr::isKnownToHaveBooleanValue() const {
   
   if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(this)) {
     switch (UO->getOpcode()) {
-    case UnaryOperator::Plus:
-    case UnaryOperator::Extension:
+    case UO_Plus:
+    case UO_Extension:
       return UO->getSubExpr()->isKnownToHaveBooleanValue();
     default:
       return false;
@@ -60,25 +60,25 @@ bool Expr::isKnownToHaveBooleanValue() const {
   if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(this)) {
     switch (BO->getOpcode()) {
     default: return false;
-    case BinaryOperator::LT:   // Relational operators.
-    case BinaryOperator::GT:
-    case BinaryOperator::LE:
-    case BinaryOperator::GE:
-    case BinaryOperator::EQ:   // Equality operators.
-    case BinaryOperator::NE:
-    case BinaryOperator::LAnd: // AND operator.
-    case BinaryOperator::LOr:  // Logical OR operator.
+    case BO_LT:   // Relational operators.
+    case BO_GT:
+    case BO_LE:
+    case BO_GE:
+    case BO_EQ:   // Equality operators.
+    case BO_NE:
+    case BO_LAnd: // AND operator.
+    case BO_LOr:  // Logical OR operator.
       return true;
         
-    case BinaryOperator::And:  // Bitwise AND operator.
-    case BinaryOperator::Xor:  // Bitwise XOR operator.
-    case BinaryOperator::Or:   // Bitwise OR operator.
+    case BO_And:  // Bitwise AND operator.
+    case BO_Xor:  // Bitwise XOR operator.
+    case BO_Or:   // Bitwise OR operator.
       // Handle things like (x==2)|(y==12).
       return BO->getLHS()->isKnownToHaveBooleanValue() &&
              BO->getRHS()->isKnownToHaveBooleanValue();
         
-    case BinaryOperator::Comma:
-    case BinaryOperator::Assign:
+    case BO_Comma:
+    case BO_Assign:
       return BO->getRHS()->isKnownToHaveBooleanValue();
     }
   }
@@ -151,7 +151,7 @@ void DeclRefExpr::computeDependence() {
     ValueDependent = true;
   }
   //  (TD)  - a template-id that is dependent,
-  else if (hasExplicitTemplateArgumentList() && 
+  else if (hasExplicitTemplateArgs() && 
            TemplateSpecializationType::anyDependentTemplateArguments(
                                                        getTemplateArgs(), 
                                                        getNumTemplateArgs())) {
@@ -204,7 +204,29 @@ DeclRefExpr::DeclRefExpr(NestedNameSpecifier *Qualifier,
   }
       
   if (TemplateArgs)
-    getExplicitTemplateArgumentList()->initializeFrom(*TemplateArgs);
+    getExplicitTemplateArgs().initializeFrom(*TemplateArgs);
+
+  computeDependence();
+}
+
+DeclRefExpr::DeclRefExpr(NestedNameSpecifier *Qualifier,
+                         SourceRange QualifierRange,
+                         ValueDecl *D, const DeclarationNameInfo &NameInfo,
+                         const TemplateArgumentListInfo *TemplateArgs,
+                         QualType T)
+  : Expr(DeclRefExprClass, T, false, false),
+    DecoratedD(D,
+               (Qualifier? HasQualifierFlag : 0) |
+               (TemplateArgs ? HasExplicitTemplateArgumentListFlag : 0)),
+    Loc(NameInfo.getLoc()), DNLoc(NameInfo.getInfo()) {
+  if (Qualifier) {
+    NameQualifier *NQ = getNameQualifier();
+    NQ->NNS = Qualifier;
+    NQ->Range = QualifierRange;
+  }
+
+  if (TemplateArgs)
+    getExplicitTemplateArgs().initializeFrom(*TemplateArgs);
 
   computeDependence();
 }
@@ -216,6 +238,18 @@ DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
                                  SourceLocation NameLoc,
                                  QualType T,
                                  const TemplateArgumentListInfo *TemplateArgs) {
+  return Create(Context, Qualifier, QualifierRange, D,
+                DeclarationNameInfo(D->getDeclName(), NameLoc),
+                T, TemplateArgs);
+}
+
+DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
+                                 NestedNameSpecifier *Qualifier,
+                                 SourceRange QualifierRange,
+                                 ValueDecl *D,
+                                 const DeclarationNameInfo &NameInfo,
+                                 QualType T,
+                                 const TemplateArgumentListInfo *TemplateArgs) {
   std::size_t Size = sizeof(DeclRefExpr);
   if (Qualifier != 0)
     Size += sizeof(NameQualifier);
@@ -224,7 +258,7 @@ DeclRefExpr *DeclRefExpr::Create(ASTContext &Context,
     Size += ExplicitTemplateArgumentList::sizeFor(*TemplateArgs);
   
   void *Mem = Context.Allocate(Size, llvm::alignof<DeclRefExpr>());
-  return new (Mem) DeclRefExpr(Qualifier, QualifierRange, D, NameLoc,
+  return new (Mem) DeclRefExpr(Qualifier, QualifierRange, D, NameInfo,
                                TemplateArgs, T);
 }
 
@@ -242,12 +276,10 @@ DeclRefExpr *DeclRefExpr::CreateEmpty(ASTContext &Context, bool HasQualifier,
 }
 
 SourceRange DeclRefExpr::getSourceRange() const {
-  // FIXME: Does not handle multi-token names well, e.g., operator[].
-  SourceRange R(Loc);
-  
+  SourceRange R = getNameInfo().getSourceRange();
   if (hasQualifier())
     R.setBegin(getQualifierRange().getBegin());
-  if (hasExplicitTemplateArgumentList())
+  if (hasExplicitTemplateArgs())
     R.setEnd(getRAngleLoc());
   return R;
 }
@@ -342,6 +374,44 @@ std::string PredefinedExpr::ComputeName(IdentType IT, const Decl *CurrentDecl) {
   return "";
 }
 
+void APNumericStorage::setIntValue(ASTContext &C, const llvm::APInt &Val) {
+  if (hasAllocation())
+    C.Deallocate(pVal);
+
+  BitWidth = Val.getBitWidth();
+  unsigned NumWords = Val.getNumWords();
+  const uint64_t* Words = Val.getRawData();
+  if (NumWords > 1) {
+    pVal = new (C) uint64_t[NumWords];
+    std::copy(Words, Words + NumWords, pVal);
+  } else if (NumWords == 1)
+    VAL = Words[0];
+  else
+    VAL = 0;
+}
+
+IntegerLiteral *
+IntegerLiteral::Create(ASTContext &C, const llvm::APInt &V,
+                       QualType type, SourceLocation l) {
+  return new (C) IntegerLiteral(C, V, type, l);
+}
+
+IntegerLiteral *
+IntegerLiteral::Create(ASTContext &C, EmptyShell Empty) {
+  return new (C) IntegerLiteral(Empty);
+}
+
+FloatingLiteral *
+FloatingLiteral::Create(ASTContext &C, const llvm::APFloat &V,
+                        bool isexact, QualType Type, SourceLocation L) {
+  return new (C) FloatingLiteral(C, V, isexact, Type, L);
+}
+
+FloatingLiteral *
+FloatingLiteral::Create(ASTContext &C, EmptyShell Empty) {
+  return new (C) FloatingLiteral(Empty);
+}
+
 /// getValueAsApproximateDouble - This returns the value as an inaccurate
 /// double.  Note that this may cause loss of precision, but is useful for
 /// debugging dumps, etc.
@@ -390,15 +460,7 @@ StringLiteral *StringLiteral::CreateEmpty(ASTContext &C, unsigned NumStrs) {
   return SL;
 }
 
-void StringLiteral::DoDestroy(ASTContext &C) {
-  C.Deallocate(const_cast<char*>(StrData));
-  Expr::DoDestroy(C);
-}
-
 void StringLiteral::setString(ASTContext &C, llvm::StringRef Str) {
-  if (StrData)
-    C.Deallocate(const_cast<char*>(StrData));
-
   char *AStrData = new (C, 1) char[Str.size()];
   memcpy(AStrData, Str.data(), Str.size());
   StrData = AStrData;
@@ -410,48 +472,47 @@ void StringLiteral::setString(ASTContext &C, llvm::StringRef Str) {
 const char *UnaryOperator::getOpcodeStr(Opcode Op) {
   switch (Op) {
   default: assert(0 && "Unknown unary operator");
-  case PostInc: return "++";
-  case PostDec: return "--";
-  case PreInc:  return "++";
-  case PreDec:  return "--";
-  case AddrOf:  return "&";
-  case Deref:   return "*";
-  case Plus:    return "+";
-  case Minus:   return "-";
-  case Not:     return "~";
-  case LNot:    return "!";
-  case Real:    return "__real";
-  case Imag:    return "__imag";
-  case Extension: return "__extension__";
-  case OffsetOf: return "__builtin_offsetof";
+  case UO_PostInc: return "++";
+  case UO_PostDec: return "--";
+  case UO_PreInc:  return "++";
+  case UO_PreDec:  return "--";
+  case UO_AddrOf:  return "&";
+  case UO_Deref:   return "*";
+  case UO_Plus:    return "+";
+  case UO_Minus:   return "-";
+  case UO_Not:     return "~";
+  case UO_LNot:    return "!";
+  case UO_Real:    return "__real";
+  case UO_Imag:    return "__imag";
+  case UO_Extension: return "__extension__";
   }
 }
 
-UnaryOperator::Opcode
+UnaryOperatorKind
 UnaryOperator::getOverloadedOpcode(OverloadedOperatorKind OO, bool Postfix) {
   switch (OO) {
   default: assert(false && "No unary operator for overloaded function");
-  case OO_PlusPlus:   return Postfix ? PostInc : PreInc;
-  case OO_MinusMinus: return Postfix ? PostDec : PreDec;
-  case OO_Amp:        return AddrOf;
-  case OO_Star:       return Deref;
-  case OO_Plus:       return Plus;
-  case OO_Minus:      return Minus;
-  case OO_Tilde:      return Not;
-  case OO_Exclaim:    return LNot;
+  case OO_PlusPlus:   return Postfix ? UO_PostInc : UO_PreInc;
+  case OO_MinusMinus: return Postfix ? UO_PostDec : UO_PreDec;
+  case OO_Amp:        return UO_AddrOf;
+  case OO_Star:       return UO_Deref;
+  case OO_Plus:       return UO_Plus;
+  case OO_Minus:      return UO_Minus;
+  case OO_Tilde:      return UO_Not;
+  case OO_Exclaim:    return UO_LNot;
   }
 }
 
 OverloadedOperatorKind UnaryOperator::getOverloadedOperator(Opcode Opc) {
   switch (Opc) {
-  case PostInc: case PreInc: return OO_PlusPlus;
-  case PostDec: case PreDec: return OO_MinusMinus;
-  case AddrOf: return OO_Amp;
-  case Deref: return OO_Star;
-  case Plus: return OO_Plus;
-  case Minus: return OO_Minus;
-  case Not: return OO_Tilde;
-  case LNot: return OO_Exclaim;
+  case UO_PostInc: case UO_PreInc: return OO_PlusPlus;
+  case UO_PostDec: case UO_PreDec: return OO_MinusMinus;
+  case UO_AddrOf: return OO_Amp;
+  case UO_Deref: return OO_Star;
+  case UO_Plus: return OO_Plus;
+  case UO_Minus: return OO_Minus;
+  case UO_Not: return OO_Tilde;
+  case UO_LNot: return OO_Exclaim;
   default: return OO_None;
   }
 }
@@ -496,13 +557,6 @@ CallExpr::CallExpr(ASTContext &C, StmtClass SC, EmptyShell Empty)
   SubExprs = new (C) Stmt*[1];
 }
 
-void CallExpr::DoDestroy(ASTContext& C) {
-  DestroyChildren(C);
-  if (SubExprs) C.Deallocate(SubExprs);
-  this->~CallExpr();
-  C.Deallocate(this);
-}
-
 Decl *CallExpr::getCalleeDecl() {
   Expr *CEE = getCallee()->IgnoreParenCasts();
   if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(CEE))
@@ -526,8 +580,6 @@ void CallExpr::setNumArgs(ASTContext& C, unsigned NumArgs) {
 
   // If shrinking # arguments, just delete the extras and forgot them.
   if (NumArgs < getNumArgs()) {
-    for (unsigned i = NumArgs, e = getNumArgs(); i != e; ++i)
-      getArg(i)->Destroy(C);
     this->NumArgs = NumArgs;
     return;
   }
@@ -640,7 +692,7 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
                                SourceRange qualrange,
                                ValueDecl *memberdecl,
                                DeclAccessPair founddecl,
-                               SourceLocation l,
+                               DeclarationNameInfo nameinfo,
                                const TemplateArgumentListInfo *targs,
                                QualType ty) {
   std::size_t Size = sizeof(MemberExpr);
@@ -655,7 +707,7 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
     Size += ExplicitTemplateArgumentList::sizeFor(*targs);
 
   void *Mem = C.Allocate(Size, llvm::alignof<MemberExpr>());
-  MemberExpr *E = new (Mem) MemberExpr(base, isarrow, memberdecl, l, ty);
+  MemberExpr *E = new (Mem) MemberExpr(base, isarrow, memberdecl, nameinfo, ty);
 
   if (hasQualOrFound) {
     if (qual && qual->isDependent()) {
@@ -672,7 +724,7 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
 
   if (targs) {
     E->HasExplicitTemplateArgumentList = true;
-    E->getExplicitTemplateArgumentList()->initializeFrom(*targs);
+    E->getExplicitTemplateArgs().initializeFrom(*targs);
   }
 
   return E;
@@ -680,72 +732,68 @@ MemberExpr *MemberExpr::Create(ASTContext &C, Expr *base, bool isarrow,
 
 const char *CastExpr::getCastKindName() const {
   switch (getCastKind()) {
-  case CastExpr::CK_Unknown:
+  case CK_Unknown:
     return "Unknown";
-  case CastExpr::CK_BitCast:
+  case CK_BitCast:
     return "BitCast";
-  case CastExpr::CK_LValueBitCast:
+  case CK_LValueBitCast:
     return "LValueBitCast";
-  case CastExpr::CK_NoOp:
+  case CK_NoOp:
     return "NoOp";
-  case CastExpr::CK_BaseToDerived:
+  case CK_BaseToDerived:
     return "BaseToDerived";
-  case CastExpr::CK_DerivedToBase:
+  case CK_DerivedToBase:
     return "DerivedToBase";
-  case CastExpr::CK_UncheckedDerivedToBase:
+  case CK_UncheckedDerivedToBase:
     return "UncheckedDerivedToBase";
-  case CastExpr::CK_Dynamic:
+  case CK_Dynamic:
     return "Dynamic";
-  case CastExpr::CK_ToUnion:
+  case CK_ToUnion:
     return "ToUnion";
-  case CastExpr::CK_ArrayToPointerDecay:
+  case CK_ArrayToPointerDecay:
     return "ArrayToPointerDecay";
-  case CastExpr::CK_FunctionToPointerDecay:
+  case CK_FunctionToPointerDecay:
     return "FunctionToPointerDecay";
-  case CastExpr::CK_NullToMemberPointer:
+  case CK_NullToMemberPointer:
     return "NullToMemberPointer";
-  case CastExpr::CK_BaseToDerivedMemberPointer:
+  case CK_BaseToDerivedMemberPointer:
     return "BaseToDerivedMemberPointer";
-  case CastExpr::CK_DerivedToBaseMemberPointer:
+  case CK_DerivedToBaseMemberPointer:
     return "DerivedToBaseMemberPointer";
-  case CastExpr::CK_UserDefinedConversion:
+  case CK_UserDefinedConversion:
     return "UserDefinedConversion";
-  case CastExpr::CK_ConstructorConversion:
+  case CK_ConstructorConversion:
     return "ConstructorConversion";
-  case CastExpr::CK_IntegralToPointer:
+  case CK_IntegralToPointer:
     return "IntegralToPointer";
-  case CastExpr::CK_PointerToIntegral:
+  case CK_PointerToIntegral:
     return "PointerToIntegral";
-  case CastExpr::CK_ToVoid:
+  case CK_ToVoid:
     return "ToVoid";
-  case CastExpr::CK_VectorSplat:
+  case CK_VectorSplat:
     return "VectorSplat";
-  case CastExpr::CK_IntegralCast:
+  case CK_IntegralCast:
     return "IntegralCast";
-  case CastExpr::CK_IntegralToFloating:
+  case CK_IntegralToFloating:
     return "IntegralToFloating";
-  case CastExpr::CK_FloatingToIntegral:
+  case CK_FloatingToIntegral:
     return "FloatingToIntegral";
-  case CastExpr::CK_FloatingCast:
+  case CK_FloatingCast:
     return "FloatingCast";
-  case CastExpr::CK_MemberPointerToBoolean:
+  case CK_MemberPointerToBoolean:
     return "MemberPointerToBoolean";
-  case CastExpr::CK_AnyPointerToObjCPointerCast:
+  case CK_AnyPointerToObjCPointerCast:
     return "AnyPointerToObjCPointerCast";
-  case CastExpr::CK_AnyPointerToBlockPointerCast:
+  case CK_AnyPointerToBlockPointerCast:
     return "AnyPointerToBlockPointerCast";
+  case CK_ObjCObjectLValueCast:
+    return "ObjCObjectLValueCast";
   }
 
   assert(0 && "Unhandled cast kind!");
   return 0;
 }
 
-void CastExpr::DoDestroy(ASTContext &C)
-{
-  BasePath.Destroy();
-  Expr::DoDestroy(C);
-}
-
 Expr *CastExpr::getSubExprAsWritten() {
   Expr *SubExpr = 0;
   CastExpr *E = this;
@@ -758,9 +806,9 @@ Expr *CastExpr::getSubExprAsWritten() {
     
     // Conversions by constructor and conversion functions have a
     // subexpression describing the call; strip it off.
-    if (E->getCastKind() == CastExpr::CK_ConstructorConversion)
+    if (E->getCastKind() == CK_ConstructorConversion)
       SubExpr = cast<CXXConstructExpr>(SubExpr)->getArg(0);
-    else if (E->getCastKind() == CastExpr::CK_UserDefinedConversion)
+    else if (E->getCastKind() == CK_UserDefinedConversion)
       SubExpr = cast<CXXMemberCallExpr>(SubExpr)->getImplicitObjectArgument();
     
     // If the subexpression we're left with is an implicit cast, look
@@ -770,82 +818,142 @@ Expr *CastExpr::getSubExprAsWritten() {
   return SubExpr;
 }
 
+CXXBaseSpecifier **CastExpr::path_buffer() {
+  switch (getStmtClass()) {
+#define ABSTRACT_STMT(x)
+#define CASTEXPR(Type, Base) \
+  case Stmt::Type##Class: \
+    return reinterpret_cast<CXXBaseSpecifier**>(static_cast<Type*>(this)+1);
+#define STMT(Type, Base)
+#include "clang/AST/StmtNodes.inc"
+  default:
+    llvm_unreachable("non-cast expressions not possible here");
+    return 0;
+  }
+}
+
+void CastExpr::setCastPath(const CXXCastPath &Path) {
+  assert(Path.size() == path_size());
+  memcpy(path_buffer(), Path.data(), Path.size() * sizeof(CXXBaseSpecifier*));
+}
+
+ImplicitCastExpr *ImplicitCastExpr::Create(ASTContext &C, QualType T,
+                                           CastKind Kind, Expr *Operand,
+                                           const CXXCastPath *BasePath,
+                                           ExprValueKind VK) {
+  unsigned PathSize = (BasePath ? BasePath->size() : 0);
+  void *Buffer =
+    C.Allocate(sizeof(ImplicitCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  ImplicitCastExpr *E =
+    new (Buffer) ImplicitCastExpr(T, Kind, Operand, PathSize, VK);
+  if (PathSize) E->setCastPath(*BasePath);
+  return E;
+}
+
+ImplicitCastExpr *ImplicitCastExpr::CreateEmpty(ASTContext &C,
+                                                unsigned PathSize) {
+  void *Buffer =
+    C.Allocate(sizeof(ImplicitCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  return new (Buffer) ImplicitCastExpr(EmptyShell(), PathSize);
+}
+
+
+CStyleCastExpr *CStyleCastExpr::Create(ASTContext &C, QualType T,
+                                       CastKind K, Expr *Op,
+                                       const CXXCastPath *BasePath,
+                                       TypeSourceInfo *WrittenTy,
+                                       SourceLocation L, SourceLocation R) {
+  unsigned PathSize = (BasePath ? BasePath->size() : 0);
+  void *Buffer =
+    C.Allocate(sizeof(CStyleCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  CStyleCastExpr *E =
+    new (Buffer) CStyleCastExpr(T, K, Op, PathSize, WrittenTy, L, R);
+  if (PathSize) E->setCastPath(*BasePath);
+  return E;
+}
+
+CStyleCastExpr *CStyleCastExpr::CreateEmpty(ASTContext &C, unsigned PathSize) {
+  void *Buffer =
+    C.Allocate(sizeof(CStyleCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  return new (Buffer) CStyleCastExpr(EmptyShell(), PathSize);
+}
+
 /// getOpcodeStr - Turn an Opcode enum value into the punctuation char it
 /// corresponds to, e.g. "<<=".
 const char *BinaryOperator::getOpcodeStr(Opcode Op) {
   switch (Op) {
-  case PtrMemD:   return ".*";
-  case PtrMemI:   return "->*";
-  case Mul:       return "*";
-  case Div:       return "/";
-  case Rem:       return "%";
-  case Add:       return "+";
-  case Sub:       return "-";
-  case Shl:       return "<<";
-  case Shr:       return ">>";
-  case LT:        return "<";
-  case GT:        return ">";
-  case LE:        return "<=";
-  case GE:        return ">=";
-  case EQ:        return "==";
-  case NE:        return "!=";
-  case And:       return "&";
-  case Xor:       return "^";
-  case Or:        return "|";
-  case LAnd:      return "&&";
-  case LOr:       return "||";
-  case Assign:    return "=";
-  case MulAssign: return "*=";
-  case DivAssign: return "/=";
-  case RemAssign: return "%=";
-  case AddAssign: return "+=";
-  case SubAssign: return "-=";
-  case ShlAssign: return "<<=";
-  case ShrAssign: return ">>=";
-  case AndAssign: return "&=";
-  case XorAssign: return "^=";
-  case OrAssign:  return "|=";
-  case Comma:     return ",";
+  case BO_PtrMemD:   return ".*";
+  case BO_PtrMemI:   return "->*";
+  case BO_Mul:       return "*";
+  case BO_Div:       return "/";
+  case BO_Rem:       return "%";
+  case BO_Add:       return "+";
+  case BO_Sub:       return "-";
+  case BO_Shl:       return "<<";
+  case BO_Shr:       return ">>";
+  case BO_LT:        return "<";
+  case BO_GT:        return ">";
+  case BO_LE:        return "<=";
+  case BO_GE:        return ">=";
+  case BO_EQ:        return "==";
+  case BO_NE:        return "!=";
+  case BO_And:       return "&";
+  case BO_Xor:       return "^";
+  case BO_Or:        return "|";
+  case BO_LAnd:      return "&&";
+  case BO_LOr:       return "||";
+  case BO_Assign:    return "=";
+  case BO_MulAssign: return "*=";
+  case BO_DivAssign: return "/=";
+  case BO_RemAssign: return "%=";
+  case BO_AddAssign: return "+=";
+  case BO_SubAssign: return "-=";
+  case BO_ShlAssign: return "<<=";
+  case BO_ShrAssign: return ">>=";
+  case BO_AndAssign: return "&=";
+  case BO_XorAssign: return "^=";
+  case BO_OrAssign:  return "|=";
+  case BO_Comma:     return ",";
   }
 
   return "";
 }
 
-BinaryOperator::Opcode
+BinaryOperatorKind
 BinaryOperator::getOverloadedOpcode(OverloadedOperatorKind OO) {
   switch (OO) {
   default: assert(false && "Not an overloadable binary operator");
-  case OO_Plus: return Add;
-  case OO_Minus: return Sub;
-  case OO_Star: return Mul;
-  case OO_Slash: return Div;
-  case OO_Percent: return Rem;
-  case OO_Caret: return Xor;
-  case OO_Amp: return And;
-  case OO_Pipe: return Or;
-  case OO_Equal: return Assign;
-  case OO_Less: return LT;
-  case OO_Greater: return GT;
-  case OO_PlusEqual: return AddAssign;
-  case OO_MinusEqual: return SubAssign;
-  case OO_StarEqual: return MulAssign;
-  case OO_SlashEqual: return DivAssign;
-  case OO_PercentEqual: return RemAssign;
-  case OO_CaretEqual: return XorAssign;
-  case OO_AmpEqual: return AndAssign;
-  case OO_PipeEqual: return OrAssign;
-  case OO_LessLess: return Shl;
-  case OO_GreaterGreater: return Shr;
-  case OO_LessLessEqual: return ShlAssign;
-  case OO_GreaterGreaterEqual: return ShrAssign;
-  case OO_EqualEqual: return EQ;
-  case OO_ExclaimEqual: return NE;
-  case OO_LessEqual: return LE;
-  case OO_GreaterEqual: return GE;
-  case OO_AmpAmp: return LAnd;
-  case OO_PipePipe: return LOr;
-  case OO_Comma: return Comma;
-  case OO_ArrowStar: return PtrMemI;
+  case OO_Plus: return BO_Add;
+  case OO_Minus: return BO_Sub;
+  case OO_Star: return BO_Mul;
+  case OO_Slash: return BO_Div;
+  case OO_Percent: return BO_Rem;
+  case OO_Caret: return BO_Xor;
+  case OO_Amp: return BO_And;
+  case OO_Pipe: return BO_Or;
+  case OO_Equal: return BO_Assign;
+  case OO_Less: return BO_LT;
+  case OO_Greater: return BO_GT;
+  case OO_PlusEqual: return BO_AddAssign;
+  case OO_MinusEqual: return BO_SubAssign;
+  case OO_StarEqual: return BO_MulAssign;
+  case OO_SlashEqual: return BO_DivAssign;
+  case OO_PercentEqual: return BO_RemAssign;
+  case OO_CaretEqual: return BO_XorAssign;
+  case OO_AmpEqual: return BO_AndAssign;
+  case OO_PipeEqual: return BO_OrAssign;
+  case OO_LessLess: return BO_Shl;
+  case OO_GreaterGreater: return BO_Shr;
+  case OO_LessLessEqual: return BO_ShlAssign;
+  case OO_GreaterGreaterEqual: return BO_ShrAssign;
+  case OO_EqualEqual: return BO_EQ;
+  case OO_ExclaimEqual: return BO_NE;
+  case OO_LessEqual: return BO_LE;
+  case OO_GreaterEqual: return BO_GE;
+  case OO_AmpAmp: return BO_LAnd;
+  case OO_PipePipe: return BO_LOr;
+  case OO_Comma: return BO_Comma;
+  case OO_ArrowStar: return BO_PtrMemI;
   }
 }
 
@@ -897,9 +1005,6 @@ void InitListExpr::reserveInits(ASTContext &C, unsigned NumInits) {
 }
 
 void InitListExpr::resizeInits(ASTContext &C, unsigned NumInits) {
-  for (unsigned Idx = NumInits, LastIdx = InitExprs.size();
-       Idx < LastIdx; ++Idx)
-    InitExprs[Idx]->Destroy(C);
   InitExprs.resize(C, NumInits, 0);
 }
 
@@ -963,24 +1068,24 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
 
     switch (UO->getOpcode()) {
     default: break;
-    case UnaryOperator::PostInc:
-    case UnaryOperator::PostDec:
-    case UnaryOperator::PreInc:
-    case UnaryOperator::PreDec:                 // ++/--
+    case UO_PostInc:
+    case UO_PostDec:
+    case UO_PreInc:
+    case UO_PreDec:                 // ++/--
       return false;  // Not a warning.
-    case UnaryOperator::Deref:
+    case UO_Deref:
       // Dereferencing a volatile pointer is a side-effect.
       if (Ctx.getCanonicalType(getType()).isVolatileQualified())
         return false;
       break;
-    case UnaryOperator::Real:
-    case UnaryOperator::Imag:
+    case UO_Real:
+    case UO_Imag:
       // accessing a piece of a volatile complex is a side-effect.
       if (Ctx.getCanonicalType(UO->getSubExpr()->getType())
           .isVolatileQualified())
         return false;
       break;
-    case UnaryOperator::Extension:
+    case UO_Extension:
       return UO->getSubExpr()->isUnusedResultAWarning(Loc, R1, R2, Ctx);
     }
     Loc = UO->getOperatorLoc();
@@ -994,7 +1099,7 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
         break;
       // Consider the RHS of comma for side effects. LHS was checked by
       // Sema::CheckCommaOperands.
-      case BinaryOperator::Comma:
+      case BO_Comma:
         // ((foo = <blah>), 0) is an idiom for hiding the result (and
         // lvalue-ness) of an assignment written in a macro.
         if (IntegerLiteral *IE =
@@ -1003,8 +1108,8 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
             return false;
         return BO->getRHS()->isUnusedResultAWarning(Loc, R1, R2, Ctx);
       // Consider '||', '&&' to have side effects if the LHS or RHS does.
-      case BinaryOperator::LAnd:
-      case BinaryOperator::LOr:
+      case BO_LAnd:
+      case BO_LOr:
         if (!BO->getLHS()->isUnusedResultAWarning(Loc, R1, R2, Ctx) ||
             !BO->getRHS()->isUnusedResultAWarning(Loc, R1, R2, Ctx))
           return false;
@@ -1137,8 +1242,8 @@ bool Expr::isUnusedResultAWarning(SourceLocation &Loc, SourceRange &R1,
     
     // If this is a cast to void or a constructor conversion, check the operand.
     // Otherwise, the result of the cast is unused.
-    if (CE->getCastKind() == CastExpr::CK_ToVoid ||
-        CE->getCastKind() == CastExpr::CK_ConstructorConversion)
+    if (CE->getCastKind() == CK_ToVoid ||
+        CE->getCastKind() == CK_ConstructorConversion)
       return (cast<CastExpr>(this)->getSubExpr()
               ->isUnusedResultAWarning(Loc, R1, R2, Ctx));
     Loc = cast<CXXFunctionalCastExpr>(this)->getTypeBeginLoc();
@@ -1287,7 +1392,7 @@ bool Expr::isDefaultArgument() const {
 /// expressions.
 static const Expr *skipTemporaryBindingsAndNoOpCasts(const Expr *E) {
   while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-    if (ICE->getCastKind() == CastExpr::CK_NoOp)
+    if (ICE->getCastKind() == CK_NoOp)
       E = ICE->getSubExpr();
     else
       break;
@@ -1297,7 +1402,7 @@ static const Expr *skipTemporaryBindingsAndNoOpCasts(const Expr *E) {
     E = BE->getSubExpr();
 
   while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-    if (ICE->getCastKind() == CastExpr::CK_NoOp)
+    if (ICE->getCastKind() == CK_NoOp)
       E = ICE->getSubExpr();
     else
       break;
@@ -1314,8 +1419,8 @@ const Expr *Expr::getTemporaryObject() const {
   if (const CastExpr *Cast = dyn_cast<CastExpr>(E)) {
     // Only user-defined and constructor conversions can produce
     // temporary objects.
-    if (Cast->getCastKind() != CastExpr::CK_ConstructorConversion &&
-        Cast->getCastKind() != CastExpr::CK_UserDefinedConversion)
+    if (Cast->getCastKind() != CK_ConstructorConversion &&
+        Cast->getCastKind() != CK_UserDefinedConversion)
       return 0;
 
     // Strip off temporary bindings and no-op casts.
@@ -1323,12 +1428,12 @@ const Expr *Expr::getTemporaryObject() const {
 
     // If this is a constructor conversion, see if we have an object
     // construction.
-    if (Cast->getCastKind() == CastExpr::CK_ConstructorConversion)
+    if (Cast->getCastKind() == CK_ConstructorConversion)
       return dyn_cast<CXXConstructExpr>(Sub);
 
     // If this is a user-defined conversion, see if we have a call to
     // a function that itself returns a temporary object.
-    if (Cast->getCastKind() == CastExpr::CK_UserDefinedConversion)
+    if (Cast->getCastKind() == CK_UserDefinedConversion)
       if (const CallExpr *CE = dyn_cast<CallExpr>(Sub))
         if (CE->getCallReturnType()->isRecordType())
           return CE;
@@ -1368,15 +1473,20 @@ bool Expr::hasAnyValueDependentArguments(Expr** Exprs, unsigned NumExprs) {
   return false;
 }
 
-bool Expr::isConstantInitializer(ASTContext &Ctx) const {
+bool Expr::isConstantInitializer(ASTContext &Ctx, bool IsForRef) const {
   // This function is attempting whether an expression is an initializer
   // which can be evaluated at compile-time.  isEvaluatable handles most
   // of the cases, but it can't deal with some initializer-specific
   // expressions, and it can't deal with aggregates; we deal with those here,
   // and fall back to isEvaluatable for the other cases.
 
-  // FIXME: This function assumes the variable being assigned to
-  // isn't a reference type!
+  // If we ever capture reference-binding directly in the AST, we can
+  // kill the second parameter.
+
+  if (IsForRef) {
+    EvalResult Result;
+    return EvaluateAsLValue(Result, Ctx) && !Result.HasSideEffects;
+  }
 
   switch (getStmtClass()) {
   default: break;
@@ -1384,12 +1494,27 @@ bool Expr::isConstantInitializer(ASTContext &Ctx) const {
   case ObjCStringLiteralClass:
   case ObjCEncodeExprClass:
     return true;
+  case CXXTemporaryObjectExprClass:
+  case CXXConstructExprClass: {
+    const CXXConstructExpr *CE = cast<CXXConstructExpr>(this);
+
+    // Only if it's
+    // 1) an application of the trivial default constructor or
+    if (!CE->getConstructor()->isTrivial()) return false;
+    if (!CE->getNumArgs()) return true;
+
+    // 2) an elidable trivial copy construction of an operand which is
+    //    itself a constant initializer.  Note that we consider the
+    //    operand on its own, *not* as a reference binding.
+    return CE->isElidable() &&
+           CE->getArg(0)->isConstantInitializer(Ctx, false);
+  }
   case CompoundLiteralExprClass: {
     // This handles gcc's extension that allows global initializers like
     // "struct x {int x;} x = (struct x) {};".
     // FIXME: This accepts other cases it shouldn't!
     const Expr *Exp = cast<CompoundLiteralExpr>(this)->getInitializer();
-    return Exp->isConstantInitializer(Ctx);
+    return Exp->isConstantInitializer(Ctx, false);
   }
   case InitListExprClass: {
     // FIXME: This doesn't deal with fields with reference types correctly.
@@ -1398,7 +1523,7 @@ bool Expr::isConstantInitializer(ASTContext &Ctx) const {
     const InitListExpr *Exp = cast<InitListExpr>(this);
     unsigned numInits = Exp->getNumInits();
     for (unsigned i = 0; i < numInits; i++) {
-      if (!Exp->getInit(i)->isConstantInitializer(Ctx))
+      if (!Exp->getInit(i)->isConstantInitializer(Ctx, false))
         return false;
     }
     return true;
@@ -1406,36 +1531,41 @@ bool Expr::isConstantInitializer(ASTContext &Ctx) const {
   case ImplicitValueInitExprClass:
     return true;
   case ParenExprClass:
-    return cast<ParenExpr>(this)->getSubExpr()->isConstantInitializer(Ctx);
+    return cast<ParenExpr>(this)->getSubExpr()
+      ->isConstantInitializer(Ctx, IsForRef);
   case UnaryOperatorClass: {
     const UnaryOperator* Exp = cast<UnaryOperator>(this);
-    if (Exp->getOpcode() == UnaryOperator::Extension)
-      return Exp->getSubExpr()->isConstantInitializer(Ctx);
+    if (Exp->getOpcode() == UO_Extension)
+      return Exp->getSubExpr()->isConstantInitializer(Ctx, false);
     break;
   }
   case BinaryOperatorClass: {
     // Special case &&foo - &&bar.  It would be nice to generalize this somehow
     // but this handles the common case.
     const BinaryOperator *Exp = cast<BinaryOperator>(this);
-    if (Exp->getOpcode() == BinaryOperator::Sub &&
+    if (Exp->getOpcode() == BO_Sub &&
         isa<AddrLabelExpr>(Exp->getLHS()->IgnoreParenNoopCasts(Ctx)) &&
         isa<AddrLabelExpr>(Exp->getRHS()->IgnoreParenNoopCasts(Ctx)))
       return true;
     break;
   }
+  case CXXFunctionalCastExprClass:
+  case CXXStaticCastExprClass:
   case ImplicitCastExprClass:
   case CStyleCastExprClass:
     // Handle casts with a destination that's a struct or union; this
     // deals with both the gcc no-op struct cast extension and the
     // cast-to-union extension.
     if (getType()->isRecordType())
-      return cast<CastExpr>(this)->getSubExpr()->isConstantInitializer(Ctx);
+      return cast<CastExpr>(this)->getSubExpr()
+        ->isConstantInitializer(Ctx, false);
       
     // Integer->integer casts can be handled here, which is important for
     // things like (int)(&&x-&&y).  Scary but true.
     if (getType()->isIntegerType() &&
         cast<CastExpr>(this)->getSubExpr()->getType()->isIntegerType())
-      return cast<CastExpr>(this)->getSubExpr()->isConstantInitializer(Ctx);
+      return cast<CastExpr>(this)->getSubExpr()
+        ->isConstantInitializer(Ctx, false);
       
     break;
   }
@@ -1508,7 +1638,8 @@ FieldDecl *Expr::getBitField() {
   Expr *E = this->IgnoreParens();
 
   while (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-    if (ICE->isLvalueCast() && ICE->getCastKind() == CastExpr::CK_NoOp)
+    if (ICE->getValueKind() != VK_RValue &&
+        ICE->getCastKind() == CK_NoOp)
       E = ICE->getSubExpr()->IgnoreParens();
     else
       break;
@@ -1530,7 +1661,8 @@ bool Expr::refersToVectorElement() const {
   const Expr *E = this->IgnoreParens();
   
   while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-    if (ICE->isLvalueCast() && ICE->getCastKind() == CastExpr::CK_NoOp)
+    if (ICE->getValueKind() != VK_RValue &&
+        ICE->getCastKind() == CK_NoOp)
       E = ICE->getSubExpr()->IgnoreParens();
     else
       break;
@@ -1773,27 +1905,6 @@ void ShuffleVectorExpr::setExprs(ASTContext &C, Expr ** Exprs,
   memcpy(SubExprs, Exprs, sizeof(Expr *) * NumExprs);
 }
 
-void ShuffleVectorExpr::DoDestroy(ASTContext& C) {
-  DestroyChildren(C);
-  if (SubExprs) C.Deallocate(SubExprs);
-  this->~ShuffleVectorExpr();
-  C.Deallocate(this);
-}
-
-void SizeOfAlignOfExpr::DoDestroy(ASTContext& C) {
-  // Override default behavior of traversing children. If this has a type
-  // operand and the type is a variable-length array, the child iteration
-  // will iterate over the size expression. However, this expression belongs
-  // to the type, not to this, so we don't want to delete it.
-  // We still want to delete this expression.
-  if (isArgumentType()) {
-    this->~SizeOfAlignOfExpr();
-    C.Deallocate(this);
-  }
-  else
-    Expr::DoDestroy(C);
-}
-
 //===----------------------------------------------------------------------===//
 //  DesignatedInitExpr
 //===----------------------------------------------------------------------===//
@@ -1878,8 +1989,6 @@ DesignatedInitExpr *DesignatedInitExpr::CreateEmpty(ASTContext &C,
 void DesignatedInitExpr::setDesignators(ASTContext &C,
                                         const Designator *Desigs,
                                         unsigned NumDesigs) {
-  DestroyDesignators(C);
-
   Designators = new (C) Designator[NumDesigs];
   NumDesignators = NumDesigs;
   for (unsigned I = 0; I != NumDesigs; ++I)
@@ -1950,23 +2059,10 @@ void DesignatedInitExpr::ExpandDesignator(ASTContext &C, unsigned Idx,
   std::copy(First, Last, NewDesignators + Idx);
   std::copy(Designators + Idx + 1, Designators + NumDesignators,
             NewDesignators + Idx + NumNewDesignators);
-  DestroyDesignators(C);
   Designators = NewDesignators;
   NumDesignators = NumDesignators - 1 + NumNewDesignators;
 }
 
-void DesignatedInitExpr::DoDestroy(ASTContext &C) {
-  DestroyDesignators(C);
-  Expr::DoDestroy(C);
-}
-
-void DesignatedInitExpr::DestroyDesignators(ASTContext &C) {
-  for (unsigned I = 0; I != NumDesignators; ++I)
-    Designators[I].~Designator();
-  C.Deallocate(Designators);
-  Designators = 0;
-}
-
 ParenListExpr::ParenListExpr(ASTContext& C, SourceLocation lparenloc,
                              Expr **exprs, unsigned nexprs,
                              SourceLocation rparenloc)
@@ -1980,13 +2076,6 @@ ParenListExpr::ParenListExpr(ASTContext& C, SourceLocation lparenloc,
     Exprs[i] = exprs[i];
 }
 
-void ParenListExpr::DoDestroy(ASTContext& C) {
-  DestroyChildren(C);
-  if (Exprs) C.Deallocate(Exprs);
-  this->~ParenListExpr();
-  C.Deallocate(this);
-}
-
 //===----------------------------------------------------------------------===//
 //  ExprIterator.
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/ExprCXX.cpp b/lib/AST/ExprCXX.cpp
index c2548eca659e..0a101300d8fa 100644
--- a/lib/AST/ExprCXX.cpp
+++ b/lib/AST/ExprCXX.cpp
@@ -118,14 +118,6 @@ void CXXNewExpr::AllocateArgsArray(ASTContext &C, bool isArray,
 }
 
 
-void CXXNewExpr::DoDestroy(ASTContext &C) {
-  DestroyChildren(C);
-  if (SubExprs)
-    C.Deallocate(SubExprs);
-  this->~CXXNewExpr();
-  C.Deallocate((void*)this);
-}
-
 Stmt::child_iterator CXXNewExpr::child_begin() { return &SubExprs[0]; }
 Stmt::child_iterator CXXNewExpr::child_end() {
   return &SubExprs[0] + Array + getNumPlacementArgs() + getNumConstructorArgs();
@@ -167,8 +159,9 @@ UnresolvedLookupExpr *
 UnresolvedLookupExpr::Create(ASTContext &C, bool Dependent,
                              CXXRecordDecl *NamingClass,
                              NestedNameSpecifier *Qualifier,
-                             SourceRange QualifierRange, DeclarationName Name,
-                             SourceLocation NameLoc, bool ADL,
+                             SourceRange QualifierRange,
+                             const DeclarationNameInfo &NameInfo,
+                             bool ADL,
                              const TemplateArgumentListInfo &Args,
                              UnresolvedSetIterator Begin, 
                              UnresolvedSetIterator End) 
@@ -179,8 +172,8 @@ UnresolvedLookupExpr::Create(ASTContext &C, bool Dependent,
     = new (Mem) UnresolvedLookupExpr(C, 
                                      Dependent ? C.DependentTy : C.OverloadTy,
                                      Dependent, NamingClass,
-                                     Qualifier, QualifierRange,
-                                     Name, NameLoc, ADL,
+                                     Qualifier, QualifierRange, NameInfo,
+                                     ADL,
                                      /*Overload*/ true,
                                      /*ExplicitTemplateArgs*/ true,
                                      Begin, End);
@@ -204,14 +197,14 @@ UnresolvedLookupExpr::CreateEmpty(ASTContext &C, unsigned NumTemplateArgs) {
 
 OverloadExpr::OverloadExpr(StmtClass K, ASTContext &C, QualType T, 
                            bool Dependent, NestedNameSpecifier *Qualifier, 
-                           SourceRange QRange, DeclarationName Name, 
-                           SourceLocation NameLoc, bool HasTemplateArgs,
+                           SourceRange QRange,
+                           const DeclarationNameInfo &NameInfo,
+                           bool HasTemplateArgs,
                            UnresolvedSetIterator Begin, 
                            UnresolvedSetIterator End)
   : Expr(K, T, Dependent, Dependent),
-  Results(0), NumResults(0), Name(Name), Qualifier(Qualifier), 
-  QualifierRange(QRange), NameLoc(NameLoc), 
-  HasExplicitTemplateArgs(HasTemplateArgs)
+  Results(0), NumResults(0), NameInfo(NameInfo), Qualifier(Qualifier), 
+  QualifierRange(QRange), HasExplicitTemplateArgs(HasTemplateArgs)
 {
   initializeResults(C, Begin, End);
 }
@@ -270,8 +263,7 @@ DependentScopeDeclRefExpr *
 DependentScopeDeclRefExpr::Create(ASTContext &C,
                                   NestedNameSpecifier *Qualifier,
                                   SourceRange QualifierRange,
-                                  DeclarationName Name,
-                                  SourceLocation NameLoc,
+                                  const DeclarationNameInfo &NameInfo,
                                   const TemplateArgumentListInfo *Args) {
   std::size_t size = sizeof(DependentScopeDeclRefExpr);
   if (Args) size += ExplicitTemplateArgumentList::sizeFor(*Args);
@@ -280,8 +272,7 @@ DependentScopeDeclRefExpr::Create(ASTContext &C,
   DependentScopeDeclRefExpr *DRE
     = new (Mem) DependentScopeDeclRefExpr(C.DependentTy,
                                           Qualifier, QualifierRange,
-                                          Name, NameLoc,
-                                          Args != 0);
+                                          NameInfo, Args != 0);
 
   if (Args)
     reinterpret_cast<ExplicitTemplateArgumentList*>(DRE+1)
@@ -299,7 +290,7 @@ DependentScopeDeclRefExpr::CreateEmpty(ASTContext &C,
   void *Mem = C.Allocate(size);
 
   return new (Mem) DependentScopeDeclRefExpr(QualType(), 0, SourceRange(),
-                                             DeclarationName(),SourceLocation(),
+                                             DeclarationNameInfo(),
                                              NumTemplateArgs != 0);
 }
 
@@ -395,6 +386,118 @@ bool UnaryTypeTraitExpr::EvaluateTrait(ASTContext& C) const {
           C.getBaseElementType(QueriedType)->getAs<RecordType>())
       return cast<CXXRecordDecl>(RT->getDecl())->hasTrivialDestructor();
     return false;
+  // TODO: Propagate nothrowness for implicitly declared special members.
+  case UTT_HasNothrowAssign:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If type is const qualified or is a reference type then the
+    //   trait is false. Otherwise if __has_trivial_assign (type)
+    //   is true then the trait is true, else if type is a cv class
+    //   or union type with copy assignment operators that are known
+    //   not to throw an exception then the trait is true, else it is
+    //   false.
+    if (C.getBaseElementType(QueriedType).isConstQualified())
+      return false;
+    if (QueriedType->isReferenceType())
+      return false;
+    if (QueriedType->isPODType())
+      return true;
+    if (const RecordType *RT = QueriedType->getAs<RecordType>()) {
+      CXXRecordDecl* RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->hasTrivialCopyAssignment())
+        return true;
+
+      bool FoundAssign = false;
+      bool AllNoThrow = true;
+      DeclarationName Name = C.DeclarationNames.getCXXOperatorName(OO_Equal);
+      DeclContext::lookup_const_iterator Op, OpEnd;
+      for (llvm::tie(Op, OpEnd) = RD->lookup(Name);
+           Op != OpEnd; ++Op) {
+        CXXMethodDecl *Operator = cast<CXXMethodDecl>(*Op);
+        if (Operator->isCopyAssignmentOperator()) {
+          FoundAssign = true;
+          const FunctionProtoType *CPT
+              = Operator->getType()->getAs<FunctionProtoType>();
+          if (!CPT->hasEmptyExceptionSpec()) {
+            AllNoThrow = false;
+            break;
+          }
+        }
+      }
+
+      return FoundAssign && AllNoThrow;
+    }
+    return false;
+  case UTT_HasNothrowCopy:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __has_trivial_copy (type) is true then the trait is true, else
+    //   if type is a cv class or union type with copy constructors that are
+    //   known not to throw an exception then the trait is true, else it is
+    //   false.
+    if (QueriedType->isPODType() || QueriedType->isReferenceType())
+      return true;
+    if (const RecordType *RT = QueriedType->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->hasTrivialCopyConstructor())
+        return true;
+
+      bool FoundConstructor = false;
+      bool AllNoThrow = true;
+      unsigned FoundTQs;
+      DeclarationName ConstructorName
+          = C.DeclarationNames.getCXXConstructorName(
+                                          C.getCanonicalType(QueriedType));
+      DeclContext::lookup_const_iterator Con, ConEnd;
+      for (llvm::tie(Con, ConEnd) = RD->lookup(ConstructorName);
+           Con != ConEnd; ++Con) {
+        CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(*Con);
+        if (Constructor->isCopyConstructor(FoundTQs)) {
+          FoundConstructor = true;
+          const FunctionProtoType *CPT
+              = Constructor->getType()->getAs<FunctionProtoType>();
+          if (!CPT->hasEmptyExceptionSpec()) {
+            AllNoThrow = false;
+            break;
+          }
+        }
+      }
+
+      return FoundConstructor && AllNoThrow;
+    }
+    return false;
+  case UTT_HasNothrowConstructor:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If __has_trivial_constructor (type) is true then the trait is
+    //   true, else if type is a cv class or union type (or array
+    //   thereof) with a default constructor that is known not to
+    //   throw an exception then the trait is true, else it is false.
+    if (QueriedType->isPODType())
+      return true;
+    if (const RecordType *RT =
+          C.getBaseElementType(QueriedType)->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      if (RD->hasTrivialConstructor())
+        return true;
+
+      if (CXXConstructorDecl *Constructor = RD->getDefaultConstructor()) {
+        const FunctionProtoType *CPT
+            = Constructor->getType()->getAs<FunctionProtoType>();
+        // TODO: check whether evaluating default arguments can throw.
+        // For now, we'll be conservative and assume that they can throw.
+        if (CPT->hasEmptyExceptionSpec() && CPT->getNumArgs() == 0)
+          return true;
+      }
+    }
+    return false;
+  case UTT_HasVirtualDestructor:
+    // http://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html:
+    //   If type is a class type with a virtual destructor ([class.dtor])
+    //   then the trait is true, else it is false.
+    if (const RecordType *Record = QueriedType->getAs<RecordType>()) {
+      CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
+      if (CXXDestructorDecl *Destructor = RD->getDestructor())
+        return Destructor->isVirtual();
+    }
+    return false;
   }
 }
 
@@ -468,6 +571,100 @@ const char *CXXNamedCastExpr::getCastName() const {
   }
 }
 
+CXXStaticCastExpr *CXXStaticCastExpr::Create(ASTContext &C, QualType T,
+                                             CastKind K, Expr *Op,
+                                             const CXXCastPath *BasePath,
+                                             TypeSourceInfo *WrittenTy,
+                                             SourceLocation L) {
+  unsigned PathSize = (BasePath ? BasePath->size() : 0);
+  void *Buffer = C.Allocate(sizeof(CXXStaticCastExpr)
+                            + PathSize * sizeof(CXXBaseSpecifier*));
+  CXXStaticCastExpr *E =
+    new (Buffer) CXXStaticCastExpr(T, K, Op, PathSize, WrittenTy, L);
+  if (PathSize) E->setCastPath(*BasePath);
+  return E;
+}
+
+CXXStaticCastExpr *CXXStaticCastExpr::CreateEmpty(ASTContext &C,
+                                                  unsigned PathSize) {
+  void *Buffer =
+    C.Allocate(sizeof(CXXStaticCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  return new (Buffer) CXXStaticCastExpr(EmptyShell(), PathSize);
+}
+
+CXXDynamicCastExpr *CXXDynamicCastExpr::Create(ASTContext &C, QualType T,
+                                               CastKind K, Expr *Op,
+                                               const CXXCastPath *BasePath,
+                                               TypeSourceInfo *WrittenTy,
+                                               SourceLocation L) {
+  unsigned PathSize = (BasePath ? BasePath->size() : 0);
+  void *Buffer = C.Allocate(sizeof(CXXDynamicCastExpr)
+                            + PathSize * sizeof(CXXBaseSpecifier*));
+  CXXDynamicCastExpr *E =
+    new (Buffer) CXXDynamicCastExpr(T, K, Op, PathSize, WrittenTy, L);
+  if (PathSize) E->setCastPath(*BasePath);
+  return E;
+}
+
+CXXDynamicCastExpr *CXXDynamicCastExpr::CreateEmpty(ASTContext &C,
+                                                    unsigned PathSize) {
+  void *Buffer =
+    C.Allocate(sizeof(CXXDynamicCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  return new (Buffer) CXXDynamicCastExpr(EmptyShell(), PathSize);
+}
+
+CXXReinterpretCastExpr *
+CXXReinterpretCastExpr::Create(ASTContext &C, QualType T, CastKind K, Expr *Op,
+                               const CXXCastPath *BasePath,
+                               TypeSourceInfo *WrittenTy, SourceLocation L) {
+  unsigned PathSize = (BasePath ? BasePath->size() : 0);
+  void *Buffer =
+    C.Allocate(sizeof(CXXReinterpretCastExpr) + PathSize * sizeof(CXXBaseSpecifier*));
+  CXXReinterpretCastExpr *E =
+    new (Buffer) CXXReinterpretCastExpr(T, K, Op, PathSize, WrittenTy, L);
+  if (PathSize) E->setCastPath(*BasePath);
+  return E;
+}
+
+CXXReinterpretCastExpr *
+CXXReinterpretCastExpr::CreateEmpty(ASTContext &C, unsigned PathSize) {
+  void *Buffer = C.Allocate(sizeof(CXXReinterpretCastExpr)
+                            + PathSize * sizeof(CXXBaseSpecifier*));
+  return new (Buffer) CXXReinterpretCastExpr(EmptyShell(), PathSize);
+}
+
+CXXConstCastExpr *CXXConstCastExpr::Create(ASTContext &C, QualType T, Expr *Op,
+                                           TypeSourceInfo *WrittenTy,
+                                           SourceLocation L) {
+  return new (C) CXXConstCastExpr(T, Op, WrittenTy, L);
+}
+
+CXXConstCastExpr *CXXConstCastExpr::CreateEmpty(ASTContext &C) {
+  return new (C) CXXConstCastExpr(EmptyShell());
+}
+
+CXXFunctionalCastExpr *
+CXXFunctionalCastExpr::Create(ASTContext &C, QualType T,
+                              TypeSourceInfo *Written, SourceLocation L,
+                              CastKind K, Expr *Op, const CXXCastPath *BasePath,
+                               SourceLocation R) {
+  unsigned PathSize = (BasePath ? BasePath->size() : 0);
+  void *Buffer = C.Allocate(sizeof(CXXFunctionalCastExpr)
+                            + PathSize * sizeof(CXXBaseSpecifier*));
+  CXXFunctionalCastExpr *E =
+    new (Buffer) CXXFunctionalCastExpr(T, Written, L, K, Op, PathSize, R);
+  if (PathSize) E->setCastPath(*BasePath);
+  return E;
+}
+
+CXXFunctionalCastExpr *
+CXXFunctionalCastExpr::CreateEmpty(ASTContext &C, unsigned PathSize) {
+  void *Buffer = C.Allocate(sizeof(CXXFunctionalCastExpr)
+                            + PathSize * sizeof(CXXBaseSpecifier*));
+  return new (Buffer) CXXFunctionalCastExpr(EmptyShell(), PathSize);
+}
+
+
 CXXDefaultArgExpr *
 CXXDefaultArgExpr::Create(ASTContext &C, SourceLocation Loc, 
                           ParmVarDecl *Param, Expr *SubExpr) {
@@ -476,23 +673,11 @@ CXXDefaultArgExpr::Create(ASTContext &C, SourceLocation Loc,
                                      SubExpr);
 }
 
-void CXXDefaultArgExpr::DoDestroy(ASTContext &C) {
-  if (Param.getInt())
-    getExpr()->Destroy(C);
-  this->~CXXDefaultArgExpr();
-  C.Deallocate(this);
-}
-
 CXXTemporary *CXXTemporary::Create(ASTContext &C,
                                    const CXXDestructorDecl *Destructor) {
   return new (C) CXXTemporary(Destructor);
 }
 
-void CXXTemporary::Destroy(ASTContext &Ctx) {
-  this->~CXXTemporary();
-  Ctx.Deallocate(this);
-}
-
 CXXBindTemporaryExpr *CXXBindTemporaryExpr::Create(ASTContext &C,
                                                    CXXTemporary *Temp,
                                                    Expr* SubExpr) {
@@ -502,25 +687,6 @@ CXXBindTemporaryExpr *CXXBindTemporaryExpr::Create(ASTContext &C,
   return new (C) CXXBindTemporaryExpr(Temp, SubExpr);
 }
 
-void CXXBindTemporaryExpr::DoDestroy(ASTContext &C) {
-  Temp->Destroy(C);
-  this->~CXXBindTemporaryExpr();
-  C.Deallocate(this);
-}
-
-CXXBindReferenceExpr *CXXBindReferenceExpr::Create(ASTContext &C, Expr *SubExpr,
-                                                   bool ExtendsLifetime, 
-                                                   bool RequiresTemporaryCopy) {
-  return new (C) CXXBindReferenceExpr(SubExpr, 
-                                      ExtendsLifetime,
-                                      RequiresTemporaryCopy);
-}
-
-void CXXBindReferenceExpr::DoDestroy(ASTContext &C) {
-  this->~CXXBindReferenceExpr();
-  C.Deallocate(this);
-}
-
 CXXTemporaryObjectExpr::CXXTemporaryObjectExpr(ASTContext &C,
                                                CXXConstructorDecl *Cons,
                                                QualType writtenTy,
@@ -569,14 +735,6 @@ CXXConstructExpr::CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
   }
 }
 
-void CXXConstructExpr::DoDestroy(ASTContext &C) {
-  DestroyChildren(C);
-  if (Args)
-    C.Deallocate(Args);
-  this->~CXXConstructExpr();
-  C.Deallocate(this);
-}
-
 CXXExprWithTemporaries::CXXExprWithTemporaries(ASTContext &C,
                                                Expr *subexpr,
                                                CXXTemporary **temps,
@@ -605,16 +763,6 @@ CXXExprWithTemporaries *CXXExprWithTemporaries::Create(ASTContext &C,
   return new (C) CXXExprWithTemporaries(C, SubExpr, Temps, NumTemps);
 }
 
-void CXXExprWithTemporaries::DoDestroy(ASTContext &C) {
-  DestroyChildren(C);
-  if (Temps)
-    C.Deallocate(Temps);
-  this->~CXXExprWithTemporaries();
-  C.Deallocate(this);
-}
-
-CXXExprWithTemporaries::~CXXExprWithTemporaries() {}
-
 // CXXBindTemporaryExpr
 Stmt::child_iterator CXXBindTemporaryExpr::child_begin() {
   return &SubExpr;
@@ -624,15 +772,6 @@ Stmt::child_iterator CXXBindTemporaryExpr::child_end() {
   return &SubExpr + 1;
 }
 
-// CXXBindReferenceExpr
-Stmt::child_iterator CXXBindReferenceExpr::child_begin() {
-  return &SubExpr;
-}
-
-Stmt::child_iterator CXXBindReferenceExpr::child_end() {
-  return &SubExpr + 1;
-}
-
 // CXXConstructExpr
 Stmt::child_iterator CXXConstructExpr::child_begin() {
   return &Args[0];
@@ -705,8 +844,7 @@ CXXDependentScopeMemberExpr::CXXDependentScopeMemberExpr(ASTContext &C,
                                                  NestedNameSpecifier *Qualifier,
                                                  SourceRange QualifierRange,
                                           NamedDecl *FirstQualifierFoundInScope,
-                                                 DeclarationName Member,
-                                                 SourceLocation MemberLoc,
+                                          DeclarationNameInfo MemberNameInfo,
                                    const TemplateArgumentListInfo *TemplateArgs)
   : Expr(CXXDependentScopeMemberExprClass, C.DependentTy, true, true),
     Base(Base), BaseType(BaseType), IsArrow(IsArrow),
@@ -714,9 +852,9 @@ CXXDependentScopeMemberExpr::CXXDependentScopeMemberExpr(ASTContext &C,
     OperatorLoc(OperatorLoc),
     Qualifier(Qualifier), QualifierRange(QualifierRange),
     FirstQualifierFoundInScope(FirstQualifierFoundInScope),
-    Member(Member), MemberLoc(MemberLoc) {
+    MemberNameInfo(MemberNameInfo) {
   if (TemplateArgs)
-    getExplicitTemplateArgumentList()->initializeFrom(*TemplateArgs);
+    getExplicitTemplateArgs().initializeFrom(*TemplateArgs);
 }
 
 CXXDependentScopeMemberExpr *
@@ -726,15 +864,14 @@ CXXDependentScopeMemberExpr::Create(ASTContext &C,
                                 NestedNameSpecifier *Qualifier,
                                 SourceRange QualifierRange,
                                 NamedDecl *FirstQualifierFoundInScope,
-                                DeclarationName Member,
-                                SourceLocation MemberLoc,
+                                DeclarationNameInfo MemberNameInfo,
                                 const TemplateArgumentListInfo *TemplateArgs) {
   if (!TemplateArgs)
     return new (C) CXXDependentScopeMemberExpr(C, Base, BaseType,
                                                IsArrow, OperatorLoc,
                                                Qualifier, QualifierRange,
                                                FirstQualifierFoundInScope,
-                                               Member, MemberLoc);
+                                               MemberNameInfo);
 
   std::size_t size = sizeof(CXXDependentScopeMemberExpr);
   if (TemplateArgs)
@@ -745,7 +882,7 @@ CXXDependentScopeMemberExpr::Create(ASTContext &C,
                                                IsArrow, OperatorLoc,
                                                Qualifier, QualifierRange,
                                                FirstQualifierFoundInScope,
-                                               Member, MemberLoc, TemplateArgs);
+                                               MemberNameInfo, TemplateArgs);
 }
 
 CXXDependentScopeMemberExpr *
@@ -755,8 +892,7 @@ CXXDependentScopeMemberExpr::CreateEmpty(ASTContext &C,
     return new (C) CXXDependentScopeMemberExpr(C, 0, QualType(),
                                                0, SourceLocation(), 0,
                                                SourceRange(), 0,
-                                               DeclarationName(),
-                                               SourceLocation());
+                                               DeclarationNameInfo());
 
   std::size_t size = sizeof(CXXDependentScopeMemberExpr) +
                      ExplicitTemplateArgumentList::sizeFor(NumTemplateArgs);
@@ -765,8 +901,7 @@ CXXDependentScopeMemberExpr::CreateEmpty(ASTContext &C,
     =  new (Mem) CXXDependentScopeMemberExpr(C, 0, QualType(),
                                              0, SourceLocation(), 0,
                                              SourceRange(), 0,
-                                             DeclarationName(),
-                                             SourceLocation(), 0);
+                                             DeclarationNameInfo(), 0);
   E->HasExplicitTemplateArgs = true;
   return E;
 }
@@ -789,13 +924,12 @@ UnresolvedMemberExpr::UnresolvedMemberExpr(ASTContext &C, QualType T,
                                            SourceLocation OperatorLoc,
                                            NestedNameSpecifier *Qualifier,
                                            SourceRange QualifierRange,
-                                           DeclarationName MemberName,
-                                           SourceLocation MemberLoc,
+                                   const DeclarationNameInfo &MemberNameInfo,
                                    const TemplateArgumentListInfo *TemplateArgs,
                                            UnresolvedSetIterator Begin, 
                                            UnresolvedSetIterator End)
   : OverloadExpr(UnresolvedMemberExprClass, C, T, Dependent,
-                 Qualifier, QualifierRange, MemberName, MemberLoc,
+                 Qualifier, QualifierRange, MemberNameInfo,
                  TemplateArgs != 0, Begin, End),
     IsArrow(IsArrow), HasUnresolvedUsing(HasUnresolvedUsing),
     Base(Base), BaseType(BaseType), OperatorLoc(OperatorLoc) {
@@ -810,8 +944,7 @@ UnresolvedMemberExpr::Create(ASTContext &C, bool Dependent,
                              SourceLocation OperatorLoc,
                              NestedNameSpecifier *Qualifier,
                              SourceRange QualifierRange,
-                             DeclarationName Member,
-                             SourceLocation MemberLoc,
+                             const DeclarationNameInfo &MemberNameInfo,
                              const TemplateArgumentListInfo *TemplateArgs,
                              UnresolvedSetIterator Begin, 
                              UnresolvedSetIterator End) {
@@ -824,7 +957,7 @@ UnresolvedMemberExpr::Create(ASTContext &C, bool Dependent,
                              Dependent ? C.DependentTy : C.OverloadTy,
                              Dependent, HasUnresolvedUsing, Base, BaseType,
                              IsArrow, OperatorLoc, Qualifier, QualifierRange,
-                             Member, MemberLoc, TemplateArgs, Begin, End);
+                             MemberNameInfo, TemplateArgs, Begin, End);
 }
 
 UnresolvedMemberExpr *
diff --git a/lib/AST/ExprClassification.cpp b/lib/AST/ExprClassification.cpp
index 60ac347c50fb..d7e38ebbf5c9 100644
--- a/lib/AST/ExprClassification.cpp
+++ b/lib/AST/ExprClassification.cpp
@@ -111,20 +111,20 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
       // C++ [expr.unary.op]p1: The unary * operator performs indirection:
       //   [...] the result is an lvalue referring to the object or function
       //   to which the expression points.
-    case UnaryOperator::Deref:
+    case UO_Deref:
       return Cl::CL_LValue;
 
       // GNU extensions, simply look through them.
-    case UnaryOperator::Real:
-    case UnaryOperator::Imag:
-    case UnaryOperator::Extension:
+    case UO_Real:
+    case UO_Imag:
+    case UO_Extension:
       return ClassifyInternal(Ctx, cast<UnaryOperator>(E)->getSubExpr());
 
       // C++ [expr.pre.incr]p1: The result is the updated operand; it is an
       //   lvalue, [...]
       // Not so in C.
-    case UnaryOperator::PreInc:
-    case UnaryOperator::PreDec:
+    case UO_PreInc:
+    case UO_PreDec:
       return Lang.CPlusPlus ? Cl::CL_LValue : Cl::CL_PRValue;
 
     default:
@@ -134,10 +134,16 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
     // Implicit casts are lvalues if they're lvalue casts. Other than that, we
     // only specifically record class temporaries.
   case Expr::ImplicitCastExprClass:
-    if (cast<ImplicitCastExpr>(E)->isLvalueCast())
+    switch (cast<ImplicitCastExpr>(E)->getValueKind()) {
+    case VK_RValue:
+      return Lang.CPlusPlus && E->getType()->isRecordType() ?
+        Cl::CL_ClassTemporary : Cl::CL_PRValue;
+    case VK_LValue:
       return Cl::CL_LValue;
-    return Lang.CPlusPlus && E->getType()->isRecordType() ?
-      Cl::CL_ClassTemporary : Cl::CL_PRValue;
+    case VK_XValue:
+      return Cl::CL_XValue;
+    }
+    llvm_unreachable("Invalid value category of implicit cast.");
 
     // C++ [expr.prim.general]p4: The presence of parentheses does not affect
     //   whether the expression is an lvalue.
@@ -223,6 +229,10 @@ static Cl::Kinds ClassifyDecl(ASTContext &Ctx, const Decl *D) {
   // In addition, NonTypeTemplateParmDecl derives from VarDecl but isn't an
   // lvalue unless it's a reference type (C++ [temp.param]p6), so we need to
   // special-case this.
+
+  if (isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance())
+    return Cl::CL_MemberFunction;
+
   bool islvalue;
   if (const NonTypeTemplateParmDecl *NTTParm =
         dyn_cast<NonTypeTemplateParmDecl>(D))
@@ -315,19 +325,19 @@ static Cl::Kinds ClassifyBinaryOp(ASTContext &Ctx, const BinaryOperator *E) {
 
   // C++ [expr.comma]p1: the result is of the same value category as its right
   //   operand, [...].
-  if (E->getOpcode() == BinaryOperator::Comma)
+  if (E->getOpcode() == BO_Comma)
     return ClassifyInternal(Ctx, E->getRHS());
 
   // C++ [expr.mptr.oper]p6: The result of a .* expression whose second operand
   //   is a pointer to a data member is of the same value category as its first
   //   operand.
-  if (E->getOpcode() == BinaryOperator::PtrMemD)
+  if (E->getOpcode() == BO_PtrMemD)
     return E->getType()->isFunctionType() ? Cl::CL_MemberFunction :
       ClassifyInternal(Ctx, E->getLHS());
 
   // C++ [expr.mptr.oper]p6: The result of an ->* expression is an lvalue if its
   //   second operand is a pointer to data member and a prvalue otherwise.
-  if (E->getOpcode() == BinaryOperator::PtrMemI)
+  if (E->getOpcode() == BO_PtrMemI)
     return E->getType()->isFunctionType() ?
       Cl::CL_MemberFunction : Cl::CL_LValue;
 
diff --git a/lib/AST/ExprConstant.cpp b/lib/AST/ExprConstant.cpp
index 3c9742033265..7347f5a43e17 100644
--- a/lib/AST/ExprConstant.cpp
+++ b/lib/AST/ExprConstant.cpp
@@ -337,7 +337,7 @@ public:
     default:
       return false;
 
-    case CastExpr::CK_NoOp:
+    case CK_NoOp:
       return Visit(E->getSubExpr());
     }
   }
@@ -481,8 +481,8 @@ static bool EvaluatePointer(const Expr* E, LValue& Result, EvalInfo &Info) {
 }
 
 bool PointerExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
-  if (E->getOpcode() != BinaryOperator::Add &&
-      E->getOpcode() != BinaryOperator::Sub)
+  if (E->getOpcode() != BO_Add &&
+      E->getOpcode() != BO_Sub)
     return false;
 
   const Expr *PExp = E->getLHS();
@@ -512,7 +512,7 @@ bool PointerExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   else
     SizeOfPointee = Info.Ctx.getTypeSizeInChars(PointeeType);
 
-  if (E->getOpcode() == BinaryOperator::Add)
+  if (E->getOpcode() == BO_Add)
     Result.Offset += AdditionalOffset * SizeOfPointee;
   else
     Result.Offset -= AdditionalOffset * SizeOfPointee;
@@ -532,7 +532,7 @@ bool PointerExprEvaluator::VisitCastExpr(CastExpr* E) {
   default:
     break;
 
-  case CastExpr::CK_Unknown: {
+  case CK_Unknown: {
     // FIXME: The handling for CK_Unknown is ugly/shouldn't be necessary!
 
     // Check for pointer->pointer cast
@@ -561,14 +561,14 @@ bool PointerExprEvaluator::VisitCastExpr(CastExpr* E) {
     break;
   }
 
-  case CastExpr::CK_NoOp:
-  case CastExpr::CK_BitCast:
-  case CastExpr::CK_LValueBitCast:
-  case CastExpr::CK_AnyPointerToObjCPointerCast:
-  case CastExpr::CK_AnyPointerToBlockPointerCast:
+  case CK_NoOp:
+  case CK_BitCast:
+  case CK_LValueBitCast:
+  case CK_AnyPointerToObjCPointerCast:
+  case CK_AnyPointerToBlockPointerCast:
     return Visit(SubExpr);
 
-  case CastExpr::CK_IntegralToPointer: {
+  case CK_IntegralToPointer: {
     APValue Value;
     if (!EvaluateIntegerOrLValue(SubExpr, Value, Info))
       break;
@@ -585,8 +585,8 @@ bool PointerExprEvaluator::VisitCastExpr(CastExpr* E) {
       return true;
     }
   }
-  case CastExpr::CK_ArrayToPointerDecay:
-  case CastExpr::CK_FunctionToPointerDecay:
+  case CK_ArrayToPointerDecay:
+  case CK_FunctionToPointerDecay:
     return EvaluateLValue(SubExpr, Result, Info);
   }
 
@@ -1008,8 +1008,11 @@ bool IntExprEvaluator::CheckReferencedDecl(const Expr* E, const Decl* D) {
 
         VD->setEvaluatingValue();
 
-        if (Visit(const_cast<Expr*>(Init))) {
+        Expr::EvalResult EResult;
+        if (Init->Evaluate(EResult, Info.Ctx) && !EResult.HasSideEffects &&
+            EResult.Val.isInt()) {
           // Cache the evaluated value in the variable declaration.
+          Result = EResult.Val;
           VD->setEvaluatedValue(Result);
           return true;
         }
@@ -1106,7 +1109,7 @@ bool IntExprEvaluator::TryEvaluateBuiltinObjectSize(CallExpr *E) {
   QualType T = GetObjectType(LVBase);
   if (T.isNull() ||
       T->isIncompleteType() ||
-      !T->isObjectType() ||
+      T->isFunctionType() ||
       T->isVariablyModifiedType() ||
       T->isDependentType())
     return false;
@@ -1161,7 +1164,7 @@ bool IntExprEvaluator::VisitCallExpr(CallExpr *E) {
 }
 
 bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
-  if (E->getOpcode() == BinaryOperator::Comma) {
+  if (E->getOpcode() == BO_Comma) {
     if (!Visit(E->getRHS()))
       return false;
 
@@ -1181,11 +1184,11 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
     if (HandleConversionToBool(E->getLHS(), lhsResult, Info)) {
       // We were able to evaluate the LHS, see if we can get away with not
       // evaluating the RHS: 0 && X -> 0, 1 || X -> 1
-      if (lhsResult == (E->getOpcode() == BinaryOperator::LOr))
+      if (lhsResult == (E->getOpcode() == BO_LOr))
         return Success(lhsResult, E);
 
       if (HandleConversionToBool(E->getRHS(), rhsResult, Info)) {
-        if (E->getOpcode() == BinaryOperator::LOr)
+        if (E->getOpcode() == BO_LOr)
           return Success(lhsResult || rhsResult, E);
         else
           return Success(lhsResult && rhsResult, E);
@@ -1194,8 +1197,8 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       if (HandleConversionToBool(E->getRHS(), rhsResult, Info)) {
         // We can't evaluate the LHS; however, sometimes the result
         // is determined by the RHS: X && 0 -> 0, X || 1 -> 1.
-        if (rhsResult == (E->getOpcode() == BinaryOperator::LOr) ||
-            !rhsResult == (E->getOpcode() == BinaryOperator::LAnd)) {
+        if (rhsResult == (E->getOpcode() == BO_LOr) ||
+            !rhsResult == (E->getOpcode() == BO_LAnd)) {
           // Since we weren't able to evaluate the left hand side, it
           // must have had side effects.
           Info.EvalResult.HasSideEffects = true;
@@ -1227,11 +1230,11 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       APFloat::cmpResult CR_i =
         LHS.getComplexFloatImag().compare(RHS.getComplexFloatImag());
 
-      if (E->getOpcode() == BinaryOperator::EQ)
+      if (E->getOpcode() == BO_EQ)
         return Success((CR_r == APFloat::cmpEqual &&
                         CR_i == APFloat::cmpEqual), E);
       else {
-        assert(E->getOpcode() == BinaryOperator::NE &&
+        assert(E->getOpcode() == BO_NE &&
                "Invalid complex comparison.");
         return Success(((CR_r == APFloat::cmpGreaterThan ||
                          CR_r == APFloat::cmpLessThan ||
@@ -1241,11 +1244,11 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
                          CR_i == APFloat::cmpUnordered)), E);
       }
     } else {
-      if (E->getOpcode() == BinaryOperator::EQ)
+      if (E->getOpcode() == BO_EQ)
         return Success((LHS.getComplexIntReal() == RHS.getComplexIntReal() &&
                         LHS.getComplexIntImag() == RHS.getComplexIntImag()), E);
       else {
-        assert(E->getOpcode() == BinaryOperator::NE &&
+        assert(E->getOpcode() == BO_NE &&
                "Invalid compex comparison.");
         return Success((LHS.getComplexIntReal() != RHS.getComplexIntReal() ||
                         LHS.getComplexIntImag() != RHS.getComplexIntImag()), E);
@@ -1268,18 +1271,18 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
     switch (E->getOpcode()) {
     default:
       assert(0 && "Invalid binary operator!");
-    case BinaryOperator::LT:
+    case BO_LT:
       return Success(CR == APFloat::cmpLessThan, E);
-    case BinaryOperator::GT:
+    case BO_GT:
       return Success(CR == APFloat::cmpGreaterThan, E);
-    case BinaryOperator::LE:
+    case BO_LE:
       return Success(CR == APFloat::cmpLessThan || CR == APFloat::cmpEqual, E);
-    case BinaryOperator::GE:
+    case BO_GE:
       return Success(CR == APFloat::cmpGreaterThan || CR == APFloat::cmpEqual,
                      E);
-    case BinaryOperator::EQ:
+    case BO_EQ:
       return Success(CR == APFloat::cmpEqual, E);
-    case BinaryOperator::NE:
+    case BO_NE:
       return Success(CR == APFloat::cmpGreaterThan
                      || CR == APFloat::cmpLessThan
                      || CR == APFloat::cmpUnordered, E);
@@ -1287,7 +1290,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   }
 
   if (LHSTy->isPointerType() && RHSTy->isPointerType()) {
-    if (E->getOpcode() == BinaryOperator::Sub || E->isEqualityOp()) {
+    if (E->getOpcode() == BO_Sub || E->isEqualityOp()) {
       LValue LHSValue;
       if (!EvaluatePointer(E->getLHS(), LHSValue, Info))
         return false;
@@ -1306,7 +1309,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
         bool bres;
         if (!EvalPointerValueAsBool(LHSValue, bres))
           return false;
-        return Success(bres ^ (E->getOpcode() == BinaryOperator::EQ), E);
+        return Success(bres ^ (E->getOpcode() == BO_EQ), E);
       } else if (RHSValue.getLValueBase()) {
         if (!E->isEqualityOp())
           return false;
@@ -1315,10 +1318,10 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
         bool bres;
         if (!EvalPointerValueAsBool(RHSValue, bres))
           return false;
-        return Success(bres ^ (E->getOpcode() == BinaryOperator::EQ), E);
+        return Success(bres ^ (E->getOpcode() == BO_EQ), E);
       }
 
-      if (E->getOpcode() == BinaryOperator::Sub) {
+      if (E->getOpcode() == BO_Sub) {
         QualType Type = E->getLHS()->getType();
         QualType ElementType = Type->getAs<PointerType>()->getPointeeType();
 
@@ -1331,7 +1334,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
         return Success(Diff / ElementSize, E);
       }
       bool Result;
-      if (E->getOpcode() == BinaryOperator::EQ) {
+      if (E->getOpcode() == BO_EQ) {
         Result = LHSValue.getLValueOffset() == RHSValue.getLValueOffset();
       } else {
         Result = LHSValue.getLValueOffset() != RHSValue.getLValueOffset();
@@ -1359,7 +1362,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
     CharUnits Offset = Result.getLValueOffset();
     CharUnits AdditionalOffset = CharUnits::fromQuantity(
                                      RHSVal.getInt().getZExtValue());
-    if (E->getOpcode() == BinaryOperator::Add)
+    if (E->getOpcode() == BO_Add)
       Offset += AdditionalOffset;
     else
       Offset -= AdditionalOffset;
@@ -1368,7 +1371,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   }
 
   // Handle cases like 4 + (unsigned long)&a
-  if (E->getOpcode() == BinaryOperator::Add &&
+  if (E->getOpcode() == BO_Add &&
         RHSVal.isLValue() && Result.isInt()) {
     CharUnits Offset = RHSVal.getLValueOffset();
     Offset += CharUnits::fromQuantity(Result.getInt().getZExtValue());
@@ -1385,38 +1388,38 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   switch (E->getOpcode()) {
   default:
     return Error(E->getOperatorLoc(), diag::note_invalid_subexpr_in_ice, E);
-  case BinaryOperator::Mul: return Success(Result.getInt() * RHS, E);
-  case BinaryOperator::Add: return Success(Result.getInt() + RHS, E);
-  case BinaryOperator::Sub: return Success(Result.getInt() - RHS, E);
-  case BinaryOperator::And: return Success(Result.getInt() & RHS, E);
-  case BinaryOperator::Xor: return Success(Result.getInt() ^ RHS, E);
-  case BinaryOperator::Or:  return Success(Result.getInt() | RHS, E);
-  case BinaryOperator::Div:
+  case BO_Mul: return Success(Result.getInt() * RHS, E);
+  case BO_Add: return Success(Result.getInt() + RHS, E);
+  case BO_Sub: return Success(Result.getInt() - RHS, E);
+  case BO_And: return Success(Result.getInt() & RHS, E);
+  case BO_Xor: return Success(Result.getInt() ^ RHS, E);
+  case BO_Or:  return Success(Result.getInt() | RHS, E);
+  case BO_Div:
     if (RHS == 0)
       return Error(E->getOperatorLoc(), diag::note_expr_divide_by_zero, E);
     return Success(Result.getInt() / RHS, E);
-  case BinaryOperator::Rem:
+  case BO_Rem:
     if (RHS == 0)
       return Error(E->getOperatorLoc(), diag::note_expr_divide_by_zero, E);
     return Success(Result.getInt() % RHS, E);
-  case BinaryOperator::Shl: {
+  case BO_Shl: {
     // FIXME: Warn about out of range shift amounts!
     unsigned SA =
       (unsigned) RHS.getLimitedValue(Result.getInt().getBitWidth()-1);
     return Success(Result.getInt() << SA, E);
   }
-  case BinaryOperator::Shr: {
+  case BO_Shr: {
     unsigned SA =
       (unsigned) RHS.getLimitedValue(Result.getInt().getBitWidth()-1);
     return Success(Result.getInt() >> SA, E);
   }
 
-  case BinaryOperator::LT: return Success(Result.getInt() < RHS, E);
-  case BinaryOperator::GT: return Success(Result.getInt() > RHS, E);
-  case BinaryOperator::LE: return Success(Result.getInt() <= RHS, E);
-  case BinaryOperator::GE: return Success(Result.getInt() >= RHS, E);
-  case BinaryOperator::EQ: return Success(Result.getInt() == RHS, E);
-  case BinaryOperator::NE: return Success(Result.getInt() != RHS, E);
+  case BO_LT: return Success(Result.getInt() < RHS, E);
+  case BO_GT: return Success(Result.getInt() > RHS, E);
+  case BO_LE: return Success(Result.getInt() <= RHS, E);
+  case BO_GE: return Success(Result.getInt() >= RHS, E);
+  case BO_EQ: return Success(Result.getInt() == RHS, E);
+  case BO_NE: return Success(Result.getInt() != RHS, E);
   }
 }
 
@@ -1573,20 +1576,7 @@ bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *E) {
 }
 
 bool IntExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) {
-  // Special case unary operators that do not need their subexpression
-  // evaluated.  offsetof/sizeof/alignof are all special.
-  if (E->isOffsetOfOp()) {
-    // The AST for offsetof is defined in such a way that we can just
-    // directly Evaluate it as an l-value.
-    LValue LV;
-    if (!EvaluateLValue(E->getSubExpr(), LV, Info))
-        return false;
-    if (LV.getLValueBase())
-        return false;
-    return Success(LV.getLValueOffset().getQuantity(), E);
-  }
-    
-  if (E->getOpcode() == UnaryOperator::LNot) {
+  if (E->getOpcode() == UO_LNot) {
     // LNot's operand isn't necessarily an integer, so we handle it specially.
     bool bres;
     if (!HandleConversionToBool(E->getSubExpr(), bres, Info))
@@ -1607,17 +1597,17 @@ bool IntExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) {
     // Address, indirect, pre/post inc/dec, etc are not valid constant exprs.
     // See C99 6.6p3.
     return Error(E->getOperatorLoc(), diag::note_invalid_subexpr_in_ice, E);
-  case UnaryOperator::Extension:
+  case UO_Extension:
     // FIXME: Should extension allow i-c-e extension expressions in its scope?
     // If so, we could clear the diagnostic ID.
     return true;
-  case UnaryOperator::Plus:
+  case UO_Plus:
     // The result is always just the subexpr.
     return true;
-  case UnaryOperator::Minus:
+  case UO_Minus:
     if (!Result.isInt()) return false;
     return Success(-Result.getInt(), E);
-  case UnaryOperator::Not:
+  case UO_Not:
     if (!Result.isInt()) return false;
     return Success(~Result.getInt(), E);
   }
@@ -1855,23 +1845,35 @@ bool FloatExprEvaluator::VisitCallExpr(const CallExpr *E) {
 }
 
 bool FloatExprEvaluator::VisitUnaryReal(const UnaryOperator *E) {
-  ComplexValue CV;
-  if (!EvaluateComplex(E->getSubExpr(), CV, Info))
-    return false;
-  Result = CV.FloatReal;
-  return true;
+  if (E->getSubExpr()->getType()->isAnyComplexType()) {
+    ComplexValue CV;
+    if (!EvaluateComplex(E->getSubExpr(), CV, Info))
+      return false;
+    Result = CV.FloatReal;
+    return true;
+  }
+
+  return Visit(E->getSubExpr());
 }
 
 bool FloatExprEvaluator::VisitUnaryImag(const UnaryOperator *E) {
-  ComplexValue CV;
-  if (!EvaluateComplex(E->getSubExpr(), CV, Info))
-    return false;
-  Result = CV.FloatImag;
+  if (E->getSubExpr()->getType()->isAnyComplexType()) {
+    ComplexValue CV;
+    if (!EvaluateComplex(E->getSubExpr(), CV, Info))
+      return false;
+    Result = CV.FloatImag;
+    return true;
+  }
+
+  if (!E->getSubExpr()->isEvaluatable(Info.Ctx))
+    Info.EvalResult.HasSideEffects = true;
+  const llvm::fltSemantics &Sem = Info.Ctx.getFloatTypeSemantics(E->getType());
+  Result = llvm::APFloat::getZero(Sem);
   return true;
 }
 
 bool FloatExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) {
-  if (E->getOpcode() == UnaryOperator::Deref)
+  if (E->getOpcode() == UO_Deref)
     return false;
 
   if (!EvaluateFloat(E->getSubExpr(), Result, Info))
@@ -1879,16 +1881,16 @@ bool FloatExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) {
 
   switch (E->getOpcode()) {
   default: return false;
-  case UnaryOperator::Plus:
+  case UO_Plus:
     return true;
-  case UnaryOperator::Minus:
+  case UO_Minus:
     Result.changeSign();
     return true;
   }
 }
 
 bool FloatExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
-  if (E->getOpcode() == BinaryOperator::Comma) {
+  if (E->getOpcode() == BO_Comma) {
     if (!EvaluateFloat(E->getRHS(), Result, Info))
       return false;
 
@@ -1910,16 +1912,16 @@ bool FloatExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
 
   switch (E->getOpcode()) {
   default: return false;
-  case BinaryOperator::Mul:
+  case BO_Mul:
     Result.multiply(RHS, APFloat::rmNearestTiesToEven);
     return true;
-  case BinaryOperator::Add:
+  case BO_Add:
     Result.add(RHS, APFloat::rmNearestTiesToEven);
     return true;
-  case BinaryOperator::Sub:
+  case BO_Sub:
     Result.subtract(RHS, APFloat::rmNearestTiesToEven);
     return true;
-  case BinaryOperator::Div:
+  case BO_Div:
     Result.divide(RHS, APFloat::rmNearestTiesToEven);
     return true;
   }
@@ -1990,138 +1992,142 @@ public:
 
   bool VisitParenExpr(ParenExpr *E) { return Visit(E->getSubExpr()); }
 
-  bool VisitImaginaryLiteral(ImaginaryLiteral *E) {
-    Expr* SubExpr = E->getSubExpr();
+  bool VisitImaginaryLiteral(ImaginaryLiteral *E);
 
-    if (SubExpr->getType()->isRealFloatingType()) {
-      Result.makeComplexFloat();
-      APFloat &Imag = Result.FloatImag;
-      if (!EvaluateFloat(SubExpr, Imag, Info))
-        return false;
+  bool VisitCastExpr(CastExpr *E);
+
+  bool VisitBinaryOperator(const BinaryOperator *E);
+  bool VisitChooseExpr(const ChooseExpr *E)
+    { return Visit(E->getChosenSubExpr(Info.Ctx)); }
+  bool VisitUnaryExtension(const UnaryOperator *E)
+    { return Visit(E->getSubExpr()); }
+  // FIXME Missing: unary +/-/~, binary div, ImplicitValueInitExpr,
+  //                conditional ?:, comma
+};
+} // end anonymous namespace
+
+static bool EvaluateComplex(const Expr *E, ComplexValue &Result,
+                            EvalInfo &Info) {
+  assert(E->getType()->isAnyComplexType());
+  return ComplexExprEvaluator(Info, Result).Visit(const_cast<Expr*>(E));
+}
+
+bool ComplexExprEvaluator::VisitImaginaryLiteral(ImaginaryLiteral *E) {
+  Expr* SubExpr = E->getSubExpr();
+
+  if (SubExpr->getType()->isRealFloatingType()) {
+    Result.makeComplexFloat();
+    APFloat &Imag = Result.FloatImag;
+    if (!EvaluateFloat(SubExpr, Imag, Info))
+      return false;
 
-      Result.FloatReal = APFloat(Imag.getSemantics());
+    Result.FloatReal = APFloat(Imag.getSemantics());
+    return true;
+  } else {
+    assert(SubExpr->getType()->isIntegerType() &&
+           "Unexpected imaginary literal.");
+
+    Result.makeComplexInt();
+    APSInt &Imag = Result.IntImag;
+    if (!EvaluateInteger(SubExpr, Imag, Info))
+      return false;
+
+    Result.IntReal = APSInt(Imag.getBitWidth(), !Imag.isSigned());
+    return true;
+  }
+}
+
+bool ComplexExprEvaluator::VisitCastExpr(CastExpr *E) {
+  Expr* SubExpr = E->getSubExpr();
+  QualType EltType = E->getType()->getAs<ComplexType>()->getElementType();
+  QualType SubType = SubExpr->getType();
+
+  if (SubType->isRealFloatingType()) {
+    APFloat &Real = Result.FloatReal;
+    if (!EvaluateFloat(SubExpr, Real, Info))
+      return false;
+
+    if (EltType->isRealFloatingType()) {
+      Result.makeComplexFloat();
+      Real = HandleFloatToFloatCast(EltType, SubType, Real, Info.Ctx);
+      Result.FloatImag = APFloat(Real.getSemantics());
       return true;
     } else {
-      assert(SubExpr->getType()->isIntegerType() &&
-             "Unexpected imaginary literal.");
-
       Result.makeComplexInt();
-      APSInt &Imag = Result.IntImag;
-      if (!EvaluateInteger(SubExpr, Imag, Info))
-        return false;
-
-      Result.IntReal = APSInt(Imag.getBitWidth(), !Imag.isSigned());
+      Result.IntReal = HandleFloatToIntCast(EltType, SubType, Real, Info.Ctx);
+      Result.IntImag = APSInt(Result.IntReal.getBitWidth(),
+                              !Result.IntReal.isSigned());
       return true;
     }
-  }
+  } else if (SubType->isIntegerType()) {
+    APSInt &Real = Result.IntReal;
+    if (!EvaluateInteger(SubExpr, Real, Info))
+      return false;
 
-  bool VisitCastExpr(CastExpr *E) {
-    Expr* SubExpr = E->getSubExpr();
-    QualType EltType = E->getType()->getAs<ComplexType>()->getElementType();
-    QualType SubType = SubExpr->getType();
+    if (EltType->isRealFloatingType()) {
+      Result.makeComplexFloat();
+      Result.FloatReal
+        = HandleIntToFloatCast(EltType, SubType, Real, Info.Ctx);
+      Result.FloatImag = APFloat(Result.FloatReal.getSemantics());
+      return true;
+    } else {
+      Result.makeComplexInt();
+      Real = HandleIntToIntCast(EltType, SubType, Real, Info.Ctx);
+      Result.IntImag = APSInt(Real.getBitWidth(), !Real.isSigned());
+      return true;
+    }
+  } else if (const ComplexType *CT = SubType->getAs<ComplexType>()) {
+    if (!Visit(SubExpr))
+      return false;
 
-    if (SubType->isRealFloatingType()) {
-      APFloat &Real = Result.FloatReal;
-      if (!EvaluateFloat(SubExpr, Real, Info))
-        return false;
+    QualType SrcType = CT->getElementType();
 
+    if (Result.isComplexFloat()) {
       if (EltType->isRealFloatingType()) {
         Result.makeComplexFloat();
-        Real = HandleFloatToFloatCast(EltType, SubType, Real, Info.Ctx);
-        Result.FloatImag = APFloat(Real.getSemantics());
+        Result.FloatReal = HandleFloatToFloatCast(EltType, SrcType,
+                                                  Result.FloatReal,
+                                                  Info.Ctx);
+        Result.FloatImag = HandleFloatToFloatCast(EltType, SrcType,
+                                                  Result.FloatImag,
+                                                  Info.Ctx);
         return true;
       } else {
         Result.makeComplexInt();
-        Result.IntReal = HandleFloatToIntCast(EltType, SubType, Real, Info.Ctx);
-        Result.IntImag = APSInt(Result.IntReal.getBitWidth(),
-                                !Result.IntReal.isSigned());
+        Result.IntReal = HandleFloatToIntCast(EltType, SrcType,
+                                              Result.FloatReal,
+                                              Info.Ctx);
+        Result.IntImag = HandleFloatToIntCast(EltType, SrcType,
+                                              Result.FloatImag,
+                                              Info.Ctx);
         return true;
       }
-    } else if (SubType->isIntegerType()) {
-      APSInt &Real = Result.IntReal;
-      if (!EvaluateInteger(SubExpr, Real, Info))
-        return false;
-
+    } else {
+      assert(Result.isComplexInt() && "Invalid evaluate result.");
       if (EltType->isRealFloatingType()) {
         Result.makeComplexFloat();
-        Result.FloatReal
-          = HandleIntToFloatCast(EltType, SubType, Real, Info.Ctx);
-        Result.FloatImag = APFloat(Result.FloatReal.getSemantics());
+        Result.FloatReal = HandleIntToFloatCast(EltType, SrcType,
+                                                Result.IntReal,
+                                                Info.Ctx);
+        Result.FloatImag = HandleIntToFloatCast(EltType, SrcType,
+                                                Result.IntImag,
+                                                Info.Ctx);
         return true;
       } else {
         Result.makeComplexInt();
-        Real = HandleIntToIntCast(EltType, SubType, Real, Info.Ctx);
-        Result.IntImag = APSInt(Real.getBitWidth(), !Real.isSigned());
+        Result.IntReal = HandleIntToIntCast(EltType, SrcType,
+                                            Result.IntReal,
+                                            Info.Ctx);
+        Result.IntImag = HandleIntToIntCast(EltType, SrcType,
+                                            Result.IntImag,
+                                            Info.Ctx);
         return true;
       }
-    } else if (const ComplexType *CT = SubType->getAs<ComplexType>()) {
-      if (!Visit(SubExpr))
-        return false;
-
-      QualType SrcType = CT->getElementType();
-
-      if (Result.isComplexFloat()) {
-        if (EltType->isRealFloatingType()) {
-          Result.makeComplexFloat();
-          Result.FloatReal = HandleFloatToFloatCast(EltType, SrcType,
-                                                    Result.FloatReal,
-                                                    Info.Ctx);
-          Result.FloatImag = HandleFloatToFloatCast(EltType, SrcType,
-                                                    Result.FloatImag,
-                                                    Info.Ctx);
-          return true;
-        } else {
-          Result.makeComplexInt();
-          Result.IntReal = HandleFloatToIntCast(EltType, SrcType,
-                                                Result.FloatReal,
-                                                Info.Ctx);
-          Result.IntImag = HandleFloatToIntCast(EltType, SrcType,
-                                                Result.FloatImag,
-                                                Info.Ctx);
-          return true;
-        }
-      } else {
-        assert(Result.isComplexInt() && "Invalid evaluate result.");
-        if (EltType->isRealFloatingType()) {
-          Result.makeComplexFloat();
-          Result.FloatReal = HandleIntToFloatCast(EltType, SrcType,
-                                                  Result.IntReal,
-                                                  Info.Ctx);
-          Result.FloatImag = HandleIntToFloatCast(EltType, SrcType,
-                                                  Result.IntImag,
-                                                  Info.Ctx);
-          return true;
-        } else {
-          Result.makeComplexInt();
-          Result.IntReal = HandleIntToIntCast(EltType, SrcType,
-                                              Result.IntReal,
-                                              Info.Ctx);
-          Result.IntImag = HandleIntToIntCast(EltType, SrcType,
-                                              Result.IntImag,
-                                              Info.Ctx);
-          return true;
-        }
-      }
     }
-
-    // FIXME: Handle more casts.
-    return false;
   }
 
-  bool VisitBinaryOperator(const BinaryOperator *E);
-  bool VisitChooseExpr(const ChooseExpr *E)
-    { return Visit(E->getChosenSubExpr(Info.Ctx)); }
-  bool VisitUnaryExtension(const UnaryOperator *E)
-    { return Visit(E->getSubExpr()); }
-  // FIXME Missing: unary +/-/~, binary div, ImplicitValueInitExpr,
-  //                conditional ?:, comma
-};
-} // end anonymous namespace
-
-static bool EvaluateComplex(const Expr *E, ComplexValue &Result,
-                            EvalInfo &Info) {
-  assert(E->getType()->isAnyComplexType());
-  return ComplexExprEvaluator(Info, Result).Visit(const_cast<Expr*>(E));
+  // FIXME: Handle more casts.
+  return false;
 }
 
 bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
@@ -2136,7 +2142,7 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
          "Invalid operands to binary operator.");
   switch (E->getOpcode()) {
   default: return false;
-  case BinaryOperator::Add:
+  case BO_Add:
     if (Result.isComplexFloat()) {
       Result.getComplexFloatReal().add(RHS.getComplexFloatReal(),
                                        APFloat::rmNearestTiesToEven);
@@ -2147,7 +2153,7 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       Result.getComplexIntImag() += RHS.getComplexIntImag();
     }
     break;
-  case BinaryOperator::Sub:
+  case BO_Sub:
     if (Result.isComplexFloat()) {
       Result.getComplexFloatReal().subtract(RHS.getComplexFloatReal(),
                                             APFloat::rmNearestTiesToEven);
@@ -2158,7 +2164,7 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
       Result.getComplexIntImag() -= RHS.getComplexIntImag();
     }
     break;
-  case BinaryOperator::Mul:
+  case BO_Mul:
     if (Result.isComplexFloat()) {
       ComplexValue LHS = Result;
       APFloat &LHS_r = LHS.getComplexFloatReal();
@@ -2321,6 +2327,8 @@ APSInt Expr::EvaluateAsInt(ASTContext &Ctx) const {
 //    the comma operator in C99 mode.
 // 2: This expression is not an ICE, and is not a legal subexpression for one.
 
+namespace {
+
 struct ICEDiag {
   unsigned Val;
   SourceLocation Loc;
@@ -2330,7 +2338,9 @@ struct ICEDiag {
   ICEDiag() : Val(0) {}
 };
 
-ICEDiag NoDiag() { return ICEDiag(); }
+}
+
+static ICEDiag NoDiag() { return ICEDiag(); }
 
 static ICEDiag CheckEvalInICE(const Expr* E, ASTContext &Ctx) {
   Expr::EvalResult EVResult;
@@ -2380,7 +2390,6 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::DependentScopeDeclRefExprClass:
   case Expr::CXXConstructExprClass:
   case Expr::CXXBindTemporaryExprClass:
-  case Expr::CXXBindReferenceExprClass:
   case Expr::CXXExprWithTemporariesClass:
   case Expr::CXXTemporaryObjectExprClass:
   case Expr::CXXUnresolvedConstructExprClass:
@@ -2476,23 +2485,21 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::UnaryOperatorClass: {
     const UnaryOperator *Exp = cast<UnaryOperator>(E);
     switch (Exp->getOpcode()) {
-    case UnaryOperator::PostInc:
-    case UnaryOperator::PostDec:
-    case UnaryOperator::PreInc:
-    case UnaryOperator::PreDec:
-    case UnaryOperator::AddrOf:
-    case UnaryOperator::Deref:
+    case UO_PostInc:
+    case UO_PostDec:
+    case UO_PreInc:
+    case UO_PreDec:
+    case UO_AddrOf:
+    case UO_Deref:
       return ICEDiag(2, E->getLocStart());
-    case UnaryOperator::Extension:
-    case UnaryOperator::LNot:
-    case UnaryOperator::Plus:
-    case UnaryOperator::Minus:
-    case UnaryOperator::Not:
-    case UnaryOperator::Real:
-    case UnaryOperator::Imag:
+    case UO_Extension:
+    case UO_LNot:
+    case UO_Plus:
+    case UO_Minus:
+    case UO_Not:
+    case UO_Real:
+    case UO_Imag:
       return CheckICE(Exp->getSubExpr(), Ctx);
-    case UnaryOperator::OffsetOf:
-      break;
     }
     
     // OffsetOf falls through here.
@@ -2515,42 +2522,42 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::BinaryOperatorClass: {
     const BinaryOperator *Exp = cast<BinaryOperator>(E);
     switch (Exp->getOpcode()) {
-    case BinaryOperator::PtrMemD:
-    case BinaryOperator::PtrMemI:
-    case BinaryOperator::Assign:
-    case BinaryOperator::MulAssign:
-    case BinaryOperator::DivAssign:
-    case BinaryOperator::RemAssign:
-    case BinaryOperator::AddAssign:
-    case BinaryOperator::SubAssign:
-    case BinaryOperator::ShlAssign:
-    case BinaryOperator::ShrAssign:
-    case BinaryOperator::AndAssign:
-    case BinaryOperator::XorAssign:
-    case BinaryOperator::OrAssign:
+    case BO_PtrMemD:
+    case BO_PtrMemI:
+    case BO_Assign:
+    case BO_MulAssign:
+    case BO_DivAssign:
+    case BO_RemAssign:
+    case BO_AddAssign:
+    case BO_SubAssign:
+    case BO_ShlAssign:
+    case BO_ShrAssign:
+    case BO_AndAssign:
+    case BO_XorAssign:
+    case BO_OrAssign:
       return ICEDiag(2, E->getLocStart());
 
-    case BinaryOperator::Mul:
-    case BinaryOperator::Div:
-    case BinaryOperator::Rem:
-    case BinaryOperator::Add:
-    case BinaryOperator::Sub:
-    case BinaryOperator::Shl:
-    case BinaryOperator::Shr:
-    case BinaryOperator::LT:
-    case BinaryOperator::GT:
-    case BinaryOperator::LE:
-    case BinaryOperator::GE:
-    case BinaryOperator::EQ:
-    case BinaryOperator::NE:
-    case BinaryOperator::And:
-    case BinaryOperator::Xor:
-    case BinaryOperator::Or:
-    case BinaryOperator::Comma: {
+    case BO_Mul:
+    case BO_Div:
+    case BO_Rem:
+    case BO_Add:
+    case BO_Sub:
+    case BO_Shl:
+    case BO_Shr:
+    case BO_LT:
+    case BO_GT:
+    case BO_LE:
+    case BO_GE:
+    case BO_EQ:
+    case BO_NE:
+    case BO_And:
+    case BO_Xor:
+    case BO_Or:
+    case BO_Comma: {
       ICEDiag LHSResult = CheckICE(Exp->getLHS(), Ctx);
       ICEDiag RHSResult = CheckICE(Exp->getRHS(), Ctx);
-      if (Exp->getOpcode() == BinaryOperator::Div ||
-          Exp->getOpcode() == BinaryOperator::Rem) {
+      if (Exp->getOpcode() == BO_Div ||
+          Exp->getOpcode() == BO_Rem) {
         // Evaluate gives an error for undefined Div/Rem, so make sure
         // we don't evaluate one.
         if (LHSResult.Val != 2 && RHSResult.Val != 2) {
@@ -2564,7 +2571,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
           }
         }
       }
-      if (Exp->getOpcode() == BinaryOperator::Comma) {
+      if (Exp->getOpcode() == BO_Comma) {
         if (Ctx.getLangOptions().C99) {
           // C99 6.6p3 introduces a strange edge case: comma can be in an ICE
           // if it isn't evaluated.
@@ -2579,15 +2586,15 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
         return LHSResult;
       return RHSResult;
     }
-    case BinaryOperator::LAnd:
-    case BinaryOperator::LOr: {
+    case BO_LAnd:
+    case BO_LOr: {
       ICEDiag LHSResult = CheckICE(Exp->getLHS(), Ctx);
       ICEDiag RHSResult = CheckICE(Exp->getRHS(), Ctx);
       if (LHSResult.Val == 0 && RHSResult.Val == 1) {
         // Rare case where the RHS has a comma "side-effect"; we need
         // to actually check the condition to see whether the side
         // with the comma is evaluated.
-        if ((Exp->getOpcode() == BinaryOperator::LAnd) !=
+        if ((Exp->getOpcode() == BO_LAnd) !=
             (Exp->getLHS()->EvaluateAsInt(Ctx) == 0))
           return RHSResult;
         return NoDiag();
diff --git a/lib/AST/FullExpr.cpp b/lib/AST/FullExpr.cpp
index f47284f3d060..93ee8d136000 100644
--- a/lib/AST/FullExpr.cpp
+++ b/lib/AST/FullExpr.cpp
@@ -43,16 +43,3 @@ FullExpr FullExpr::Create(ASTContext &Context, Expr *SubExpr,
   return E;
 }
 
-void FullExpr::Destroy(ASTContext &Context) {
-  if (Expr *E = SubExpr.dyn_cast<Expr *>()) {
-    E->Destroy(Context);
-    return;
-  }
-  
-  ExprAndTemporaries *ET = SubExpr.get<ExprAndTemporaries *>();
-  for (ExprAndTemporaries::temps_iterator i = ET->temps_begin(), 
-       e = ET->temps_end(); i != e; ++i)
-    (*i)->Destroy(Context);
-
-  Context.Deallocate(ET);
-}
diff --git a/lib/AST/ItaniumCXXABI.cpp b/lib/AST/ItaniumCXXABI.cpp
new file mode 100644
index 000000000000..c3fa4666537b
--- /dev/null
+++ b/lib/AST/ItaniumCXXABI.cpp
@@ -0,0 +1,52 @@
+//===------- ItaniumCXXABI.cpp - AST support for the Itanium C++ ABI ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides C++ AST support targetting the Itanium C++ ABI, which is
+// documented at:
+//  http://www.codesourcery.com/public/cxx-abi/abi.html
+//  http://www.codesourcery.com/public/cxx-abi/abi-eh.html
+//
+// It also supports the closely-related ARM C++ ABI, documented at:
+// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0041c/IHI0041C_cppabi.pdf
+//
+//===----------------------------------------------------------------------===//
+
+#include "CXXABI.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Type.h"
+
+using namespace clang;
+
+namespace {
+class ItaniumCXXABI : public CXXABI {
+protected:
+  ASTContext &Context;
+public:
+  ItaniumCXXABI(ASTContext &Ctx) : Context(Ctx) { }
+
+  unsigned getMemberPointerSize(const MemberPointerType *MPT) const {
+    QualType Pointee = MPT->getPointeeType();
+    if (Pointee->isFunctionType()) return 2;
+    return 1;
+  }
+};
+
+class ARMCXXABI : public ItaniumCXXABI {
+public:
+  ARMCXXABI(ASTContext &Ctx) : ItaniumCXXABI(Ctx) { }
+};
+}
+
+CXXABI *clang::CreateItaniumCXXABI(ASTContext &Ctx) {
+  return new ItaniumCXXABI(Ctx);
+}
+
+CXXABI *clang::CreateARMCXXABI(ASTContext &Ctx) {
+  return new ARMCXXABI(Ctx);
+}
diff --git a/lib/AST/Makefile b/lib/AST/Makefile
index 7a1672b81728..65383c5552d4 100644
--- a/lib/AST/Makefile
+++ b/lib/AST/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangAST
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/AST/MicrosoftCXXABI.cpp b/lib/AST/MicrosoftCXXABI.cpp
new file mode 100644
index 000000000000..87b77673925f
--- /dev/null
+++ b/lib/AST/MicrosoftCXXABI.cpp
@@ -0,0 +1,48 @@
+//===------- MicrosoftCXXABI.cpp - AST support for the Microsoft C++ ABI --===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides C++ AST support targetting the Microsoft Visual C++
+// ABI.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CXXABI.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Type.h"
+#include "clang/AST/DeclCXX.h"
+
+using namespace clang;
+
+namespace {
+class MicrosoftCXXABI : public CXXABI {
+  ASTContext &Context;
+public:
+  MicrosoftCXXABI(ASTContext &Ctx) : Context(Ctx) { }
+
+  unsigned getMemberPointerSize(const MemberPointerType *MPT) const;
+};
+}
+
+unsigned MicrosoftCXXABI::getMemberPointerSize(const MemberPointerType *MPT) const {
+  QualType Pointee = MPT->getPointeeType();
+  CXXRecordDecl *RD = MPT->getClass()->getAsCXXRecordDecl();
+  if (RD->getNumVBases() > 0) {
+    if (Pointee->isFunctionType())
+      return 3;
+    else
+      return 2;
+  } else if (RD->getNumBases() > 1 && Pointee->isFunctionType())
+    return 2;
+  return 1;
+}
+
+CXXABI *clang::CreateMicrosoftCXXABI(ASTContext &Ctx) {
+  return new MicrosoftCXXABI(Ctx);
+}
+
diff --git a/lib/AST/NestedNameSpecifier.cpp b/lib/AST/NestedNameSpecifier.cpp
index d6594cdfd02f..212def8565ea 100644
--- a/lib/AST/NestedNameSpecifier.cpp
+++ b/lib/AST/NestedNameSpecifier.cpp
@@ -176,11 +176,6 @@ NestedNameSpecifier::print(llvm::raw_ostream &OS,
   OS << "::";
 }
 
-void NestedNameSpecifier::Destroy(ASTContext &Context) {
-  this->~NestedNameSpecifier();
-  Context.Deallocate((void *)this);
-}
-
 void NestedNameSpecifier::dump(const LangOptions &LO) {
   print(llvm::errs(), PrintingPolicy(LO));
 }
diff --git a/lib/AST/ParentMap.cpp b/lib/AST/ParentMap.cpp
index 48251d52fd2a..5fe873acf7ac 100644
--- a/lib/AST/ParentMap.cpp
+++ b/lib/AST/ParentMap.cpp
@@ -73,7 +73,7 @@ bool ParentMap::isConsumedExpr(Expr* E) const {
       BinaryOperator *BE = cast<BinaryOperator>(P);
       // If it is a comma, only the right side is consumed.
       // If it isn't a comma, both sides are consumed.
-      return BE->getOpcode()!=BinaryOperator::Comma ||DirectChild==BE->getRHS();
+      return BE->getOpcode()!=BO_Comma ||DirectChild==BE->getRHS();
     }
     case Stmt::ForStmtClass:
       return DirectChild == cast<ForStmt>(P)->getCond();
diff --git a/lib/AST/RecordLayout.cpp b/lib/AST/RecordLayout.cpp
index 262c4597f846..4d9c51633626 100644
--- a/lib/AST/RecordLayout.cpp
+++ b/lib/AST/RecordLayout.cpp
@@ -19,8 +19,10 @@ using namespace clang;
 void ASTRecordLayout::Destroy(ASTContext &Ctx) {
   if (FieldOffsets)
     Ctx.Deallocate(FieldOffsets);
-  if (CXXInfo)
+  if (CXXInfo) {
     Ctx.Deallocate(CXXInfo);
+    CXXInfo->~CXXRecordLayoutInfo();
+  }
   this->~ASTRecordLayout();
   Ctx.Deallocate(this);
 }
diff --git a/lib/AST/RecordLayoutBuilder.cpp b/lib/AST/RecordLayoutBuilder.cpp
index 88d71ce04287..13fae299d877 100644
--- a/lib/AST/RecordLayoutBuilder.cpp
+++ b/lib/AST/RecordLayoutBuilder.cpp
@@ -73,22 +73,11 @@ class EmptySubobjectMap {
   /// member subobject that is empty.
   void ComputeEmptySubobjectSizes();
   
-  bool CanPlaceSubobjectAtOffset(const CXXRecordDecl *RD, 
-                                 uint64_t Offset) const;
-
   void AddSubobjectAtOffset(const CXXRecordDecl *RD, uint64_t Offset);
   
-  bool CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
-                                     uint64_t Offset);
   void UpdateEmptyBaseSubobjects(const BaseSubobjectInfo *Info,
                                  uint64_t Offset, bool PlacingEmptyBase);
   
-  bool CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD, 
-                                      const CXXRecordDecl *Class,
-                                      uint64_t Offset) const;
-  bool CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
-                                      uint64_t Offset) const;
-  
   void UpdateEmptyFieldSubobjects(const CXXRecordDecl *RD, 
                                   const CXXRecordDecl *Class,
                                   uint64_t Offset);
@@ -100,6 +89,19 @@ class EmptySubobjectMap {
     return Offset <= MaxEmptyClassOffset;
   }
 
+protected:
+  bool CanPlaceSubobjectAtOffset(const CXXRecordDecl *RD, 
+                                 uint64_t Offset) const;
+
+  bool CanPlaceBaseSubobjectAtOffset(const BaseSubobjectInfo *Info,
+                                     uint64_t Offset);
+
+  bool CanPlaceFieldSubobjectAtOffset(const CXXRecordDecl *RD, 
+                                      const CXXRecordDecl *Class,
+                                      uint64_t Offset) const;
+  bool CanPlaceFieldSubobjectAtOffset(const FieldDecl *FD,
+                                      uint64_t Offset) const;
+
 public:
   /// This holds the size of the largest empty subobject (either a base
   /// or a member). Will be zero if the record being built doesn't contain
@@ -513,6 +515,7 @@ void EmptySubobjectMap::UpdateEmptyFieldSubobjects(const FieldDecl *FD,
 }
 
 class RecordLayoutBuilder {
+protected:
   // FIXME: Remove this and make the appropriate fields public.
   friend class clang::ASTContext;
 
@@ -623,12 +626,14 @@ class RecordLayoutBuilder {
 
   void SelectPrimaryVBase(const CXXRecordDecl *RD);
 
+  virtual uint64_t GetVirtualPointersSize(const CXXRecordDecl *RD) const;
+
   /// IdentifyPrimaryBases - Identify all virtual base classes, direct or
   /// indirect, that are primary base classes for some other direct or indirect
   /// base class.
   void IdentifyPrimaryBases(const CXXRecordDecl *RD);
 
-  bool IsNearlyEmpty(const CXXRecordDecl *RD) const;
+  virtual bool IsNearlyEmpty(const CXXRecordDecl *RD) const;
 
   /// LayoutNonVirtualBases - Determines the primary base class (if any) and
   /// lays it out. Will then proceed to lay out all non-virtual base clasess.
@@ -638,7 +643,7 @@ class RecordLayoutBuilder {
   void LayoutNonVirtualBase(const BaseSubobjectInfo *Base);
 
   void AddPrimaryVirtualBaseOffsets(const BaseSubobjectInfo *Info, 
-                                    uint64_t Offset);
+                                            uint64_t Offset);
 
   /// LayoutVirtualBases - Lays out all the virtual bases.
   void LayoutVirtualBases(const CXXRecordDecl *RD,
@@ -664,6 +669,8 @@ class RecordLayoutBuilder {
   void operator=(const RecordLayoutBuilder&); // DO NOT IMPLEMENT
 public:
   static const CXXMethodDecl *ComputeKeyFunction(const CXXRecordDecl *RD);
+
+  virtual ~RecordLayoutBuilder() { }
 };
 } // end anonymous namespace
 
@@ -734,6 +741,11 @@ RecordLayoutBuilder::SelectPrimaryVBase(const CXXRecordDecl *RD) {
   }
 }
 
+uint64_t
+RecordLayoutBuilder::GetVirtualPointersSize(const CXXRecordDecl *RD) const {
+  return Context.Target.getPointerWidth(0);
+}
+
 /// DeterminePrimaryBase - Determine the primary base of the given class.
 void RecordLayoutBuilder::DeterminePrimaryBase(const CXXRecordDecl *RD) {
   // If the class isn't dynamic, it won't have a primary base.
@@ -794,7 +806,7 @@ void RecordLayoutBuilder::DeterminePrimaryBase(const CXXRecordDecl *RD) {
   assert(DataSize == 0 && "Vtable pointer must be at offset zero!");
 
   // Update the size.
-  Size += Context.Target.getPointerWidth(0);
+  Size += GetVirtualPointersSize(RD);
   DataSize = Size;
 
   // Update the alignment.
@@ -1123,8 +1135,8 @@ void RecordLayoutBuilder::InitializeLayout(const Decl *D) {
     if (const MaxFieldAlignmentAttr *MFAA = D->getAttr<MaxFieldAlignmentAttr>())
       MaxFieldAlignment = MFAA->getAlignment();
 
-    if (const AlignedAttr *AA = D->getAttr<AlignedAttr>())
-      UpdateAlignment(AA->getMaxAlignment());
+    if (unsigned MaxAlign = D->getMaxAlignment())
+      UpdateAlignment(MaxAlign);
   }
 }
 
@@ -1287,8 +1299,7 @@ void RecordLayoutBuilder::LayoutBitField(const FieldDecl *D) {
 
   if (FieldPacked || !Context.Target.useBitFieldTypeAlignment())
     FieldAlign = 1;
-  if (const AlignedAttr *AA = D->getAttr<AlignedAttr>())
-    FieldAlign = std::max(FieldAlign, AA->getMaxAlignment());
+  FieldAlign = std::max(FieldAlign, D->getMaxAlignment());
 
   // The maximum field alignment overrides the aligned attribute.
   if (MaxFieldAlignment)
@@ -1357,8 +1368,7 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
 
   if (FieldPacked)
     FieldAlign = 8;
-  if (const AlignedAttr *AA = D->getAttr<AlignedAttr>())
-    FieldAlign = std::max(FieldAlign, AA->getMaxAlignment());
+  FieldAlign = std::max(FieldAlign, D->getMaxAlignment());
 
   // The maximum field alignment overrides the aligned attribute.
   if (MaxFieldAlignment)
@@ -1453,6 +1463,37 @@ RecordLayoutBuilder::ComputeKeyFunction(const CXXRecordDecl *RD) {
   return 0;
 }
 
+// This class implements layout specific to the Microsoft ABI.
+class MSRecordLayoutBuilder: public RecordLayoutBuilder {
+public:
+  MSRecordLayoutBuilder(ASTContext& Ctx, EmptySubobjectMap *EmptySubobjects):
+    RecordLayoutBuilder(Ctx, EmptySubobjects) {}
+
+  virtual bool IsNearlyEmpty(const CXXRecordDecl *RD) const;
+  virtual uint64_t GetVirtualPointersSize(const CXXRecordDecl *RD) const;
+};
+
+bool MSRecordLayoutBuilder::IsNearlyEmpty(const CXXRecordDecl *RD) const {
+  // FIXME: Audit the corners
+  if (!RD->isDynamicClass())
+    return false;
+  const ASTRecordLayout &BaseInfo = Context.getASTRecordLayout(RD);
+  // In the Microsoft ABI, classes can have one or two vtable pointers.
+  if (BaseInfo.getNonVirtualSize() == Context.Target.getPointerWidth(0) ||
+      BaseInfo.getNonVirtualSize() == Context.Target.getPointerWidth(0) * 2)
+    return true;
+  return false;
+}
+
+uint64_t
+MSRecordLayoutBuilder::GetVirtualPointersSize(const CXXRecordDecl *RD) const {
+  // We should reserve space for two pointers if the class has both
+  // virtual functions and virtual bases.
+  if (RD->isPolymorphic() && RD->getNumVBases() > 0)
+    return 2 * Context.Target.getPointerWidth(0);
+  return Context.Target.getPointerWidth(0);
+}
+
 /// getASTRecordLayout - Get or compute information about the layout of the
 /// specified record (struct/union/class), which indicates its size and field
 /// position information.
@@ -1471,8 +1512,16 @@ const ASTRecordLayout &ASTContext::getASTRecordLayout(const RecordDecl *D) {
   if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
     EmptySubobjectMap EmptySubobjects(*this, RD);
 
-    RecordLayoutBuilder Builder(*this, &EmptySubobjects);
-    Builder.Layout(RD);
+    // When compiling for Microsoft, use the special MS builder.
+    llvm::OwningPtr<RecordLayoutBuilder> Builder;
+    switch (Target.getCXXABI()) {
+    default:
+      Builder.reset(new RecordLayoutBuilder(*this, &EmptySubobjects));
+      break;
+    case CXXABI_Microsoft:
+      Builder.reset(new MSRecordLayoutBuilder(*this, &EmptySubobjects));
+    }
+    Builder->Layout(RD);
 
     // FIXME: This is not always correct. See the part about bitfields at
     // http://www.codesourcery.com/public/cxx-abi/abi.html#POD for more info.
@@ -1481,20 +1530,20 @@ const ASTRecordLayout &ASTContext::getASTRecordLayout(const RecordDecl *D) {
 
     // FIXME: This should be done in FinalizeLayout.
     uint64_t DataSize =
-      IsPODForThePurposeOfLayout ? Builder.Size : Builder.DataSize;
+      IsPODForThePurposeOfLayout ? Builder->Size : Builder->DataSize;
     uint64_t NonVirtualSize =
-      IsPODForThePurposeOfLayout ? DataSize : Builder.NonVirtualSize;
+      IsPODForThePurposeOfLayout ? DataSize : Builder->NonVirtualSize;
 
     NewEntry =
-      new (*this) ASTRecordLayout(*this, Builder.Size, Builder.Alignment,
-                                  DataSize, Builder.FieldOffsets.data(),
-                                  Builder.FieldOffsets.size(),
+      new (*this) ASTRecordLayout(*this, Builder->Size, Builder->Alignment,
+                                  DataSize, Builder->FieldOffsets.data(),
+                                  Builder->FieldOffsets.size(),
                                   NonVirtualSize,
-                                  Builder.NonVirtualAlignment,
+                                  Builder->NonVirtualAlignment,
                                   EmptySubobjects.SizeOfLargestEmptySubobject,
-                                  Builder.PrimaryBase,
-                                  Builder.PrimaryBaseIsVirtual,
-                                  Builder.Bases, Builder.VBases);
+                                  Builder->PrimaryBase,
+                                  Builder->PrimaryBaseIsVirtual,
+                                  Builder->Bases, Builder->VBases);
   } else {
     RecordLayoutBuilder Builder(*this, /*EmptySubobjects=*/0);
     Builder.Layout(D);
@@ -1630,7 +1679,7 @@ static void DumpCXXRecordLayout(llvm::raw_ostream &OS,
     if (const RecordType *RT = Field->getType()->getAs<RecordType>()) {
       if (const CXXRecordDecl *D = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
         DumpCXXRecordLayout(OS, D, C, FieldOffset, IndentLevel,
-                            Field->getNameAsCString(),
+                            Field->getName().data(),
                             /*IncludeVirtualBases=*/true);
         continue;
       }
diff --git a/lib/AST/Stmt.cpp b/lib/AST/Stmt.cpp
index 6dbe8f4d18c1..fc8898173f3e 100644
--- a/lib/AST/Stmt.cpp
+++ b/lib/AST/Stmt.cpp
@@ -119,7 +119,7 @@ bool Stmt::hasImplicitControlFlow() const {
 
     case Stmt::BinaryOperatorClass: {
       const BinaryOperator* B = cast<BinaryOperator>(this);
-      if (B->isLogicalOp() || B->getOpcode() == BinaryOperator::Comma)
+      if (B->isLogicalOp() || B->getOpcode() == BO_Comma)
         return true;
       else
         return false;
@@ -215,8 +215,9 @@ int AsmStmt::getNamedOperand(llvm::StringRef SymbolicName) const {
 /// true, otherwise return false.
 unsigned AsmStmt::AnalyzeAsmString(llvm::SmallVectorImpl<AsmStringPiece>&Pieces,
                                    ASTContext &C, unsigned &DiagOffs) const {
-  const char *StrStart = getAsmString()->getStrData();
-  const char *StrEnd = StrStart + getAsmString()->getByteLength();
+  llvm::StringRef Str = getAsmString()->getString();
+  const char *StrStart = Str.begin();
+  const char *StrEnd = Str.end();
   const char *CurPtr = StrStart;
 
   // "Simple" inline asms have no constraints or operands, just convert the asm
@@ -451,6 +452,15 @@ CXXTryStmt *CXXTryStmt::Create(ASTContext &C, SourceLocation tryLoc,
   return new (Mem) CXXTryStmt(tryLoc, tryBlock, handlers, numHandlers);
 }
 
+CXXTryStmt *CXXTryStmt::Create(ASTContext &C, EmptyShell Empty,
+                               unsigned numHandlers) {
+  std::size_t Size = sizeof(CXXTryStmt);
+  Size += ((numHandlers + 1) * sizeof(Stmt));
+
+  void *Mem = C.Allocate(Size, llvm::alignof<CXXTryStmt>());
+  return new (Mem) CXXTryStmt(Empty, numHandlers);
+}
+
 CXXTryStmt::CXXTryStmt(SourceLocation tryLoc, Stmt *tryBlock,
                        Stmt **handlers, unsigned numHandlers)
   : Stmt(CXXTryStmtClass), TryLoc(tryLoc), NumHandlers(numHandlers) {
@@ -459,46 +469,6 @@ CXXTryStmt::CXXTryStmt(SourceLocation tryLoc, Stmt *tryBlock,
   std::copy(handlers, handlers + NumHandlers, Stmts + 1);
 }
 
-//===----------------------------------------------------------------------===//
-// AST Destruction.
-//===----------------------------------------------------------------------===//
-
-void Stmt::DestroyChildren(ASTContext &C) {
-  for (child_iterator I = child_begin(), E = child_end(); I !=E; )
-    if (Stmt* Child = *I++) Child->Destroy(C);
-}
-
-static void BranchDestroy(ASTContext &C, Stmt *S, Stmt **SubExprs,
-                          unsigned NumExprs) {
-  // We do not use child_iterator here because that will include
-  // the expressions referenced by the condition variable.
-  for (Stmt **I = SubExprs, **E = SubExprs + NumExprs; I != E; ++I)
-    if (Stmt *Child = *I) Child->Destroy(C);
-  
-  S->~Stmt();
-  C.Deallocate((void *) S);
-}
-
-void Stmt::DoDestroy(ASTContext &C) {
-  DestroyChildren(C);
-  this->~Stmt();
-  C.Deallocate((void *)this);
-}
-
-void CXXCatchStmt::DoDestroy(ASTContext& C) {
-  if (ExceptionDecl)
-    ExceptionDecl->Destroy(C);
-  Stmt::DoDestroy(C);
-}
-
-void DeclStmt::DoDestroy(ASTContext &C) {
-  // Don't use StmtIterator to iterate over the Decls, as that can recurse
-  // into VLA size expressions (which are owned by the VLA).  Further, Decls
-  // are owned by the DeclContext, and will be destroyed with them.
-  if (DG.isDeclGroup())
-    DG.getDeclGroup().Destroy(C);
-}
-
 IfStmt::IfStmt(ASTContext &C, SourceLocation IL, VarDecl *var, Expr *cond, 
                Stmt *then, SourceLocation EL, Stmt *elsev)
   : Stmt(IfStmtClass), IfLoc(IL), ElseLoc(EL)
@@ -528,10 +498,6 @@ void IfStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
                                    V->getSourceRange().getEnd());
 }
 
-void IfStmt::DoDestroy(ASTContext &C) {
-  BranchDestroy(C, this, SubExprs, END_EXPR);
-}
-
 ForStmt::ForStmt(ASTContext &C, Stmt *Init, Expr *Cond, VarDecl *condVar, 
                  Expr *Inc, Stmt *Body, SourceLocation FL, SourceLocation LP, 
                  SourceLocation RP)
@@ -563,10 +529,6 @@ void ForStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
                                        V->getSourceRange().getEnd());
 }
 
-void ForStmt::DoDestroy(ASTContext &C) {
-  BranchDestroy(C, this, SubExprs, END_EXPR);
-}
-
 SwitchStmt::SwitchStmt(ASTContext &C, VarDecl *Var, Expr *cond) 
   : Stmt(SwitchStmtClass), FirstCase(0) 
 {
@@ -594,20 +556,6 @@ void SwitchStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
                                    V->getSourceRange().getEnd());
 }
 
-void SwitchStmt::DoDestroy(ASTContext &C) {
-  // Destroy the SwitchCase statements in this switch. In the normal
-  // case, this loop will merely decrement the reference counts from
-  // the Retain() calls in addSwitchCase();
-  SwitchCase *SC = FirstCase;
-  while (SC) {
-    SwitchCase *Next = SC->getNextSwitchCase();
-    SC->Destroy(C);
-    SC = Next;
-  }
-  
-  BranchDestroy(C, this, SubExprs, END_EXPR);
-}
-
 WhileStmt::WhileStmt(ASTContext &C, VarDecl *Var, Expr *cond, Stmt *body, 
                      SourceLocation WL)
 : Stmt(WhileStmtClass)
@@ -637,22 +585,6 @@ void WhileStmt::setConditionVariable(ASTContext &C, VarDecl *V) {
                                    V->getSourceRange().getEnd());
 }
 
-void WhileStmt::DoDestroy(ASTContext &C) {
-  BranchDestroy(C, this, SubExprs, END_EXPR);
-}
-
-void AsmStmt::DoDestroy(ASTContext &C) {
-  DestroyChildren(C);
-  
-  C.Deallocate(Names);
-  C.Deallocate(Constraints);
-  C.Deallocate(Exprs);
-  C.Deallocate(Clobbers);
-  
-  this->~AsmStmt();
-  C.Deallocate((void *)this);
-}
-
 //===----------------------------------------------------------------------===//
 //  Child Iterators for iterating over subexpressions/substatements
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/StmtDumper.cpp b/lib/AST/StmtDumper.cpp
index b388a3b18ffa..5c236a45a690 100644
--- a/lib/AST/StmtDumper.cpp
+++ b/lib/AST/StmtDumper.cpp
@@ -65,6 +65,13 @@ namespace  {
               OS << '\n';
               DumpSubTree(*CI++);
             }
+            if (const ConditionalOperator *CO = 
+                  dyn_cast<ConditionalOperator>(S)) {
+              if (CO->getSAVE()) {
+                OS << '\n';
+                DumpSubTree(CO->getSAVE());
+              }
+            }
           }
         }
         OS << ')';
@@ -225,7 +232,7 @@ void StmtDumper::DumpDeclarator(Decl *D) {
     OS << "\"";
     // Emit storage class for vardecls.
     if (VarDecl *V = dyn_cast<VarDecl>(VD)) {
-      if (V->getStorageClass() != VarDecl::None)
+      if (V->getStorageClass() != SC_None)
         OS << VarDecl::getStorageClassSpecifierString(V->getStorageClass())
            << " ";
     }
@@ -308,13 +315,13 @@ void StmtDumper::VisitExpr(Expr *Node) {
 }
 
 static void DumpBasePath(llvm::raw_ostream &OS, CastExpr *Node) {
-  if (Node->getBasePath().empty())
+  if (Node->path_empty())
     return;
 
   OS << " (";
   bool First = true;
-  for (CXXBaseSpecifierArray::iterator I = Node->getBasePath().begin(),
-       E = Node->getBasePath().end(); I != E; ++I) {
+  for (CastExpr::path_iterator
+         I = Node->path_begin(), E = Node->path_end(); I != E; ++I) {
     const CXXBaseSpecifier *Base = *I;
     if (!First)
       OS << " -> ";
@@ -340,8 +347,16 @@ void StmtDumper::VisitCastExpr(CastExpr *Node) {
 
 void StmtDumper::VisitImplicitCastExpr(ImplicitCastExpr *Node) {
   VisitCastExpr(Node);
-  if (Node->isLvalueCast())
+  switch (Node->getValueKind()) {
+  case VK_LValue:
     OS << " lvalue";
+    break;
+  case VK_XValue:
+    OS << " xvalue";
+    break;
+  case VK_RValue:
+    break;
+  }
 }
 
 void StmtDumper::VisitDeclRefExpr(DeclRefExpr *Node) {
@@ -421,8 +436,7 @@ void StmtDumper::VisitStringLiteral(StringLiteral *Str) {
   if (Str->isWide())
     OS << "L";
   OS << '"';
-  OS.write_escaped(llvm::StringRef(Str->getStrData(),
-                                   Str->getByteLength()));
+  OS.write_escaped(Str->getString());
   OS << '"';
 }
 
@@ -511,6 +525,8 @@ void StmtDumper::VisitCXXConstructExpr(CXXConstructExpr *Node) {
   DumpType(Ctor->getType());
   if (Node->isElidable())
     OS << " elidable";
+  if (Node->requiresZeroInitialization())
+    OS << " zeroing";
 }
 
 void StmtDumper::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *Node) {
@@ -623,9 +639,13 @@ void StmtDumper::VisitObjCSuperExpr(ObjCSuperExpr *Node) {
 /// specified node and a few nodes underneath it, but not the whole subtree.
 /// This is useful in a debugger.
 void Stmt::dump(SourceManager &SM) const {
-  StmtDumper P(&SM, llvm::errs(), 4);
+  dump(llvm::errs(), SM);
+}
+
+void Stmt::dump(llvm::raw_ostream &OS, SourceManager &SM) const {
+  StmtDumper P(&SM, OS, 4);
   P.DumpSubTree(const_cast<Stmt*>(this));
-  llvm::errs() << "\n";
+  OS << "\n";
 }
 
 /// dump - This does a local dump of the specified AST fragment.  It dumps the
diff --git a/lib/AST/StmtPrinter.cpp b/lib/AST/StmtPrinter.cpp
index 7043c3551628..5236a6672622 100644
--- a/lib/AST/StmtPrinter.cpp
+++ b/lib/AST/StmtPrinter.cpp
@@ -77,16 +77,21 @@ namespace  {
       return OS;
     }
 
-    bool PrintOffsetOfDesignator(Expr *E);
-    void VisitUnaryOffsetOf(UnaryOperator *Node);
-
     void Visit(Stmt* S) {
       if (Helper && Helper->handledStmt(S,OS))
           return;
       else StmtVisitor<StmtPrinter>::Visit(S);
     }
+    
+    void VisitStmt(Stmt *Node) ATTRIBUTE_UNUSED {
+      Indent() << "<<unknown stmt type>>\n";
+    }
+    void VisitExpr(Expr *Node) ATTRIBUTE_UNUSED {
+      OS << "<<unknown expr type>>";
+    }
+    void VisitCXXNamedCastExpr(CXXNamedCastExpr *Node);
 
-    void VisitStmt(Stmt *Node);
+#define ABSTRACT_STMT(CLASS)
 #define STMT(CLASS, PARENT) \
     void Visit##CLASS(CLASS *Node);
 #include "clang/AST/StmtNodes.inc"
@@ -97,10 +102,6 @@ namespace  {
 //  Stmt printing methods.
 //===----------------------------------------------------------------------===//
 
-void StmtPrinter::VisitStmt(Stmt *Node) {
-  Indent() << "<<unknown stmt type>>\n";
-}
-
 /// PrintRawCompoundStmt - Print a compound stmt without indenting the {, and
 /// with no newline after the }.
 void StmtPrinter::PrintRawCompoundStmt(CompoundStmt *Node) {
@@ -465,15 +466,11 @@ void StmtPrinter::VisitCXXTryStmt(CXXTryStmt *Node) {
 //  Expr printing methods.
 //===----------------------------------------------------------------------===//
 
-void StmtPrinter::VisitExpr(Expr *Node) {
-  OS << "<<unknown expr type>>";
-}
-
 void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
   if (NestedNameSpecifier *Qualifier = Node->getQualifier())
     Qualifier->print(OS, Policy);
-  OS << Node->getDecl();
-  if (Node->hasExplicitTemplateArgumentList())
+  OS << Node->getNameInfo();
+  if (Node->hasExplicitTemplateArgs())
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
                                                     Node->getTemplateArgs(),
                                                     Node->getNumTemplateArgs(),
@@ -483,7 +480,7 @@ void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
 void StmtPrinter::VisitDependentScopeDeclRefExpr(
                                            DependentScopeDeclRefExpr *Node) {
   Node->getQualifier()->print(OS, Policy);
-  OS << Node->getDeclName().getAsString();
+  OS << Node->getNameInfo();
   if (Node->hasExplicitTemplateArgs())
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
                                                    Node->getTemplateArgs(),
@@ -494,7 +491,7 @@ void StmtPrinter::VisitDependentScopeDeclRefExpr(
 void StmtPrinter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *Node) {
   if (Node->getQualifier())
     Node->getQualifier()->print(OS, Policy);
-  OS << Node->getName().getAsString();
+  OS << Node->getNameInfo();
   if (Node->hasExplicitTemplateArgs())
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
                                                    Node->getTemplateArgs(),
@@ -515,7 +512,7 @@ void StmtPrinter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *Node) {
     PrintExpr(Node->getBase());
     OS << ".";
   }
-  OS << Node->getProperty()->getNameAsCString();
+  OS << Node->getProperty()->getName();
 }
 
 void StmtPrinter::VisitObjCImplicitSetterGetterRefExpr(
@@ -624,8 +621,10 @@ void StmtPrinter::VisitStringLiteral(StringLiteral *Str) {
   OS << '"';
 
   // FIXME: this doesn't print wstrings right.
-  for (unsigned i = 0, e = Str->getByteLength(); i != e; ++i) {
-    unsigned char Char = Str->getStrData()[i];
+  llvm::StringRef StrData = Str->getString();
+  for (llvm::StringRef::iterator I = StrData.begin(), E = StrData.end(); 
+                                                             I != E; ++I) {
+    unsigned char Char = *I;
 
     switch (Char) {
     default:
@@ -661,13 +660,13 @@ void StmtPrinter::VisitUnaryOperator(UnaryOperator *Node) {
     // it might be concatenated incorrectly like '+'.
     switch (Node->getOpcode()) {
     default: break;
-    case UnaryOperator::Real:
-    case UnaryOperator::Imag:
-    case UnaryOperator::Extension:
+    case UO_Real:
+    case UO_Imag:
+    case UO_Extension:
       OS << ' ';
       break;
-    case UnaryOperator::Plus:
-    case UnaryOperator::Minus:
+    case UO_Plus:
+    case UO_Minus:
       if (isa<UnaryOperator>(Node->getSubExpr()))
         OS << ' ';
       break;
@@ -679,31 +678,6 @@ void StmtPrinter::VisitUnaryOperator(UnaryOperator *Node) {
     OS << UnaryOperator::getOpcodeStr(Node->getOpcode());
 }
 
-bool StmtPrinter::PrintOffsetOfDesignator(Expr *E) {
-  if (isa<UnaryOperator>(E)) {
-    // Base case, print the type and comma.
-    OS << E->getType().getAsString(Policy) << ", ";
-    return true;
-  } else if (ArraySubscriptExpr *ASE = dyn_cast<ArraySubscriptExpr>(E)) {
-    PrintOffsetOfDesignator(ASE->getLHS());
-    OS << "[";
-    PrintExpr(ASE->getRHS());
-    OS << "]";
-    return false;
-  } else {
-    MemberExpr *ME = cast<MemberExpr>(E);
-    bool IsFirst = PrintOffsetOfDesignator(ME->getBase());
-    OS << (IsFirst ? "" : ".") << ME->getMemberDecl();
-    return false;
-  }
-}
-
-void StmtPrinter::VisitUnaryOffsetOf(UnaryOperator *Node) {
-  OS << "__builtin_offsetof(";
-  PrintOffsetOfDesignator(Node->getSubExpr());
-  OS << ")";
-}
-
 void StmtPrinter::VisitOffsetOfExpr(OffsetOfExpr *Node) {
   OS << "__builtin_offsetof(";
   OS << Node->getTypeSourceInfo()->getType().getAsString(Policy) << ", ";
@@ -777,9 +751,9 @@ void StmtPrinter::VisitMemberExpr(MemberExpr *Node) {
   if (NestedNameSpecifier *Qualifier = Node->getQualifier())
     Qualifier->print(OS, Policy);
 
-  OS << Node->getMemberDecl();
+  OS << Node->getMemberNameInfo();
 
-  if (Node->hasExplicitTemplateArgumentList())
+  if (Node->hasExplicitTemplateArgs())
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
                                                     Node->getTemplateArgs(),
                                                     Node->getNumTemplateArgs(),
@@ -795,12 +769,6 @@ void StmtPrinter::VisitExtVectorElementExpr(ExtVectorElementExpr *Node) {
   OS << ".";
   OS << Node->getAccessor().getName();
 }
-void StmtPrinter::VisitCastExpr(CastExpr *) {
-  assert(0 && "CastExpr is an abstract class");
-}
-void StmtPrinter::VisitExplicitCastExpr(ExplicitCastExpr *) {
-  assert(0 && "ExplicitCastExpr is an abstract class");
-}
 void StmtPrinter::VisitCStyleCastExpr(CStyleCastExpr *Node) {
   OS << "(" << Node->getType().getAsString(Policy) << ")";
   PrintExpr(Node->getSubExpr());
@@ -1069,10 +1037,6 @@ void StmtPrinter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *Node) {
   PrintExpr(Node->getSubExpr());
 }
 
-void StmtPrinter::VisitCXXBindReferenceExpr(CXXBindReferenceExpr *Node) {
-  PrintExpr(Node->getSubExpr());
-}
-
 void StmtPrinter::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *Node) {
   OS << Node->getType().getAsString(Policy);
   OS << "(";
@@ -1201,7 +1165,7 @@ void StmtPrinter::VisitCXXDependentScopeMemberExpr(
     // FIXME: Track use of "template" keyword explicitly?
     OS << "template ";
 
-  OS << Node->getMember().getAsString();
+  OS << Node->getMemberNameInfo();
 
   if (Node->hasExplicitTemplateArgs()) {
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
@@ -1221,7 +1185,7 @@ void StmtPrinter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *Node) {
 
   // FIXME: this might originally have been written with 'template'
 
-  OS << Node->getMemberName().getAsString();
+  OS << Node->getMemberNameInfo();
 
   if (Node->hasExplicitTemplateArgs()) {
     OS << TemplateSpecializationType::PrintTemplateArgumentList(
@@ -1368,7 +1332,7 @@ void Stmt::printPretty(llvm::raw_ostream &OS, ASTContext& Context,
   }
 
   if (Policy.Dump && &Context) {
-    dump(Context.getSourceManager());
+    dump(OS, Context.getSourceManager());
     return;
   }
 
diff --git a/lib/AST/StmtProfile.cpp b/lib/AST/StmtProfile.cpp
index cff86a4e1cec..78a336d2bfbe 100644
--- a/lib/AST/StmtProfile.cpp
+++ b/lib/AST/StmtProfile.cpp
@@ -325,7 +325,7 @@ void StmtProfiler::VisitCastExpr(CastExpr *S) {
 
 void StmtProfiler::VisitImplicitCastExpr(ImplicitCastExpr *S) {
   VisitCastExpr(S);
-  ID.AddBoolean(S->isLvalueCast());
+  ID.AddInteger(S->getValueKind());
 }
 
 void StmtProfiler::VisitExplicitCastExpr(ExplicitCastExpr *S) {
@@ -436,8 +436,8 @@ void StmtProfiler::VisitBlockDeclRefExpr(BlockDeclRefExpr *S) {
 }
 
 static Stmt::StmtClass DecodeOperatorCall(CXXOperatorCallExpr *S,
-                                          UnaryOperator::Opcode &UnaryOp,
-                                          BinaryOperator::Opcode &BinaryOp) {
+                                          UnaryOperatorKind &UnaryOp,
+                                          BinaryOperatorKind &BinaryOp) {
   switch (S->getOperator()) {
   case OO_None:
   case OO_New:
@@ -453,165 +453,165 @@ static Stmt::StmtClass DecodeOperatorCall(CXXOperatorCallExpr *S,
       
   case OO_Plus:
     if (S->getNumArgs() == 1) {
-      UnaryOp = UnaryOperator::Plus;
+      UnaryOp = UO_Plus;
       return Stmt::UnaryOperatorClass;
     }
     
-    BinaryOp = BinaryOperator::Add;
+    BinaryOp = BO_Add;
     return Stmt::BinaryOperatorClass;
       
   case OO_Minus:
     if (S->getNumArgs() == 1) {
-      UnaryOp = UnaryOperator::Minus;
+      UnaryOp = UO_Minus;
       return Stmt::UnaryOperatorClass;
     }
     
-    BinaryOp = BinaryOperator::Sub;
+    BinaryOp = BO_Sub;
     return Stmt::BinaryOperatorClass;
 
   case OO_Star:
     if (S->getNumArgs() == 1) {
-      UnaryOp = UnaryOperator::Minus;
+      UnaryOp = UO_Minus;
       return Stmt::UnaryOperatorClass;
     }
     
-    BinaryOp = BinaryOperator::Sub;
+    BinaryOp = BO_Sub;
     return Stmt::BinaryOperatorClass;
 
   case OO_Slash:
-    BinaryOp = BinaryOperator::Div;
+    BinaryOp = BO_Div;
     return Stmt::BinaryOperatorClass;
       
   case OO_Percent:
-    BinaryOp = BinaryOperator::Rem;
+    BinaryOp = BO_Rem;
     return Stmt::BinaryOperatorClass;
 
   case OO_Caret:
-    BinaryOp = BinaryOperator::Xor;
+    BinaryOp = BO_Xor;
     return Stmt::BinaryOperatorClass;
 
   case OO_Amp:
     if (S->getNumArgs() == 1) {
-      UnaryOp = UnaryOperator::AddrOf;
+      UnaryOp = UO_AddrOf;
       return Stmt::UnaryOperatorClass;
     }
     
-    BinaryOp = BinaryOperator::And;
+    BinaryOp = BO_And;
     return Stmt::BinaryOperatorClass;
       
   case OO_Pipe:
-    BinaryOp = BinaryOperator::Or;
+    BinaryOp = BO_Or;
     return Stmt::BinaryOperatorClass;
 
   case OO_Tilde:
-    UnaryOp = UnaryOperator::Not;
+    UnaryOp = UO_Not;
     return Stmt::UnaryOperatorClass;
 
   case OO_Exclaim:
-    UnaryOp = UnaryOperator::LNot;
+    UnaryOp = UO_LNot;
     return Stmt::UnaryOperatorClass;
 
   case OO_Equal:
-    BinaryOp = BinaryOperator::Assign;
+    BinaryOp = BO_Assign;
     return Stmt::BinaryOperatorClass;
 
   case OO_Less:
-    BinaryOp = BinaryOperator::LT;
+    BinaryOp = BO_LT;
     return Stmt::BinaryOperatorClass;
 
   case OO_Greater:
-    BinaryOp = BinaryOperator::GT;
+    BinaryOp = BO_GT;
     return Stmt::BinaryOperatorClass;
       
   case OO_PlusEqual:
-    BinaryOp = BinaryOperator::AddAssign;
+    BinaryOp = BO_AddAssign;
     return Stmt::CompoundAssignOperatorClass;
 
   case OO_MinusEqual:
-    BinaryOp = BinaryOperator::SubAssign;
+    BinaryOp = BO_SubAssign;
     return Stmt::CompoundAssignOperatorClass;
 
   case OO_StarEqual:
-    BinaryOp = BinaryOperator::MulAssign;
+    BinaryOp = BO_MulAssign;
     return Stmt::CompoundAssignOperatorClass;
 
   case OO_SlashEqual:
-    BinaryOp = BinaryOperator::DivAssign;
+    BinaryOp = BO_DivAssign;
     return Stmt::CompoundAssignOperatorClass;
 
   case OO_PercentEqual:
-    BinaryOp = BinaryOperator::RemAssign;
+    BinaryOp = BO_RemAssign;
     return Stmt::CompoundAssignOperatorClass;
 
   case OO_CaretEqual:
-    BinaryOp = BinaryOperator::XorAssign;
+    BinaryOp = BO_XorAssign;
     return Stmt::CompoundAssignOperatorClass;
     
   case OO_AmpEqual:
-    BinaryOp = BinaryOperator::AndAssign;
+    BinaryOp = BO_AndAssign;
     return Stmt::CompoundAssignOperatorClass;
     
   case OO_PipeEqual:
-    BinaryOp = BinaryOperator::OrAssign;
+    BinaryOp = BO_OrAssign;
     return Stmt::CompoundAssignOperatorClass;
       
   case OO_LessLess:
-    BinaryOp = BinaryOperator::Shl;
+    BinaryOp = BO_Shl;
     return Stmt::BinaryOperatorClass;
     
   case OO_GreaterGreater:
-    BinaryOp = BinaryOperator::Shr;
+    BinaryOp = BO_Shr;
     return Stmt::BinaryOperatorClass;
 
   case OO_LessLessEqual:
-    BinaryOp = BinaryOperator::ShlAssign;
+    BinaryOp = BO_ShlAssign;
     return Stmt::CompoundAssignOperatorClass;
     
   case OO_GreaterGreaterEqual:
-    BinaryOp = BinaryOperator::ShrAssign;
+    BinaryOp = BO_ShrAssign;
     return Stmt::CompoundAssignOperatorClass;
 
   case OO_EqualEqual:
-    BinaryOp = BinaryOperator::EQ;
+    BinaryOp = BO_EQ;
     return Stmt::BinaryOperatorClass;
     
   case OO_ExclaimEqual:
-    BinaryOp = BinaryOperator::NE;
+    BinaryOp = BO_NE;
     return Stmt::BinaryOperatorClass;
       
   case OO_LessEqual:
-    BinaryOp = BinaryOperator::LE;
+    BinaryOp = BO_LE;
     return Stmt::BinaryOperatorClass;
     
   case OO_GreaterEqual:
-    BinaryOp = BinaryOperator::GE;
+    BinaryOp = BO_GE;
     return Stmt::BinaryOperatorClass;
       
   case OO_AmpAmp:
-    BinaryOp = BinaryOperator::LAnd;
+    BinaryOp = BO_LAnd;
     return Stmt::BinaryOperatorClass;
     
   case OO_PipePipe:
-    BinaryOp = BinaryOperator::LOr;
+    BinaryOp = BO_LOr;
     return Stmt::BinaryOperatorClass;
 
   case OO_PlusPlus:
-    UnaryOp = S->getNumArgs() == 1? UnaryOperator::PreInc 
-                                  : UnaryOperator::PostInc;
+    UnaryOp = S->getNumArgs() == 1? UO_PreInc 
+                                  : UO_PostInc;
     return Stmt::UnaryOperatorClass;
 
   case OO_MinusMinus:
-    UnaryOp = S->getNumArgs() == 1? UnaryOperator::PreDec
-                                  : UnaryOperator::PostDec;
+    UnaryOp = S->getNumArgs() == 1? UO_PreDec
+                                  : UO_PostDec;
     return Stmt::UnaryOperatorClass;
 
   case OO_Comma:
-    BinaryOp = BinaryOperator::Comma;
+    BinaryOp = BO_Comma;
     return Stmt::BinaryOperatorClass;
 
 
   case OO_ArrowStar:
-    BinaryOp = BinaryOperator::PtrMemI;
+    BinaryOp = BO_PtrMemI;
     return Stmt::BinaryOperatorClass;
       
   case OO_Subscript:
@@ -626,8 +626,8 @@ void StmtProfiler::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *S) {
   if (S->isTypeDependent()) {
     // Type-dependent operator calls are profiled like their underlying
     // syntactic operator.
-    UnaryOperator::Opcode UnaryOp = UnaryOperator::Extension;
-    BinaryOperator::Opcode BinaryOp = BinaryOperator::Comma;
+    UnaryOperatorKind UnaryOp = UO_Extension;
+    BinaryOperatorKind BinaryOp = BO_Comma;
     Stmt::StmtClass SC = DecodeOperatorCall(S, UnaryOp, BinaryOp);
     
     ID.AddInteger(SC);
@@ -706,10 +706,6 @@ void StmtProfiler::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *S) {
          const_cast<CXXDestructorDecl *>(S->getTemporary()->getDestructor()));
 }
 
-void StmtProfiler::VisitCXXBindReferenceExpr(CXXBindReferenceExpr *S) {
-  VisitExpr(S);
-}
-
 void StmtProfiler::VisitCXXConstructExpr(CXXConstructExpr *S) {
   VisitExpr(S);
   VisitDecl(S->getConstructor());
@@ -757,14 +753,19 @@ void StmtProfiler::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *S) {
   VisitType(S->getDestroyedType());
 }
 
-void
-StmtProfiler::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *S) {
+void StmtProfiler::VisitOverloadExpr(OverloadExpr *S) {
   VisitExpr(S);
   VisitNestedNameSpecifier(S->getQualifier());
   VisitName(S->getName());
   ID.AddBoolean(S->hasExplicitTemplateArgs());
   if (S->hasExplicitTemplateArgs())
-    VisitTemplateArguments(S->getTemplateArgs(), S->getNumTemplateArgs());
+    VisitTemplateArguments(S->getExplicitTemplateArgs().getTemplateArgs(),
+                           S->getExplicitTemplateArgs().NumTemplateArgs);
+}
+
+void
+StmtProfiler::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *S) {
+  VisitOverloadExpr(S);
 }
 
 void StmtProfiler::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *S) {
diff --git a/lib/AST/TemplateBase.cpp b/lib/AST/TemplateBase.cpp
index 02e648879ad7..a3bf1459c283 100644
--- a/lib/AST/TemplateBase.cpp
+++ b/lib/AST/TemplateBase.cpp
@@ -125,19 +125,22 @@ SourceRange TemplateArgumentLoc::getSourceRange() const {
   switch (Argument.getKind()) {
   case TemplateArgument::Expression:
     return getSourceExpression()->getSourceRange();
-      
+
   case TemplateArgument::Declaration:
     return getSourceDeclExpression()->getSourceRange();
-      
+
   case TemplateArgument::Type:
-    return getTypeSourceInfo()->getTypeLoc().getSourceRange();
-      
+    if (TypeSourceInfo *TSI = getTypeSourceInfo())
+      return TSI->getTypeLoc().getSourceRange();
+    else
+      return SourceRange();
+
   case TemplateArgument::Template:
     if (getTemplateQualifierRange().isValid())
       return SourceRange(getTemplateQualifierRange().getBegin(),
                          getTemplateNameLoc());
     return SourceRange(getTemplateNameLoc());
-      
+
   case TemplateArgument::Integral:
   case TemplateArgument::Pack:
   case TemplateArgument::Null:
@@ -152,7 +155,9 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
                                            const TemplateArgument &Arg) {
   switch (Arg.getKind()) {
   case TemplateArgument::Null:
-    return DB;
+    // This is bad, but not as bad as crashing because of argument
+    // count mismatches.
+    return DB << "(null template argument)";
       
   case TemplateArgument::Type:
     return DB << Arg.getAsType();
diff --git a/lib/AST/Type.cpp b/lib/AST/Type.cpp
index d7929304233f..ca10532e729e 100644
--- a/lib/AST/Type.cpp
+++ b/lib/AST/Type.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/CharUnits.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
@@ -21,6 +22,7 @@
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include <algorithm>
 using namespace clang;
 
 bool QualType::isConstant(QualType T, ASTContext &Ctx) {
@@ -33,24 +35,32 @@ bool QualType::isConstant(QualType T, ASTContext &Ctx) {
   return false;
 }
 
-void Type::Destroy(ASTContext& C) {
-  this->~Type();
-  C.Deallocate(this);
-}
+Type::~Type() { }
+
+unsigned ConstantArrayType::getNumAddressingBits(ASTContext &Context,
+                                                 QualType ElementType,
+                                               const llvm::APInt &NumElements) {
+  llvm::APSInt SizeExtended(NumElements, true);
+  unsigned SizeTypeBits = Context.getTypeSize(Context.getSizeType());
+  SizeExtended.extend(std::max(SizeTypeBits, SizeExtended.getBitWidth()) * 2);
 
-void VariableArrayType::Destroy(ASTContext& C) {
-  if (SizeExpr)
-    SizeExpr->Destroy(C);
-  this->~VariableArrayType();
-  C.Deallocate(this);
+  uint64_t ElementSize
+    = Context.getTypeSizeInChars(ElementType).getQuantity();
+  llvm::APSInt TotalSize(llvm::APInt(SizeExtended.getBitWidth(), ElementSize));
+  TotalSize *= SizeExtended;  
+  
+  return TotalSize.getActiveBits();
 }
 
-void DependentSizedArrayType::Destroy(ASTContext& C) {
-  // FIXME: Resource contention like in ConstantArrayWithExprType ?
-  // May crash, depending on platform or a particular build.
-  // SizeExpr->Destroy(C);
-  this->~DependentSizedArrayType();
-  C.Deallocate(this);
+unsigned ConstantArrayType::getMaxSizeBits(ASTContext &Context) {
+  unsigned Bits = Context.getTypeSize(Context.getSizeType());
+  
+  // GCC appears to only allow 63 bits worth of address space when compiling
+  // for 64-bit, so we do the same.
+  if (Bits == 64)
+    --Bits;
+  
+  return Bits;
 }
 
 void DependentSizedArrayType::Profile(llvm::FoldingSetNodeID &ID,
@@ -73,14 +83,6 @@ DependentSizedExtVectorType::Profile(llvm::FoldingSetNodeID &ID,
   SizeExpr->Profile(ID, Context, true);
 }
 
-void DependentSizedExtVectorType::Destroy(ASTContext& C) {
-  // FIXME: Deallocate size expression, once we're cloning properly.
-//  if (SizeExpr)
-//    SizeExpr->Destroy(C);
-  this->~DependentSizedExtVectorType();
-  C.Deallocate(this);
-}
-
 /// getArrayElementTypeNoTypeQual - If this is an array type, return the
 /// element type of the array, potentially with type qualifiers missing.
 /// This method should never be used when type qualifiers are meaningful.
@@ -192,13 +194,6 @@ bool Type::isVoidType() const {
   return false;
 }
 
-bool Type::isObjectType() const {
-  if (isa<FunctionType>(CanonicalType) || isa<ReferenceType>(CanonicalType) ||
-      isa<IncompleteArrayType>(CanonicalType) || isVoidType())
-    return false;
-  return true;
-}
-
 bool Type::isDerivedType() const {
   switch (CanonicalType->getTypeClass()) {
   case Pointer:
@@ -348,11 +343,6 @@ const RecordType *Type::getAsUnionType() const {
   return 0;
 }
 
-void ObjCInterfaceType::Destroy(ASTContext& C) {
-  this->~ObjCInterfaceType();
-  C.Deallocate(this);
-}
-
 ObjCObjectType::ObjCObjectType(QualType Canonical, QualType Base,
                                ObjCProtocolDecl * const *Protocols,
                                unsigned NumProtocols)
@@ -366,11 +356,6 @@ ObjCObjectType::ObjCObjectType(QualType Canonical, QualType Base,
            NumProtocols * sizeof(ObjCProtocolDecl*));
 }
 
-void ObjCObjectTypeImpl::Destroy(ASTContext& C) {
-  this->~ObjCObjectTypeImpl();
-  C.Deallocate(this);
-}
-
 const ObjCObjectType *Type::getAsObjCQualifiedInterfaceType() const {
   // There is no sugar for ObjCObjectType's, just return the canonical
   // type pointer if it is the right class.  There is no typedef information to
@@ -385,11 +370,6 @@ bool Type::isObjCQualifiedInterfaceType() const {
   return getAsObjCQualifiedInterfaceType() != 0;
 }
 
-void ObjCObjectPointerType::Destroy(ASTContext& C) {
-  this->~ObjCObjectPointerType();
-  C.Deallocate(this);
-}
-
 const ObjCObjectPointerType *Type::getAsObjCQualifiedIdType() const {
   // There is no sugar for ObjCQualifiedIdType's, just return the canonical
   // type pointer if it is the right class.
@@ -434,9 +414,14 @@ bool Type::isIntegerType() const {
     // FIXME: In C++, enum types are never integer types.
     if (TT->getDecl()->isEnum() && TT->getDecl()->isDefinition())
       return true;
+  return false;
+}
+
+bool Type::hasIntegerRepresentation() const {
   if (const VectorType *VT = dyn_cast<VectorType>(CanonicalType))
     return VT->getElementType()->isIntegerType();
-  return false;
+  else
+    return isIntegerType();
 }
 
 /// \brief Determine whether this type is an integral type.
@@ -475,10 +460,13 @@ bool Type::isIntegralOrEnumerationType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->getKind() >= BuiltinType::Bool &&
            BT->getKind() <= BuiltinType::Int128;
-  
-  if (isa<EnumType>(CanonicalType))
-    return true;
-  
+
+  // Check for a complete enum type; incomplete enum types are not properly an
+  // enumeration type in the sense required here.
+  if (const TagType *TT = dyn_cast<TagType>(CanonicalType))
+    if (TT->getDecl()->isEnum() && TT->getDecl()->isDefinition())
+      return true;
+
   return false;  
 }
 
@@ -523,8 +511,7 @@ bool Type::isAnyCharacterType() const {
 
 /// isSignedIntegerType - Return true if this is an integer type that is
 /// signed, according to C99 6.2.5p4 [char, signed char, short, int, long..],
-/// an enum decl which has a signed representation, or a vector of signed
-/// integer element type.
+/// an enum decl which has a signed representation
 bool Type::isSignedIntegerType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType)) {
     return BT->getKind() >= BuiltinType::Char_S &&
@@ -534,15 +521,19 @@ bool Type::isSignedIntegerType() const {
   if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
     return ET->getDecl()->getIntegerType()->isSignedIntegerType();
 
+  return false;
+}
+
+bool Type::hasSignedIntegerRepresentation() const {
   if (const VectorType *VT = dyn_cast<VectorType>(CanonicalType))
     return VT->getElementType()->isSignedIntegerType();
-  return false;
+  else
+    return isSignedIntegerType();
 }
 
 /// isUnsignedIntegerType - Return true if this is an integer type that is
 /// unsigned, according to C99 6.2.5p6 [which returns true for _Bool], an enum
-/// decl which has an unsigned representation, or a vector of unsigned integer
-/// element type.
+/// decl which has an unsigned representation
 bool Type::isUnsignedIntegerType() const {
   if (const BuiltinType *BT = dyn_cast<BuiltinType>(CanonicalType)) {
     return BT->getKind() >= BuiltinType::Bool &&
@@ -552,9 +543,14 @@ bool Type::isUnsignedIntegerType() const {
   if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
     return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
 
+  return false;
+}
+
+bool Type::hasUnsignedIntegerRepresentation() const {
   if (const VectorType *VT = dyn_cast<VectorType>(CanonicalType))
     return VT->getElementType()->isUnsignedIntegerType();
-  return false;
+  else
+    return isUnsignedIntegerType();
 }
 
 bool Type::isFloatingType() const {
@@ -671,10 +667,11 @@ bool Type::isIncompleteType() const {
     // An array of unknown size is an incomplete type (C99 6.2.5p22).
     return true;
   case ObjCObject:
-    return cast<ObjCObjectType>(this)->getBaseType()->isIncompleteType();
+    return cast<ObjCObjectType>(CanonicalType)->getBaseType()
+                                                         ->isIncompleteType();
   case ObjCInterface:
     // ObjC interfaces are incomplete if they are @class, not @interface.
-    return cast<ObjCInterfaceType>(this)->getDecl()->isForwardDecl();
+    return cast<ObjCInterfaceType>(CanonicalType)->getDecl()->isForwardDecl();
   }
 }
 
@@ -894,15 +891,6 @@ ElaboratedType::~ElaboratedType() {}
 DependentNameType::~DependentNameType() {}
 DependentTemplateSpecializationType::~DependentTemplateSpecializationType() {}
 
-void DependentTemplateSpecializationType::Destroy(ASTContext &C) {
-  for (unsigned Arg = 0; Arg < NumArgs; ++Arg) {
-    // FIXME: Not all expressions get cloned, so we can't yet perform
-    // this destruction.
-    //    if (Expr *E = getArg(Arg).getAsExpr())
-    //      E->Destroy(C);
-  }
-}
-
 DependentTemplateSpecializationType::DependentTemplateSpecializationType(
                          ElaboratedTypeKeyword Keyword,
                          NestedNameSpecifier *NNS, const IdentifierInfo *Name,
@@ -1017,6 +1005,7 @@ llvm::StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   case CC_X86StdCall: return "stdcall";
   case CC_X86FastCall: return "fastcall";
   case CC_X86ThisCall: return "thiscall";
+  case CC_X86Pascal: return "pascal";
   }
 }
 
@@ -1108,7 +1097,30 @@ void DependentDecltypeType::Profile(llvm::FoldingSetNodeID &ID,
 
 TagType::TagType(TypeClass TC, const TagDecl *D, QualType can)
   : Type(TC, can, D->isDependentType()),
-    decl(const_cast<TagDecl*>(D), 0) {}
+    decl(const_cast<TagDecl*>(D)) {}
+
+static TagDecl *getInterestingTagDecl(TagDecl *decl) {
+  for (TagDecl::redecl_iterator I = decl->redecls_begin(),
+                                E = decl->redecls_end();
+       I != E; ++I) {
+    if (I->isDefinition() || I->isBeingDefined())
+      return *I;
+  }
+  // If there's no definition (not even in progress), return what we have.
+  return decl;
+}
+
+TagDecl *TagType::getDecl() const {
+  return getInterestingTagDecl(decl);
+}
+
+bool TagType::isBeingDefined() const {
+  return getDecl()->isBeingDefined();
+}
+
+CXXRecordDecl *InjectedClassNameType::getDecl() const {
+  return cast<CXXRecordDecl>(getInterestingTagDecl(Decl));
+}
 
 bool RecordType::classof(const TagType *TT) {
   return isa<RecordDecl>(TT->getDecl());
@@ -1196,15 +1208,6 @@ TemplateSpecializationType(TemplateName T,
     new (&TemplateArgs[Arg]) TemplateArgument(Args[Arg]);
 }
 
-void TemplateSpecializationType::Destroy(ASTContext& C) {
-  for (unsigned Arg = 0; Arg < NumArgs; ++Arg) {
-    // FIXME: Not all expressions get cloned, so we can't yet perform
-    // this destruction.
-    //    if (Expr *E = getArg(Arg).getAsExpr())
-    //      E->Destroy(C);
-  }
-}
-
 void
 TemplateSpecializationType::Profile(llvm::FoldingSetNodeID &ID,
                                     TemplateName T,
diff --git a/lib/AST/TypeLoc.cpp b/lib/AST/TypeLoc.cpp
index 4893b384dd10..66578fb3dc49 100644
--- a/lib/AST/TypeLoc.cpp
+++ b/lib/AST/TypeLoc.cpp
@@ -74,20 +74,6 @@ TypeLoc TypeLoc::getNextTypeLocImpl(TypeLoc TL) {
   return NextLoc().Visit(TL);
 }
 
-namespace {
-  struct TypeLocInitializer : public TypeLocVisitor<TypeLocInitializer> {
-    SourceLocation Loc;
-    TypeLocInitializer(SourceLocation Loc) : Loc(Loc) {}
-  
-#define ABSTRACT_TYPELOC(CLASS, PARENT)
-#define TYPELOC(CLASS, PARENT) \
-    void Visit##CLASS##TypeLoc(CLASS##TypeLoc TyLoc) { \
-      TyLoc.initializeLocal(Loc); \
-    }
-#include "clang/AST/TypeLocNodes.def"
-  };
-}
-
 /// \brief Initializes a type location, and all of its children
 /// recursively, as if the entire tree had been written in the
 /// given location.
diff --git a/lib/AST/TypePrinter.cpp b/lib/AST/TypePrinter.cpp
index a08ee1ae695d..d3a6b645537c 100644
--- a/lib/AST/TypePrinter.cpp
+++ b/lib/AST/TypePrinter.cpp
@@ -299,6 +299,9 @@ void TypePrinter::PrintFunctionProto(const FunctionProtoType *T,
   case CC_X86ThisCall:
     S += " __attribute__((thiscall))";
     break;
+  case CC_X86Pascal:
+    S += " __attribute__((pascal))";
+    break;
   }
   if (Info.getNoReturn())
     S += " __attribute__((noreturn))";
@@ -430,9 +433,10 @@ void TypePrinter::PrintTag(TagDecl *D, std::string &InnerString) {
     Buffer += ' ';
   }
 
+  // Compute the full nested-name-specifier for this type.
+  // In C, this will always be empty except when the type
+  // being printed is anonymous within other Record.
   if (!Policy.SuppressScope)
-    // Compute the full nested-name-specifier for this type. In C,
-    // this will always be empty.
     AppendScope(D->getDeclContext(), Buffer);
 
   if (const IdentifierInfo *II = D->getIdentifier())
@@ -463,7 +467,6 @@ void TypePrinter::PrintTag(TagDecl *D, std::string &InnerString) {
     }
     
     OS << '>';
-    OS.flush();
   }
 
   // If this is a class template specialization, print the template
diff --git a/lib/Analysis/AnalysisContext.cpp b/lib/Analysis/AnalysisContext.cpp
index 06d8aec3910e..bf9f96719001 100644
--- a/lib/Analysis/AnalysisContext.cpp
+++ b/lib/Analysis/AnalysisContext.cpp
@@ -18,6 +18,7 @@
 #include "clang/AST/ParentMap.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Analysis/Analyses/LiveVariables.h"
+#include "clang/Analysis/Analyses/PseudoConstantAnalysis.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/CFG.h"
 #include "clang/Analysis/Support/BumpVector.h"
@@ -54,8 +55,11 @@ const ImplicitParamDecl *AnalysisContext::getSelfDecl() const {
 }
 
 CFG *AnalysisContext::getCFG() {
+  if (UseUnoptimizedCFG)
+    return getUnoptimizedCFG();
+
   if (!builtCFG) {
-    cfg = CFG::buildCFG(D, getBody(), &D->getASTContext(), AddEHEdges);
+    cfg = CFG::buildCFG(D, getBody(), &D->getASTContext(), true, AddEHEdges);
     // Even when the cfg is not successfully built, we don't
     // want to try building it again.
     builtCFG = true;
@@ -63,12 +67,29 @@ CFG *AnalysisContext::getCFG() {
   return cfg;
 }
 
+CFG *AnalysisContext::getUnoptimizedCFG() {
+  if (!builtCompleteCFG) {
+    completeCFG = CFG::buildCFG(D, getBody(), &D->getASTContext(),
+                                false, AddEHEdges);
+    // Even when the cfg is not successfully built, we don't
+    // want to try building it again.
+    builtCompleteCFG = true;
+  }
+  return completeCFG;
+}
+
 ParentMap &AnalysisContext::getParentMap() {
   if (!PM)
     PM = new ParentMap(getBody());
   return *PM;
 }
 
+PseudoConstantAnalysis *AnalysisContext::getPseudoConstantAnalysis() {
+  if (!PCA)
+    PCA = new PseudoConstantAnalysis(getBody());
+  return PCA;
+}
+
 LiveVariables *AnalysisContext::getLiveVariables() {
   if (!liveness) {
     CFG *c = getCFG();
@@ -83,10 +104,25 @@ LiveVariables *AnalysisContext::getLiveVariables() {
   return liveness;
 }
 
-AnalysisContext *AnalysisContextManager::getContext(const Decl *D) {
+LiveVariables *AnalysisContext::getRelaxedLiveVariables() {
+  if (!relaxedLiveness) {
+    CFG *c = getCFG();
+    if (!c)
+      return 0;
+
+    relaxedLiveness = new LiveVariables(*this, false);
+    relaxedLiveness->runOnCFG(*c);
+    relaxedLiveness->runOnAllBlocks(*c, 0, true);
+  }
+
+  return relaxedLiveness;
+}
+
+AnalysisContext *AnalysisContextManager::getContext(const Decl *D,
+                                                    idx::TranslationUnit *TU) {
   AnalysisContext *&AC = Contexts[D];
   if (!AC)
-    AC = new AnalysisContext(D);
+    AC = new AnalysisContext(D, TU, UseUnoptimizedCFG);
 
   return AC;
 }
@@ -296,8 +332,11 @@ AnalysisContext::getReferencedBlockVars(const BlockDecl *BD) {
 
 AnalysisContext::~AnalysisContext() {
   delete cfg;
+  delete completeCFG;
   delete liveness;
+  delete relaxedLiveness;
   delete PM;
+  delete PCA;
   delete ReferencedBlockVars;
 }
 
diff --git a/lib/Analysis/CFG.cpp b/lib/Analysis/CFG.cpp
index 08543aacba5c..c97b9165bca2 100644
--- a/lib/Analysis/CFG.cpp
+++ b/lib/Analysis/CFG.cpp
@@ -35,7 +35,7 @@ static SourceLocation GetEndLoc(Decl* D) {
 
   return D->getLocation();
 }
-  
+
 class AddStmtChoice {
 public:
   enum Kind { NotAlwaysAdd = 0,
@@ -99,7 +99,8 @@ public:
                           TryTerminatedBlock(NULL) {}
 
   // buildCFG - Used by external clients to construct the CFG.
-  CFG* buildCFG(const Decl *D, Stmt *Statement, ASTContext *C, bool AddEHEdges,
+  CFG* buildCFG(const Decl *D, Stmt *Statement, ASTContext *C,
+                bool pruneTriviallyFalseEdges, bool AddEHEdges,
                 bool AddScopes);
 
 private:
@@ -174,12 +175,12 @@ private:
   CFGBlock *addStmt(Stmt *S) {
     return Visit(S, AddStmtChoice::AlwaysAdd);
   }
-  
+
   void AppendStmt(CFGBlock *B, Stmt *S,
                   AddStmtChoice asc = AddStmtChoice::AlwaysAdd) {
     B->appendStmt(S, cfg->getBumpVectorContext(), asc.asLValue());
   }
-  
+
   void AddSuccessor(CFGBlock *B, CFGBlock *S) {
     B->addSuccessor(S, cfg->getBumpVectorContext());
   }
@@ -206,6 +207,9 @@ private:
   /// TryEvaluateBool - Try and evaluate the Stmt and return 0 or 1
   /// if we can evaluate to a known value, otherwise return -1.
   TryResult TryEvaluateBool(Expr *S) {
+    if (!PruneTriviallyFalseEdges)
+      return TryResult();
+
     Expr::EvalResult Result;
     if (!S->isTypeDependent() && !S->isValueDependent() &&
         S->Evaluate(Result, *Context) && Result.Val.isInt())
@@ -216,6 +220,9 @@ private:
 
   bool badCFG;
 
+  // True iff trivially false edges should be pruned from the CFG.
+  bool PruneTriviallyFalseEdges;
+
   // True iff EH edges on CallExprs should be added to the CFG.
   bool AddEHEdges;
 
@@ -243,8 +250,12 @@ static VariableArrayType* FindVA(Type* t) {
 ///  transferred to the caller.  If CFG construction fails, this method returns
 ///  NULL.
 CFG* CFGBuilder::buildCFG(const Decl *D, Stmt* Statement, ASTContext* C,
+                          bool pruneTriviallyFalseEdges,
                           bool addehedges, bool AddScopes) {
+
   AddEHEdges = addehedges;
+  PruneTriviallyFalseEdges = pruneTriviallyFalseEdges;
+
   Context = C;
   assert(cfg.get());
   if (!Statement)
@@ -359,6 +370,7 @@ tryAgain:
       return VisitBreakStmt(cast<BreakStmt>(S));
 
     case Stmt::CallExprClass:
+    case Stmt::CXXOperatorCallExprClass:
       return VisitCallExpr(cast<CallExpr>(S), asc);
 
     case Stmt::CaseStmtClass:
@@ -379,15 +391,21 @@ tryAgain:
     case Stmt::CXXCatchStmtClass:
       return VisitCXXCatchStmt(cast<CXXCatchStmt>(S));
 
+    case Stmt::CXXExprWithTemporariesClass: {
+      // FIXME: Handle temporaries.  For now, just visit the subexpression
+      // so we don't artificially create extra blocks.
+      return Visit(cast<CXXExprWithTemporaries>(S)->getSubExpr(), asc);
+    }
+
     case Stmt::CXXMemberCallExprClass:
       return VisitCXXMemberCallExpr(cast<CXXMemberCallExpr>(S), asc);
 
     case Stmt::CXXThrowExprClass:
       return VisitCXXThrowExpr(cast<CXXThrowExpr>(S));
-      
+
     case Stmt::CXXTryStmtClass:
       return VisitCXXTryStmt(cast<CXXTryStmt>(S));
-      
+
     case Stmt::DeclStmtClass:
       return VisitDeclStmt(cast<DeclStmt>(S));
 
@@ -515,15 +533,15 @@ CFGBlock *CFGBuilder::VisitBinaryOperator(BinaryOperator *B,
 
     // See if this is a known constant.
     TryResult KnownVal = TryEvaluateBool(B->getLHS());
-    if (KnownVal.isKnown() && (B->getOpcode() == BinaryOperator::LOr))
+    if (KnownVal.isKnown() && (B->getOpcode() == BO_LOr))
       KnownVal.negate();
 
     // Now link the LHSBlock with RHSBlock.
-    if (B->getOpcode() == BinaryOperator::LOr) {
+    if (B->getOpcode() == BO_LOr) {
       AddSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
       AddSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
     } else {
-      assert(B->getOpcode() == BinaryOperator::LAnd);
+      assert(B->getOpcode() == BO_LAnd);
       AddSuccessor(LHSBlock, KnownVal.isFalse() ? NULL : RHSBlock);
       AddSuccessor(LHSBlock, KnownVal.isTrue() ? NULL : ConfluenceBlock);
     }
@@ -532,7 +550,7 @@ CFGBlock *CFGBuilder::VisitBinaryOperator(BinaryOperator *B,
     Block = LHSBlock;
     return addStmt(B->getLHS());
   }
-  else if (B->getOpcode() == BinaryOperator::Comma) { // ,
+  else if (B->getOpcode() == BO_Comma) { // ,
     autoCreateBlock();
     AppendStmt(Block, B, asc);
     addStmt(B->getRHS());
@@ -543,7 +561,7 @@ CFGBlock *CFGBuilder::VisitBinaryOperator(BinaryOperator *B,
       autoCreateBlock();
       AppendStmt(Block, B, asc);
     }
-    
+
     Visit(B->getRHS());
     return Visit(B->getLHS(), AddStmtChoice::AsLValueNotAlwaysAdd);
   }
@@ -586,7 +604,7 @@ static bool CanThrow(Expr *E) {
     Ty = Ty->getAs<PointerType>()->getPointeeType();
   else if (Ty->isBlockPointerType())
     Ty = Ty->getAs<BlockPointerType>()->getPointeeType();
-    
+
   const FunctionType *FT = Ty->getAs<FunctionType>();
   if (FT) {
     if (const FunctionProtoType *Proto = dyn_cast<FunctionProtoType>(FT))
@@ -691,7 +709,10 @@ CFGBlock* CFGBuilder::VisitCompoundStmt(CompoundStmt* C) {
 
   for (CompoundStmt::reverse_body_iterator I=C->body_rbegin(), E=C->body_rend();
        I != E; ++I ) {
-    LastBlock = addStmt(*I);
+    // If we hit a segment of code just containing ';' (NullStmts), we can
+    // get a null block back.  In such cases, just use the LastBlock
+    if (CFGBlock *newBlock = addStmt(*I))
+      LastBlock = newBlock;
 
     if (badCFG)
       return NULL;
@@ -901,7 +922,7 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
   // new blocks as the condition may contain control-flow.  Any newly created
   // blocks will be pointed to be "Block".
   Block = addStmt(I->getCond());
-  
+
   // Finally, if the IfStmt contains a condition variable, add both the IfStmt
   // and the condition variable initialization to the CFG.
   if (VarDecl *VD = I->getConditionVariable()) {
@@ -911,7 +932,7 @@ CFGBlock* CFGBuilder::VisitIfStmt(IfStmt* I) {
       addStmt(Init);
     }
   }
-  
+
   return Block;
 }
 
@@ -1029,7 +1050,7 @@ CFGBlock* CFGBuilder::VisitForStmt(ForStmt* F) {
         assert(Block == EntryConditionBlock);
       }
     }
-    
+
     if (Block) {
       if (!FinishBlock(EntryConditionBlock))
         return 0;
@@ -1277,7 +1298,7 @@ CFGBlock* CFGBuilder::VisitWhileStmt(WhileStmt* W) {
     Block = ExitConditionBlock;
     EntryConditionBlock = addStmt(C);
     assert(Block == EntryConditionBlock);
-    
+
     // If this block contains a condition variable, add both the condition
     // variable and initializer to the CFG.
     if (VarDecl *VD = W->getConditionVariable()) {
@@ -1389,7 +1410,7 @@ CFGBlock* CFGBuilder::VisitCXXThrowExpr(CXXThrowExpr* T) {
   if (TryTerminatedBlock)
     // The current try statement is the only successor.
     AddSuccessor(Block, TryTerminatedBlock);
-  else 
+  else
     // otherwise the Exit block is the only successor.
     AddSuccessor(Block, &cfg->getExit());
 
@@ -1465,18 +1486,22 @@ CFGBlock *CFGBuilder::VisitDoStmt(DoStmt* D) {
         return 0;
     }
 
-    // Add an intermediate block between the BodyBlock and the
-    // ExitConditionBlock to represent the "loop back" transition.  Create an
-    // empty block to represent the transition block for looping back to the
-    // head of the loop.
-    // FIXME: Can we do this more efficiently without adding another block?
-    Block = NULL;
-    Succ = BodyBlock;
-    CFGBlock *LoopBackBlock = createBlock();
-    LoopBackBlock->setLoopTarget(D);
-
-    // Add the loop body entry as a successor to the condition.
-    AddSuccessor(ExitConditionBlock, KnownVal.isFalse() ? NULL : LoopBackBlock);
+    if (!KnownVal.isFalse()) {
+      // Add an intermediate block between the BodyBlock and the
+      // ExitConditionBlock to represent the "loop back" transition.  Create an
+      // empty block to represent the transition block for looping back to the
+      // head of the loop.
+      // FIXME: Can we do this more efficiently without adding another block?
+      Block = NULL;
+      Succ = BodyBlock;
+      CFGBlock *LoopBackBlock = createBlock();
+      LoopBackBlock->setLoopTarget(D);
+
+      // Add the loop body entry as a successor to the condition.
+      AddSuccessor(ExitConditionBlock, LoopBackBlock);
+    }
+    else
+      AddSuccessor(ExitConditionBlock, NULL);
   }
 
   // Link up the condition block with the code that follows the loop.
@@ -1589,7 +1614,7 @@ CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
   assert(Terminator->getCond() && "switch condition must be non-NULL");
   Block = SwitchTerminatedBlock;
   Block = addStmt(Terminator->getCond());
-  
+
   // Finally, if the SwitchStmt contains a condition variable, add both the
   // SwitchStmt and the condition variable initialization to the CFG.
   if (VarDecl *VD = Terminator->getConditionVariable()) {
@@ -1599,16 +1624,37 @@ CFGBlock* CFGBuilder::VisitSwitchStmt(SwitchStmt* Terminator) {
       addStmt(Init);
     }
   }
-  
+
   return Block;
 }
 
 CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
   // CaseStmts are essentially labels, so they are the first statement in a
   // block.
+  CFGBlock *TopBlock = 0, *LastBlock = 0;
+  
+  if (Stmt *Sub = CS->getSubStmt()) {
+    // For deeply nested chains of CaseStmts, instead of doing a recursion
+    // (which can blow out the stack), manually unroll and create blocks
+    // along the way.
+    while (isa<CaseStmt>(Sub)) {
+      CFGBlock *CurrentBlock = createBlock(false);
+      CurrentBlock->setLabel(CS);
+
+      if (TopBlock)
+        AddSuccessor(LastBlock, CurrentBlock);
+      else
+        TopBlock = CurrentBlock;
+
+      AddSuccessor(SwitchTerminatedBlock, CurrentBlock);
+      LastBlock = CurrentBlock;
+
+      CS = cast<CaseStmt>(Sub);
+      Sub = CS->getSubStmt();
+    }
 
-  if (CS->getSubStmt())
-    addStmt(CS->getSubStmt());
+    addStmt(Sub);
+  }
 
   CFGBlock* CaseBlock = Block;
   if (!CaseBlock)
@@ -1629,10 +1675,16 @@ CFGBlock* CFGBuilder::VisitCaseStmt(CaseStmt* CS) {
   // We set Block to NULL to allow lazy creation of a new block (if necessary)
   Block = NULL;
 
-  // This block is now the implicit successor of other blocks.
-  Succ = CaseBlock;
+  if (TopBlock) {
+    AddSuccessor(LastBlock, CaseBlock);
+    Succ = TopBlock;
+  }
+  else {
+    // This block is now the implicit successor of other blocks.
+    Succ = CaseBlock;
+  }
 
-  return CaseBlock;
+  return Succ;
 }
 
 CFGBlock* CFGBuilder::VisitDefaultStmt(DefaultStmt* Terminator) {
@@ -1742,9 +1794,9 @@ CFGBlock* CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt* CS) {
   return CatchBlock;
 }
 
-CFGBlock *CFGBuilder::VisitCXXMemberCallExpr(CXXMemberCallExpr *C, 
+CFGBlock *CFGBuilder::VisitCXXMemberCallExpr(CXXMemberCallExpr *C,
                                              AddStmtChoice asc) {
-  AddStmtChoice::Kind K = asc.asLValue() ? AddStmtChoice::AlwaysAddAsLValue 
+  AddStmtChoice::Kind K = asc.asLValue() ? AddStmtChoice::AlwaysAddAsLValue
                                          : AddStmtChoice::AlwaysAdd;
   autoCreateBlock();
   AppendStmt(Block, C, AddStmtChoice(K));
@@ -1795,9 +1847,11 @@ CFGBlock* CFG::createBlock() {
 /// buildCFG - Constructs a CFG from an AST.  Ownership of the returned
 ///  CFG is returned to the caller.
 CFG* CFG::buildCFG(const Decl *D, Stmt* Statement, ASTContext *C,
+                   bool PruneTriviallyFalse,
                    bool AddEHEdges, bool AddScopes) {
   CFGBuilder Builder;
-  return Builder.buildCFG(D, Statement, C, AddEHEdges, AddScopes);
+  return Builder.buildCFG(D, Statement, C, PruneTriviallyFalse,
+                          AddEHEdges, AddScopes);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1814,10 +1868,10 @@ static void FindSubExprAssignments(Stmt *S,
     return;
 
   for (Stmt::child_iterator I=S->child_begin(), E=S->child_end(); I!=E; ++I) {
-    Stmt *child = *I;    
+    Stmt *child = *I;
     if (!child)
       continue;
-    
+
     if (BinaryOperator* B = dyn_cast<BinaryOperator>(child))
       if (B->isAssignmentOp()) Set.insert(B);
 
@@ -2037,10 +2091,10 @@ public:
     B->getLHS()->printPretty(OS, Helper, Policy);
 
     switch (B->getOpcode()) {
-      case BinaryOperator::LOr:
+      case BO_LOr:
         OS << " || ...";
         return;
-      case BinaryOperator::LAnd:
+      case BO_LAnd:
         OS << " && ...";
         return;
       default:
@@ -2057,7 +2111,6 @@ public:
 
 static void print_stmt(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
                        const CFGElement &E) {
-  Stmt *Terminator = E;
 
   if (E.asStartScope()) {
     OS << "start scope\n";
@@ -2068,9 +2121,11 @@ static void print_stmt(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
     return;
   }
 
+  Stmt *S = E;
+  
   if (Helper) {
     // special printing for statement-expressions.
-    if (StmtExpr* SE = dyn_cast<StmtExpr>(Terminator)) {
+    if (StmtExpr* SE = dyn_cast<StmtExpr>(S)) {
       CompoundStmt* Sub = SE->getSubStmt();
 
       if (Sub->child_begin() != Sub->child_end()) {
@@ -2082,8 +2137,8 @@ static void print_stmt(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
     }
 
     // special printing for comma expressions.
-    if (BinaryOperator* B = dyn_cast<BinaryOperator>(Terminator)) {
-      if (B->getOpcode() == BinaryOperator::Comma) {
+    if (BinaryOperator* B = dyn_cast<BinaryOperator>(S)) {
+      if (B->getOpcode() == BO_Comma) {
         OS << "... , ";
         Helper->handledStmt(B->getRHS(),OS);
         OS << '\n';
@@ -2092,10 +2147,19 @@ static void print_stmt(llvm::raw_ostream &OS, StmtPrinterHelper* Helper,
     }
   }
 
-  Terminator->printPretty(OS, Helper, PrintingPolicy(Helper->getLangOpts()));
+  S->printPretty(OS, Helper, PrintingPolicy(Helper->getLangOpts()));
+  
+  if (isa<CXXOperatorCallExpr>(S)) {
+    OS << " (OperatorCall)";    
+  }
+  else if (isa<CXXBindTemporaryExpr>(S)) {
+    OS << " (BindTemporary)";    
+  }
+
 
   // Expressions need a newline.
-  if (isa<Expr>(Terminator)) OS << '\n';
+  if (isa<Expr>(S))
+    OS << '\n';
 }
 
 static void print_block(llvm::raw_ostream& OS, const CFG* cfg,
diff --git a/lib/Analysis/CFGStmtMap.cpp b/lib/Analysis/CFGStmtMap.cpp
new file mode 100644
index 000000000000..965eca1b3c90
--- /dev/null
+++ b/lib/Analysis/CFGStmtMap.cpp
@@ -0,0 +1,88 @@
+//===--- CFGStmtMap.h - Map from Stmt* to CFGBlock* -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the CFGStmtMap class, which defines a mapping from
+//  Stmt* to CFGBlock*
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/DenseMap.h"
+#include "clang/AST/ParentMap.h"
+#include "clang/Analysis/CFG.h"
+#include "clang/Analysis/CFGStmtMap.h"
+
+using namespace clang;
+
+typedef llvm::DenseMap<Stmt*,CFGBlock*> SMap;
+static SMap *AsMap(void *m) { return (SMap*) m; }
+
+CFGStmtMap::~CFGStmtMap() { delete AsMap(M); }
+
+CFGBlock *CFGStmtMap::getBlock(Stmt *S) {  
+  SMap *SM = AsMap(M);
+  Stmt *X = S;
+
+  // If 'S' isn't in the map, walk the ParentMap to see if one of its ancestors
+  // is in the map.
+  while (X) {
+    SMap::iterator I = SM->find(X);
+    if (I != SM->end()) {
+      CFGBlock *B = I->second;
+      // Memoize this lookup.
+      if (X != S)
+        (*SM)[X] = B;
+      return B;
+    }
+
+    X = PM->getParentIgnoreParens(X);
+  }
+  
+  return 0;
+}
+
+static void Accumulate(SMap &SM, CFGBlock *B) {
+  // First walk the block-level expressions.
+  for (CFGBlock::iterator I = B->begin(), E = B->end(); I != E; ++I) {
+    const CFGElement &CE = *I;
+    if (Stmt *S = CE.getStmt()) {
+      CFGBlock *&Entry = SM[S];
+      // If 'Entry' is already initialized (e.g., a terminator was already),
+      // skip.
+      if (Entry)
+        continue;
+      
+      Entry = B;
+    }
+  }
+  
+  // Look at the label of the block.
+  if (Stmt *Label = B->getLabel())
+    SM[Label] = B;
+
+  // Finally, look at the terminator.  If the terminator was already added
+  // because it is a block-level expression in another block, overwrite
+  // that mapping.
+  if (Stmt *Term = B->getTerminator())
+    SM[Term] = B;
+}
+
+CFGStmtMap *CFGStmtMap::Build(CFG *C, ParentMap *PM) {
+  if (!C || !PM)
+    return 0;
+
+  SMap *SM = new SMap();
+
+  // Walk all blocks, accumulating the block-level expressions, labels,
+  // and terminators.  
+  for (CFG::iterator I = C->begin(), E = C->end(); I != E; ++I)
+    Accumulate(*SM, *I);
+
+  return new CFGStmtMap(PM, SM);
+}
+
diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt
index f2916c2068e1..850e9b468100 100644
--- a/lib/Analysis/CMakeLists.txt
+++ b/lib/Analysis/CMakeLists.txt
@@ -3,9 +3,13 @@ set(LLVM_NO_RTTI 1)
 add_clang_library(clangAnalysis
   AnalysisContext.cpp
   CFG.cpp
+  CFGStmtMap.cpp
+  FormatString.cpp
   LiveVariables.cpp
   PrintfFormatString.cpp
+  PseudoConstantAnalysis.cpp
   ReachableCode.cpp
+  ScanfFormatString.cpp
   UninitializedValues.cpp
   )
 
diff --git a/lib/Analysis/FormatString.cpp b/lib/Analysis/FormatString.cpp
new file mode 100644
index 000000000000..388b9d34a238
--- /dev/null
+++ b/lib/Analysis/FormatString.cpp
@@ -0,0 +1,474 @@
+// FormatString.cpp - Common stuff for handling printf/scanf formats -*- C++ -*-
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Shared details for processing format strings of printf and scanf
+// (and friends).
+//
+//===----------------------------------------------------------------------===//
+
+#include "FormatStringParsing.h"
+
+using clang::analyze_format_string::ArgTypeResult;
+using clang::analyze_format_string::FormatStringHandler;
+using clang::analyze_format_string::FormatSpecifier;
+using clang::analyze_format_string::LengthModifier;
+using clang::analyze_format_string::OptionalAmount;
+using clang::analyze_format_string::PositionContext;
+using clang::analyze_format_string::ConversionSpecifier;
+using namespace clang;
+
+// Key function to FormatStringHandler.
+FormatStringHandler::~FormatStringHandler() {}
+
+//===----------------------------------------------------------------------===//
+// Functions for parsing format strings components in both printf and
+// scanf format strings.
+//===----------------------------------------------------------------------===//
+
+OptionalAmount
+clang::analyze_format_string::ParseAmount(const char *&Beg, const char *E) {
+  const char *I = Beg;
+  UpdateOnReturn <const char*> UpdateBeg(Beg, I);
+
+  unsigned accumulator = 0;
+  bool hasDigits = false;
+
+  for ( ; I != E; ++I) {
+    char c = *I;
+    if (c >= '0' && c <= '9') {
+      hasDigits = true;
+      accumulator = (accumulator * 10) + (c - '0');
+      continue;
+    }
+
+    if (hasDigits)
+      return OptionalAmount(OptionalAmount::Constant, accumulator, Beg, I - Beg,
+          false);
+
+    break;
+  }
+
+  return OptionalAmount();
+}
+
+OptionalAmount
+clang::analyze_format_string::ParseNonPositionAmount(const char *&Beg,
+                                                     const char *E,
+                                                     unsigned &argIndex) {
+  if (*Beg == '*') {
+    ++Beg;
+    return OptionalAmount(OptionalAmount::Arg, argIndex++, Beg, 0, false);
+  }
+
+  return ParseAmount(Beg, E);
+}
+
+OptionalAmount
+clang::analyze_format_string::ParsePositionAmount(FormatStringHandler &H,
+                                                  const char *Start,
+                                                  const char *&Beg,
+                                                  const char *E,
+                                                  PositionContext p) {
+  if (*Beg == '*') {
+    const char *I = Beg + 1;
+    const OptionalAmount &Amt = ParseAmount(I, E);
+
+    if (Amt.getHowSpecified() == OptionalAmount::NotSpecified) {
+      H.HandleInvalidPosition(Beg, I - Beg, p);
+      return OptionalAmount(false);
+    }
+
+    if (I == E) {
+      // No more characters left?
+      H.HandleIncompleteSpecifier(Start, E - Start);
+      return OptionalAmount(false);
+    }
+
+    assert(Amt.getHowSpecified() == OptionalAmount::Constant);
+
+    if (*I == '$') {
+      // Handle positional arguments
+
+      // Special case: '*0$', since this is an easy mistake.
+      if (Amt.getConstantAmount() == 0) {
+        H.HandleZeroPosition(Beg, I - Beg + 1);
+        return OptionalAmount(false);
+      }
+
+      const char *Tmp = Beg;
+      Beg = ++I;
+
+      return OptionalAmount(OptionalAmount::Arg, Amt.getConstantAmount() - 1,
+                            Tmp, 0, true);
+    }
+
+    H.HandleInvalidPosition(Beg, I - Beg, p);
+    return OptionalAmount(false);
+  }
+
+  return ParseAmount(Beg, E);
+}
+
+
+bool
+clang::analyze_format_string::ParseFieldWidth(FormatStringHandler &H,
+                                              FormatSpecifier &CS,
+                                              const char *Start,
+                                              const char *&Beg, const char *E,
+                                              unsigned *argIndex) {
+  // FIXME: Support negative field widths.
+  if (argIndex) {
+    CS.setFieldWidth(ParseNonPositionAmount(Beg, E, *argIndex));
+  }
+  else {
+    const OptionalAmount Amt =
+      ParsePositionAmount(H, Start, Beg, E,
+                          analyze_format_string::FieldWidthPos);
+
+    if (Amt.isInvalid())
+      return true;
+    CS.setFieldWidth(Amt);
+  }
+  return false;
+}
+
+bool
+clang::analyze_format_string::ParseArgPosition(FormatStringHandler &H,
+                                               FormatSpecifier &FS,
+                                               const char *Start,
+                                               const char *&Beg,
+                                               const char *E) {
+  const char *I = Beg;
+
+  const OptionalAmount &Amt = ParseAmount(I, E);
+
+  if (I == E) {
+    // No more characters left?
+    H.HandleIncompleteSpecifier(Start, E - Start);
+    return true;
+  }
+
+  if (Amt.getHowSpecified() == OptionalAmount::Constant && *(I++) == '$') {
+    // Special case: '%0$', since this is an easy mistake.
+    if (Amt.getConstantAmount() == 0) {
+      H.HandleZeroPosition(Start, I - Start);
+      return true;
+    }
+
+    FS.setArgIndex(Amt.getConstantAmount() - 1);
+    FS.setUsesPositionalArg();
+    // Update the caller's pointer if we decided to consume
+    // these characters.
+    Beg = I;
+    return false;
+  }
+
+  return false;
+}
+
+bool
+clang::analyze_format_string::ParseLengthModifier(FormatSpecifier &FS,
+                                                  const char *&I,
+                                                  const char *E) {
+  LengthModifier::Kind lmKind = LengthModifier::None;
+  const char *lmPosition = I;
+  switch (*I) {
+    default:
+      return false;
+    case 'h':
+      ++I;
+      lmKind = (I != E && *I == 'h') ?
+      ++I, LengthModifier::AsChar : LengthModifier::AsShort;
+      break;
+    case 'l':
+      ++I;
+      lmKind = (I != E && *I == 'l') ?
+      ++I, LengthModifier::AsLongLong : LengthModifier::AsLong;
+      break;
+    case 'j': lmKind = LengthModifier::AsIntMax;     ++I; break;
+    case 'z': lmKind = LengthModifier::AsSizeT;      ++I; break;
+    case 't': lmKind = LengthModifier::AsPtrDiff;    ++I; break;
+    case 'L': lmKind = LengthModifier::AsLongDouble; ++I; break;
+    case 'q': lmKind = LengthModifier::AsLongLong;   ++I; break;
+  }
+  LengthModifier lm(lmPosition, lmKind);
+  FS.setLengthModifier(lm);
+  return true;
+}
+
+//===----------------------------------------------------------------------===//
+// Methods on ArgTypeResult.
+//===----------------------------------------------------------------------===//
+
+bool ArgTypeResult::matchesType(ASTContext &C, QualType argTy) const {
+  switch (K) {
+    case InvalidTy:
+      assert(false && "ArgTypeResult must be valid");
+      return true;
+
+    case UnknownTy:
+      return true;
+
+    case SpecificTy: {
+      argTy = C.getCanonicalType(argTy).getUnqualifiedType();
+      if (T == argTy)
+        return true;
+      if (const BuiltinType *BT = argTy->getAs<BuiltinType>())
+        switch (BT->getKind()) {
+          default:
+            break;
+          case BuiltinType::Char_S:
+          case BuiltinType::SChar:
+            return T == C.UnsignedCharTy;
+          case BuiltinType::Char_U:
+          case BuiltinType::UChar:
+            return T == C.SignedCharTy;
+          case BuiltinType::Short:
+            return T == C.UnsignedShortTy;
+          case BuiltinType::UShort:
+            return T == C.ShortTy;
+          case BuiltinType::Int:
+            return T == C.UnsignedIntTy;
+          case BuiltinType::UInt:
+            return T == C.IntTy;
+          case BuiltinType::Long:
+            return T == C.UnsignedLongTy;
+          case BuiltinType::ULong:
+            return T == C.LongTy;
+          case BuiltinType::LongLong:
+            return T == C.UnsignedLongLongTy;
+          case BuiltinType::ULongLong:
+            return T == C.LongLongTy;
+        }
+      return false;
+    }
+
+    case CStrTy: {
+      const PointerType *PT = argTy->getAs<PointerType>();
+      if (!PT)
+        return false;
+      QualType pointeeTy = PT->getPointeeType();
+      if (const BuiltinType *BT = pointeeTy->getAs<BuiltinType>())
+        switch (BT->getKind()) {
+          case BuiltinType::Void:
+          case BuiltinType::Char_U:
+          case BuiltinType::UChar:
+          case BuiltinType::Char_S:
+          case BuiltinType::SChar:
+            return true;
+          default:
+            break;
+        }
+
+      return false;
+    }
+
+    case WCStrTy: {
+      const PointerType *PT = argTy->getAs<PointerType>();
+      if (!PT)
+        return false;
+      QualType pointeeTy =
+        C.getCanonicalType(PT->getPointeeType()).getUnqualifiedType();
+      return pointeeTy == C.getWCharType();
+    }
+    
+    case WIntTy: {
+      // Instead of doing a lookup for the definition of 'wint_t' (which
+      // is defined by the system headers) instead see if wchar_t and
+      // the argument type promote to the same type.
+      QualType PromoWChar =
+        C.getWCharType()->isPromotableIntegerType() 
+          ? C.getPromotedIntegerType(C.getWCharType()) : C.getWCharType();
+      QualType PromoArg = 
+        argTy->isPromotableIntegerType()
+          ? C.getPromotedIntegerType(argTy) : argTy;
+      
+      PromoWChar = C.getCanonicalType(PromoWChar).getUnqualifiedType();
+      PromoArg = C.getCanonicalType(PromoArg).getUnqualifiedType();
+      
+      return PromoWChar == PromoArg;
+    }
+
+    case CPointerTy:
+      return argTy->getAs<PointerType>() != NULL ||
+             argTy->getAs<ObjCObjectPointerType>() != NULL;
+
+    case ObjCPointerTy:
+      return argTy->getAs<ObjCObjectPointerType>() != NULL;
+  }
+
+  // FIXME: Should be unreachable, but Clang is currently emitting
+  // a warning.
+  return false;
+}
+
+QualType ArgTypeResult::getRepresentativeType(ASTContext &C) const {
+  switch (K) {
+    case InvalidTy:
+      assert(false && "No representative type for Invalid ArgTypeResult");
+      // Fall-through.
+    case UnknownTy:
+      return QualType();
+    case SpecificTy:
+      return T;
+    case CStrTy:
+      return C.getPointerType(C.CharTy);
+    case WCStrTy:
+      return C.getPointerType(C.getWCharType());
+    case ObjCPointerTy:
+      return C.ObjCBuiltinIdTy;
+    case CPointerTy:
+      return C.VoidPtrTy;
+    case WIntTy: {
+      QualType WC = C.getWCharType();
+      return WC->isPromotableIntegerType() ? C.getPromotedIntegerType(WC) : WC;
+    }
+  }
+
+  // FIXME: Should be unreachable, but Clang is currently emitting
+  // a warning.
+  return QualType();
+}
+
+//===----------------------------------------------------------------------===//
+// Methods on OptionalAmount.
+//===----------------------------------------------------------------------===//
+
+ArgTypeResult
+analyze_format_string::OptionalAmount::getArgType(ASTContext &Ctx) const {
+  return Ctx.IntTy;
+}
+
+//===----------------------------------------------------------------------===//
+// Methods on LengthModifier.
+//===----------------------------------------------------------------------===//
+
+const char *
+analyze_format_string::LengthModifier::toString() const {
+  switch (kind) {
+  case AsChar:
+    return "hh";
+  case AsShort:
+    return "h";
+  case AsLong: // or AsWideChar
+    return "l";
+  case AsLongLong:
+    return "ll";
+  case AsIntMax:
+    return "j";
+  case AsSizeT:
+    return "z";
+  case AsPtrDiff:
+    return "t";
+  case AsLongDouble:
+    return "L";
+  case None:
+    return "";
+  }
+  return NULL;
+}
+
+//===----------------------------------------------------------------------===//
+// Methods on OptionalAmount.
+//===----------------------------------------------------------------------===//
+
+void OptionalAmount::toString(llvm::raw_ostream &os) const {
+  switch (hs) {
+  case Invalid:
+  case NotSpecified:
+    return;
+  case Arg:
+    if (UsesDotPrefix)
+        os << ".";
+    if (usesPositionalArg())
+      os << "*" << getPositionalArgIndex() << "$";
+    else
+      os << "*";
+    break;
+  case Constant:
+    if (UsesDotPrefix)
+        os << ".";
+    os << amt;
+    break;
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Methods on ConversionSpecifier.
+//===----------------------------------------------------------------------===//
+
+bool FormatSpecifier::hasValidLengthModifier() const {
+  switch (LM.getKind()) {
+    case LengthModifier::None:
+      return true;
+      
+        // Handle most integer flags
+    case LengthModifier::AsChar:
+    case LengthModifier::AsShort:
+    case LengthModifier::AsLongLong:
+    case LengthModifier::AsIntMax:
+    case LengthModifier::AsSizeT:
+    case LengthModifier::AsPtrDiff:
+      switch (CS.getKind()) {
+        case ConversionSpecifier::dArg:
+        case ConversionSpecifier::iArg:
+        case ConversionSpecifier::oArg:
+        case ConversionSpecifier::uArg:
+        case ConversionSpecifier::xArg:
+        case ConversionSpecifier::XArg:
+        case ConversionSpecifier::nArg:
+          return true;
+        default:
+          return false;
+      }
+      
+        // Handle 'l' flag
+    case LengthModifier::AsLong:
+      switch (CS.getKind()) {
+        case ConversionSpecifier::dArg:
+        case ConversionSpecifier::iArg:
+        case ConversionSpecifier::oArg:
+        case ConversionSpecifier::uArg:
+        case ConversionSpecifier::xArg:
+        case ConversionSpecifier::XArg:
+        case ConversionSpecifier::aArg:
+        case ConversionSpecifier::AArg:
+        case ConversionSpecifier::fArg:
+        case ConversionSpecifier::FArg:
+        case ConversionSpecifier::eArg:
+        case ConversionSpecifier::EArg:
+        case ConversionSpecifier::gArg:
+        case ConversionSpecifier::GArg:
+        case ConversionSpecifier::nArg:
+        case ConversionSpecifier::cArg:
+        case ConversionSpecifier::sArg:
+          return true;
+        default:
+          return false;
+      }
+      
+    case LengthModifier::AsLongDouble:
+      switch (CS.getKind()) {
+        case ConversionSpecifier::aArg:
+        case ConversionSpecifier::AArg:
+        case ConversionSpecifier::fArg:
+        case ConversionSpecifier::FArg:
+        case ConversionSpecifier::eArg:
+        case ConversionSpecifier::EArg:
+        case ConversionSpecifier::gArg:
+        case ConversionSpecifier::GArg:
+          return true;
+        default:
+          return false;
+      }
+  }
+  return false;
+}
+
+
diff --git a/lib/Analysis/FormatStringParsing.h b/lib/Analysis/FormatStringParsing.h
new file mode 100644
index 000000000000..607e99ccd076
--- /dev/null
+++ b/lib/Analysis/FormatStringParsing.h
@@ -0,0 +1,72 @@
+#ifndef LLVM_CLANG_FORMAT_PARSING_H
+#define LLVM_CLANG_FORMAT_PARSING_H
+
+#include "clang/Analysis/Analyses/FormatString.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Type.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace clang {
+
+template <typename T>
+class UpdateOnReturn {
+  T &ValueToUpdate;
+  const T &ValueToCopy;
+public:
+  UpdateOnReturn(T &valueToUpdate, const T &valueToCopy)
+    : ValueToUpdate(valueToUpdate), ValueToCopy(valueToCopy) {}
+
+  ~UpdateOnReturn() {
+    ValueToUpdate = ValueToCopy;
+  }
+};
+
+namespace analyze_format_string {
+  
+OptionalAmount ParseAmount(const char *&Beg, const char *E);
+OptionalAmount ParseNonPositionAmount(const char *&Beg, const char *E,
+                                      unsigned &argIndex);
+
+OptionalAmount ParsePositionAmount(FormatStringHandler &H,
+                                   const char *Start, const char *&Beg,
+                                   const char *E, PositionContext p);
+  
+bool ParseFieldWidth(FormatStringHandler &H,
+                     FormatSpecifier &CS,
+                     const char *Start, const char *&Beg, const char *E,
+                     unsigned *argIndex);
+    
+bool ParseArgPosition(FormatStringHandler &H,
+                      FormatSpecifier &CS, const char *Start,
+                      const char *&Beg, const char *E);
+
+/// Returns true if a LengthModifier was parsed and installed in the
+/// FormatSpecifier& argument, and false otherwise.
+bool ParseLengthModifier(FormatSpecifier &FS, const char *&Beg, const char *E);
+  
+template <typename T> class SpecifierResult {
+  T FS;
+  const char *Start;
+  bool Stop;
+public:
+  SpecifierResult(bool stop = false)
+  : Start(0), Stop(stop) {}
+  SpecifierResult(const char *start,
+                  const T &fs)
+  : FS(fs), Start(start), Stop(false) {}
+  
+  const char *getStart() const { return Start; }
+  bool shouldStop() const { return Stop; }
+  bool hasValue() const { return Start != 0; }
+  const T &getValue() const {
+    assert(hasValue());
+    return FS;
+  }
+  const T &getValue() { return FS; }
+};
+  
+} // end analyze_format_string namespace
+} // end clang namespace
+
+#endif
+
diff --git a/lib/Analysis/LiveVariables.cpp b/lib/Analysis/LiveVariables.cpp
index 4efe25ea1e0e..47b2e3d6040f 100644
--- a/lib/Analysis/LiveVariables.cpp
+++ b/lib/Analysis/LiveVariables.cpp
@@ -77,12 +77,13 @@ public:
 };
 } // end anonymous namespace
 
-LiveVariables::LiveVariables(AnalysisContext &AC) {  
+LiveVariables::LiveVariables(AnalysisContext &AC, bool killAtAssign) {
   // Register all referenced VarDecls.
   CFG &cfg = *AC.getCFG();
   getAnalysisData().setCFG(cfg);
   getAnalysisData().setContext(AC.getASTContext());
   getAnalysisData().AC = &AC;
+  getAnalysisData().killAtAssign = killAtAssign;
 
   RegisterDecls R(getAnalysisData());
   cfg.VisitBlockStmts(R);
@@ -229,10 +230,10 @@ void TransferFuncs::VisitUnaryOperator(UnaryOperator* U) {
   Expr *E = U->getSubExpr();
 
   switch (U->getOpcode()) {
-  case UnaryOperator::PostInc:
-  case UnaryOperator::PostDec:
-  case UnaryOperator::PreInc:
-  case UnaryOperator::PreDec:
+  case UO_PostInc:
+  case UO_PostDec:
+  case UO_PreInc:
+  case UO_PreDec:
     // Walk through the subexpressions, blasting through ParenExprs
     // until we either find a DeclRefExpr or some non-DeclRefExpr
     // expression.
@@ -260,15 +261,16 @@ void TransferFuncs::VisitAssign(BinaryOperator* B) {
     if (DR->getDecl()->getType()->isReferenceType()) {
       VisitDeclRefExpr(DR);
     } else {
-      // Update liveness inforamtion.
-      unsigned bit = AD.getIdx(DR->getDecl());
-      LiveState.getDeclBit(bit) = Dead | AD.AlwaysLive.getDeclBit(bit);
-
-      if (AD.Observer) { AD.Observer->ObserverKill(DR); }
+      if (AD.killAtAssign) {
+        // Update liveness inforamtion.
+        unsigned bit = AD.getIdx(DR->getDecl());
+        LiveState.getDeclBit(bit) = Dead | AD.AlwaysLive.getDeclBit(bit);
 
+        if (AD.Observer) { AD.Observer->ObserverKill(DR); }
+      }
       // Handle things like +=, etc., which also generate "uses"
       // of a variable.  Do this just by visiting the subexpression.
-      if (B->getOpcode() != BinaryOperator::Assign)
+      if (B->getOpcode() != BO_Assign)
         VisitDeclRefExpr(DR);
     }
   }
diff --git a/lib/Analysis/Makefile b/lib/Analysis/Makefile
index 03bf7a6712f3..fbbb83d71003 100644
--- a/lib/Analysis/Makefile
+++ b/lib/Analysis/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangAnalysis
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Analysis/PrintfFormatString.cpp b/lib/Analysis/PrintfFormatString.cpp
index 558d38af0c6e..b8c327cdeba6 100644
--- a/lib/Analysis/PrintfFormatString.cpp
+++ b/lib/Analysis/PrintfFormatString.cpp
@@ -1,4 +1,4 @@
-//= PrintfFormatStrings.cpp - Analysis of printf format strings --*- C++ -*-==//
+//== PrintfFormatString.cpp - Analysis of printf format strings --*- C++ -*-==//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -12,141 +12,28 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Analysis/Analyses/PrintfFormatString.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Type.h"
-#include "llvm/Support/raw_ostream.h"
+#include "clang/Analysis/Analyses/FormatString.h"
+#include "FormatStringParsing.h"
 
-using clang::analyze_printf::ArgTypeResult;
-using clang::analyze_printf::FormatSpecifier;
-using clang::analyze_printf::FormatStringHandler;
-using clang::analyze_printf::OptionalAmount;
-using clang::analyze_printf::PositionContext;
-using clang::analyze_printf::ConversionSpecifier;
-using clang::analyze_printf::LengthModifier;
+using clang::analyze_format_string::ArgTypeResult;
+using clang::analyze_format_string::FormatStringHandler;
+using clang::analyze_format_string::LengthModifier;
+using clang::analyze_format_string::OptionalAmount;
+using clang::analyze_format_string::ConversionSpecifier;
+using clang::analyze_printf::PrintfSpecifier;
 
 using namespace clang;
 
-namespace {
-class FormatSpecifierResult {
-  FormatSpecifier FS;
-  const char *Start;
-  bool Stop;
-public:
-  FormatSpecifierResult(bool stop = false)
-    : Start(0), Stop(stop) {}
-  FormatSpecifierResult(const char *start,
-                        const FormatSpecifier &fs)
-    : FS(fs), Start(start), Stop(false) {}
-
-  const char *getStart() const { return Start; }
-  bool shouldStop() const { return Stop; }
-  bool hasValue() const { return Start != 0; }
-  const FormatSpecifier &getValue() const {
-    assert(hasValue());
-    return FS;
-  }
-  const FormatSpecifier &getValue() { return FS; }
-};
-} // end anonymous namespace
-
-template <typename T>
-class UpdateOnReturn {
-  T &ValueToUpdate;
-  const T &ValueToCopy;
-public:
-  UpdateOnReturn(T &valueToUpdate, const T &valueToCopy)
-    : ValueToUpdate(valueToUpdate), ValueToCopy(valueToCopy) {}
-
-  ~UpdateOnReturn() {
-    ValueToUpdate = ValueToCopy;
-  }
-};
+typedef clang::analyze_format_string::SpecifierResult<PrintfSpecifier>
+        PrintfSpecifierResult;
 
 //===----------------------------------------------------------------------===//
 // Methods for parsing format strings.
 //===----------------------------------------------------------------------===//
 
-static OptionalAmount ParseAmount(const char *&Beg, const char *E) {
-  const char *I = Beg;
-  UpdateOnReturn <const char*> UpdateBeg(Beg, I);
-
-  unsigned accumulator = 0;
-  bool hasDigits = false;
-
-  for ( ; I != E; ++I) {
-    char c = *I;
-    if (c >= '0' && c <= '9') {
-      hasDigits = true;
-      accumulator = (accumulator * 10) + (c - '0');
-      continue;
-    }
-
-    if (hasDigits)
-      return OptionalAmount(OptionalAmount::Constant, accumulator, Beg, I - Beg,
-          false);
-
-    break;
-  }
-
-  return OptionalAmount();
-}
-
-static OptionalAmount ParseNonPositionAmount(const char *&Beg, const char *E,
-                                             unsigned &argIndex) {
-  if (*Beg == '*') {
-    ++Beg;
-    return OptionalAmount(OptionalAmount::Arg, argIndex++, Beg, 0, false);
-  }
-
-  return ParseAmount(Beg, E);
-}
-
-static OptionalAmount ParsePositionAmount(FormatStringHandler &H,
-                                          const char *Start,
-                                          const char *&Beg, const char *E,
-                                          PositionContext p) {
-  if (*Beg == '*') {
-    const char *I = Beg + 1;
-    const OptionalAmount &Amt = ParseAmount(I, E);
-
-    if (Amt.getHowSpecified() == OptionalAmount::NotSpecified) {
-      H.HandleInvalidPosition(Beg, I - Beg, p);
-      return OptionalAmount(false);
-    }
-
-    if (I== E) {
-      // No more characters left?
-      H.HandleIncompleteFormatSpecifier(Start, E - Start);
-      return OptionalAmount(false);
-    }
-
-    assert(Amt.getHowSpecified() == OptionalAmount::Constant);
-
-    if (*I == '$') {
-      // Handle positional arguments
-
-      // Special case: '*0$', since this is an easy mistake.
-      if (Amt.getConstantAmount() == 0) {
-        H.HandleZeroPosition(Beg, I - Beg + 1);
-        return OptionalAmount(false);
-      }
-
-      const char *Tmp = Beg;
-      Beg = ++I;
-
-      return OptionalAmount(OptionalAmount::Arg, Amt.getConstantAmount() - 1,
-                            Tmp, 0, true);
-    }
-
-    H.HandleInvalidPosition(Beg, I - Beg, p);
-    return OptionalAmount(false);
-  }
-
-  return ParseAmount(Beg, E);
-}
+using analyze_format_string::ParseNonPositionAmount;
 
-static bool ParsePrecision(FormatStringHandler &H, FormatSpecifier &FS,
+static bool ParsePrecision(FormatStringHandler &H, PrintfSpecifier &FS,
                            const char *Start, const char *&Beg, const char *E,
                            unsigned *argIndex) {
   if (argIndex) {
@@ -154,7 +41,7 @@ static bool ParsePrecision(FormatStringHandler &H, FormatSpecifier &FS,
   }
   else {
     const OptionalAmount Amt = ParsePositionAmount(H, Start, Beg, E,
-                                                  analyze_printf::PrecisionPos);
+                                           analyze_format_string::PrecisionPos);
     if (Amt.isInvalid())
       return true;
     FS.setPrecision(Amt);
@@ -162,61 +49,12 @@ static bool ParsePrecision(FormatStringHandler &H, FormatSpecifier &FS,
   return false;
 }
 
-static bool ParseFieldWidth(FormatStringHandler &H, FormatSpecifier &FS,
-                            const char *Start, const char *&Beg, const char *E,
-                            unsigned *argIndex) {
-  // FIXME: Support negative field widths.
-  if (argIndex) {
-    FS.setFieldWidth(ParseNonPositionAmount(Beg, E, *argIndex));
-  }
-  else {
-    const OptionalAmount Amt = ParsePositionAmount(H, Start, Beg, E,
-                                                 analyze_printf::FieldWidthPos);
-    if (Amt.isInvalid())
-      return true;
-    FS.setFieldWidth(Amt);
-  }
-  return false;
-}
-
-static bool ParseArgPosition(FormatStringHandler &H,
-                             FormatSpecifier &FS, const char *Start,
-                             const char *&Beg, const char *E) {
-
-  using namespace clang::analyze_printf;
-  const char *I = Beg;
-
-  const OptionalAmount &Amt = ParseAmount(I, E);
-
-  if (I == E) {
-    // No more characters left?
-    H.HandleIncompleteFormatSpecifier(Start, E - Start);
-    return true;
-  }
-
-  if (Amt.getHowSpecified() == OptionalAmount::Constant && *(I++) == '$') {
-    // Special case: '%0$', since this is an easy mistake.
-    if (Amt.getConstantAmount() == 0) {
-      H.HandleZeroPosition(Start, I - Start);
-      return true;
-    }
-
-    FS.setArgIndex(Amt.getConstantAmount() - 1);
-    FS.setUsesPositionalArg();
-    // Update the caller's pointer if we decided to consume
-    // these characters.
-    Beg = I;
-    return false;
-  }
-
-  return false;
-}
-
-static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
+static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
                                                   const char *&Beg,
                                                   const char *E,
                                                   unsigned &argIndex) {
 
+  using namespace clang::analyze_format_string;
   using namespace clang::analyze_printf;
 
   const char *I = Beg;
@@ -243,17 +81,17 @@ static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
 
   if (I == E) {
     // No more characters left?
-    H.HandleIncompleteFormatSpecifier(Start, E - Start);
+    H.HandleIncompleteSpecifier(Start, E - Start);
     return true;
   }
 
-  FormatSpecifier FS;
+  PrintfSpecifier FS;
   if (ParseArgPosition(H, FS, Start, I, E))
     return true;
 
   if (I == E) {
     // No more characters left?
-    H.HandleIncompleteFormatSpecifier(Start, E - Start);
+    H.HandleIncompleteSpecifier(Start, E - Start);
     return true;
   }
 
@@ -274,7 +112,7 @@ static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
 
   if (I == E) {
     // No more characters left?
-    H.HandleIncompleteFormatSpecifier(Start, E - Start);
+    H.HandleIncompleteSpecifier(Start, E - Start);
     return true;
   }
 
@@ -285,7 +123,7 @@ static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
 
   if (I == E) {
     // No more characters left?
-    H.HandleIncompleteFormatSpecifier(Start, E - Start);
+    H.HandleIncompleteSpecifier(Start, E - Start);
     return true;
   }
 
@@ -293,7 +131,7 @@ static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
   if (*I == '.') {
     ++I;
     if (I == E) {
-      H.HandleIncompleteFormatSpecifier(Start, E - Start);
+      H.HandleIncompleteSpecifier(Start, E - Start);
       return true;
     }
 
@@ -303,39 +141,15 @@ static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
 
     if (I == E) {
       // No more characters left?
-      H.HandleIncompleteFormatSpecifier(Start, E - Start);
+      H.HandleIncompleteSpecifier(Start, E - Start);
       return true;
     }
   }
 
   // Look for the length modifier.
-  LengthModifier::Kind lmKind = LengthModifier::None;
-  const char *lmPosition = I;
-  switch (*I) {
-    default:
-      break;
-    case 'h':
-      ++I;
-      lmKind = (I != E && *I == 'h') ?
-          ++I, LengthModifier::AsChar : LengthModifier::AsShort;
-      break;
-    case 'l':
-      ++I;
-      lmKind = (I != E && *I == 'l') ?
-          ++I, LengthModifier::AsLongLong : LengthModifier::AsLong;
-      break;
-    case 'j': lmKind = LengthModifier::AsIntMax;     ++I; break;
-    case 'z': lmKind = LengthModifier::AsSizeT;      ++I; break;
-    case 't': lmKind = LengthModifier::AsPtrDiff;    ++I; break;
-    case 'L': lmKind = LengthModifier::AsLongDouble; ++I; break;
-    case 'q': lmKind = LengthModifier::AsLongLong;   ++I; break;
-  }
-  LengthModifier lm(lmPosition, lmKind);
-  FS.setLengthModifier(lm);
-
-  if (I == E) {
+  if (ParseLengthModifier(FS, I, E) && I == E) {
     // No more characters left?
-    H.HandleIncompleteFormatSpecifier(Start, E - Start);
+    H.HandleIncompleteSpecifier(Start, E - Start);
     return true;
   }
 
@@ -359,46 +173,47 @@ static FormatSpecifierResult ParseFormatSpecifier(FormatStringHandler &H,
     case 'G': k = ConversionSpecifier::GArg; break;
     case 'X': k = ConversionSpecifier::XArg; break;
     case 'a': k = ConversionSpecifier::aArg; break;
-    case 'c': k = ConversionSpecifier::IntAsCharArg; break;
+    case 'c': k = ConversionSpecifier::cArg; break;
     case 'd': k = ConversionSpecifier::dArg; break;
     case 'e': k = ConversionSpecifier::eArg; break;
     case 'f': k = ConversionSpecifier::fArg; break;
     case 'g': k = ConversionSpecifier::gArg; break;
     case 'i': k = ConversionSpecifier::iArg; break;
-    case 'n': k = ConversionSpecifier::OutIntPtrArg; break;
+    case 'n': k = ConversionSpecifier::nArg; break;
     case 'o': k = ConversionSpecifier::oArg; break;
-    case 'p': k = ConversionSpecifier::VoidPtrArg;   break;
-    case 's': k = ConversionSpecifier::CStrArg;      break;
+    case 'p': k = ConversionSpecifier::pArg;   break;
+    case 's': k = ConversionSpecifier::sArg;      break;
     case 'u': k = ConversionSpecifier::uArg; break;
     case 'x': k = ConversionSpecifier::xArg; break;
     // Mac OS X (unicode) specific
     case 'C': k = ConversionSpecifier::CArg; break;
-    case 'S': k = ConversionSpecifier::UnicodeStrArg; break;
+    case 'S': k = ConversionSpecifier::SArg; break;
     // Objective-C.
     case '@': k = ConversionSpecifier::ObjCObjArg; break;
     // Glibc specific.
     case 'm': k = ConversionSpecifier::PrintErrno; break;
   }
-  ConversionSpecifier CS(conversionPosition, k);
+  PrintfConversionSpecifier CS(conversionPosition, k);
   FS.setConversionSpecifier(CS);
   if (CS.consumesDataArgument() && !FS.usesPositionalArg())
     FS.setArgIndex(argIndex++);
 
   if (k == ConversionSpecifier::InvalidSpecifier) {
     // Assume the conversion takes one argument.
-    return !H.HandleInvalidConversionSpecifier(FS, Beg, I - Beg);
+    return !H.HandleInvalidPrintfConversionSpecifier(FS, Beg, I - Beg);
   }
-  return FormatSpecifierResult(Start, FS);
+  return PrintfSpecifierResult(Start, FS);
 }
 
-bool clang::analyze_printf::ParseFormatString(FormatStringHandler &H,
-                       const char *I, const char *E) {
+bool clang::analyze_format_string::ParsePrintfString(FormatStringHandler &H,
+                                                     const char *I,
+                                                     const char *E) {
 
   unsigned argIndex = 0;
 
   // Keep looking for a format specifier until we have exhausted the string.
   while (I != E) {
-    const FormatSpecifierResult &FSR = ParseFormatSpecifier(H, I, E, argIndex);
+    const PrintfSpecifierResult &FSR = ParsePrintfSpecifier(H, I, E, argIndex);
     // Did a fail-stop error of any kind occur when parsing the specifier?
     // If so, don't do any more processing.
     if (FSR.shouldStop())
@@ -408,7 +223,7 @@ bool clang::analyze_printf::ParseFormatString(FormatStringHandler &H,
     if (!FSR.hasValue())
       continue;
     // We have a format specifier.  Pass it to the callback.
-    if (!H.HandleFormatSpecifier(FSR.getValue(), FSR.getStart(),
+    if (!H.HandlePrintfSpecifier(FSR.getValue(), FSR.getStart(),
                                  I - FSR.getStart()))
       return true;
   }
@@ -416,129 +231,6 @@ bool clang::analyze_printf::ParseFormatString(FormatStringHandler &H,
   return false;
 }
 
-FormatStringHandler::~FormatStringHandler() {}
-
-//===----------------------------------------------------------------------===//
-// Methods on ArgTypeResult.
-//===----------------------------------------------------------------------===//
-
-bool ArgTypeResult::matchesType(ASTContext &C, QualType argTy) const {
-  switch (K) {
-    case InvalidTy:
-      assert(false && "ArgTypeResult must be valid");
-      return true;
-
-    case UnknownTy:
-      return true;
-
-    case SpecificTy: {
-      argTy = C.getCanonicalType(argTy).getUnqualifiedType();
-      if (T == argTy)
-        return true;
-      if (const BuiltinType *BT = argTy->getAs<BuiltinType>())
-        switch (BT->getKind()) {
-          default:
-            break;
-          case BuiltinType::Char_S:
-          case BuiltinType::SChar:
-            return T == C.UnsignedCharTy;
-          case BuiltinType::Char_U:
-          case BuiltinType::UChar:
-            return T == C.SignedCharTy;
-          case BuiltinType::Short:
-            return T == C.UnsignedShortTy;
-          case BuiltinType::UShort:
-            return T == C.ShortTy;
-          case BuiltinType::Int:
-            return T == C.UnsignedIntTy;
-          case BuiltinType::UInt:
-            return T == C.IntTy;
-          case BuiltinType::Long:
-            return T == C.UnsignedLongTy;
-          case BuiltinType::ULong:
-            return T == C.LongTy;
-          case BuiltinType::LongLong:
-            return T == C.UnsignedLongLongTy;
-          case BuiltinType::ULongLong:
-            return T == C.LongLongTy;
-        }
-      return false;
-    }
-
-    case CStrTy: {
-      const PointerType *PT = argTy->getAs<PointerType>();
-      if (!PT)
-        return false;
-      QualType pointeeTy = PT->getPointeeType();
-      if (const BuiltinType *BT = pointeeTy->getAs<BuiltinType>())
-        switch (BT->getKind()) {
-          case BuiltinType::Void:
-          case BuiltinType::Char_U:
-          case BuiltinType::UChar:
-          case BuiltinType::Char_S:
-          case BuiltinType::SChar:
-            return true;
-          default:
-            break;
-        }
-
-      return false;
-    }
-
-    case WCStrTy: {
-      const PointerType *PT = argTy->getAs<PointerType>();
-      if (!PT)
-        return false;
-      QualType pointeeTy =
-        C.getCanonicalType(PT->getPointeeType()).getUnqualifiedType();
-      return pointeeTy == C.getWCharType();
-    }
-
-    case CPointerTy:
-      return argTy->getAs<PointerType>() != NULL ||
-      	     argTy->getAs<ObjCObjectPointerType>() != NULL;
-
-    case ObjCPointerTy:
-      return argTy->getAs<ObjCObjectPointerType>() != NULL;
-  }
-
-  // FIXME: Should be unreachable, but Clang is currently emitting
-  // a warning.
-  return false;
-}
-
-QualType ArgTypeResult::getRepresentativeType(ASTContext &C) const {
-  switch (K) {
-    case InvalidTy:
-      assert(false && "No representative type for Invalid ArgTypeResult");
-      // Fall-through.
-    case UnknownTy:
-      return QualType();
-    case SpecificTy:
-      return T;
-    case CStrTy:
-      return C.getPointerType(C.CharTy);
-    case WCStrTy:
-      return C.getPointerType(C.getWCharType());
-    case ObjCPointerTy:
-      return C.ObjCBuiltinIdTy;
-    case CPointerTy:
-      return C.VoidPtrTy;
-  }
-
-  // FIXME: Should be unreachable, but Clang is currently emitting
-  // a warning.
-  return QualType();
-}
-
-//===----------------------------------------------------------------------===//
-// Methods on OptionalAmount.
-//===----------------------------------------------------------------------===//
-
-ArgTypeResult OptionalAmount::getArgType(ASTContext &Ctx) const {
-  return Ctx.IntTy;
-}
-
 //===----------------------------------------------------------------------===//
 // Methods on ConversionSpecifier.
 //===----------------------------------------------------------------------===//
@@ -558,16 +250,17 @@ const char *ConversionSpecifier::toString() const {
   case GArg: return "G";
   case aArg: return "a";
   case AArg: return "A";
-  case IntAsCharArg:     return "c";
-  case CStrArg:          return "s";
-  case VoidPtrArg:       return "p";
-  case OutIntPtrArg:     return "n";
-  case PercentArg:       return "%";
+  case cArg: return "c";
+  case sArg: return "s";
+  case pArg: return "p";
+  case nArg: return "n";
+  case PercentArg:  return "%";
+  case ScanListArg: return "[";
   case InvalidSpecifier: return NULL;
 
   // MacOS X unicode extensions.
-  case CArg:          return "C";
-  case UnicodeStrArg: return "S";
+  case CArg: return "C";
+  case SArg: return "S";
 
   // Objective-C specific specifiers.
   case ObjCObjArg: return "@";
@@ -579,66 +272,23 @@ const char *ConversionSpecifier::toString() const {
 }
 
 //===----------------------------------------------------------------------===//
-// Methods on LengthModifier.
-//===----------------------------------------------------------------------===//
-
-const char *LengthModifier::toString() const {
-  switch (kind) {
-  case AsChar:
-    return "hh";
-  case AsShort:
-    return "h";
-  case AsLong: // or AsWideChar
-    return "l";
-  case AsLongLong:
-    return "ll";
-  case AsIntMax:
-    return "j";
-  case AsSizeT:
-    return "z";
-  case AsPtrDiff:
-    return "t";
-  case AsLongDouble:
-    return "L";
-  case None:
-    return "";
-  }
-  return NULL;
-}
-
-//===----------------------------------------------------------------------===//
-// Methods on OptionalAmount.
+// Methods on PrintfSpecifier.
 //===----------------------------------------------------------------------===//
 
-void OptionalAmount::toString(llvm::raw_ostream &os) const {
-  switch (hs) {
-  case Invalid:
-  case NotSpecified:
-    return;
-  case Arg:
-    if (UsesDotPrefix)
-        os << ".";
-    if (usesPositionalArg())
-      os << "*" << getPositionalArgIndex() << "$";
-    else
-      os << "*";
-    break;
-  case Constant:
-    if (UsesDotPrefix)
-        os << ".";
-    os << amt;
-    break;
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Methods on FormatSpecifier.
-//===----------------------------------------------------------------------===//
-
-ArgTypeResult FormatSpecifier::getArgType(ASTContext &Ctx) const {
+ArgTypeResult PrintfSpecifier::getArgType(ASTContext &Ctx) const {
+  const PrintfConversionSpecifier &CS = getConversionSpecifier();
+  
   if (!CS.consumesDataArgument())
     return ArgTypeResult::Invalid();
 
+  if (CS.getKind() == ConversionSpecifier::cArg)
+    switch (LM.getKind()) {
+      case LengthModifier::None: return Ctx.IntTy;
+      case LengthModifier::AsLong: return ArgTypeResult::WIntTy;
+      default:
+        return ArgTypeResult::Invalid();
+    }
+  
   if (CS.isIntArg())
     switch (LM.getKind()) {
       case LengthModifier::AsLongDouble:
@@ -684,15 +334,15 @@ ArgTypeResult FormatSpecifier::getArgType(ASTContext &Ctx) const {
   }
 
   switch (CS.getKind()) {
-    case ConversionSpecifier::CStrArg:
+    case ConversionSpecifier::sArg:
       return ArgTypeResult(LM.getKind() == LengthModifier::AsWideChar ?
           ArgTypeResult::WCStrTy : ArgTypeResult::CStrTy);
-    case ConversionSpecifier::UnicodeStrArg:
+    case ConversionSpecifier::SArg:
       // FIXME: This appears to be Mac OS X specific.
       return ArgTypeResult::WCStrTy;
     case ConversionSpecifier::CArg:
       return Ctx.WCharTy;
-    case ConversionSpecifier::VoidPtrArg:
+    case ConversionSpecifier::pArg:
       return ArgTypeResult::CPointerTy;
     default:
       break;
@@ -702,10 +352,10 @@ ArgTypeResult FormatSpecifier::getArgType(ASTContext &Ctx) const {
   return ArgTypeResult();
 }
 
-bool FormatSpecifier::fixType(QualType QT) {
+bool PrintfSpecifier::fixType(QualType QT) {
   // Handle strings first (char *, wchar_t *)
   if (QT->isPointerType() && (QT->getPointeeType()->isAnyCharacterType())) {
-    CS.setKind(ConversionSpecifier::CStrArg);
+    CS.setKind(ConversionSpecifier::sArg);
 
     // Disable irrelevant flags
     HasAlternativeForm = 0;
@@ -750,7 +400,7 @@ bool FormatSpecifier::fixType(QualType QT) {
 
   // Set conversion specifier and disable any flags which do not apply to it.
   if (QT->isAnyCharacterType()) {
-    CS.setKind(ConversionSpecifier::IntAsCharArg);
+    CS.setKind(ConversionSpecifier::cArg);
     Precision.setHowSpecified(OptionalAmount::NotSpecified);
     HasAlternativeForm = 0;
     HasLeadingZeroes = 0;
@@ -761,7 +411,7 @@ bool FormatSpecifier::fixType(QualType QT) {
     CS.setKind(ConversionSpecifier::fArg);
   }
   else if (QT->isPointerType()) {
-    CS.setKind(ConversionSpecifier::VoidPtrArg);
+    CS.setKind(ConversionSpecifier::pArg);
     Precision.setHowSpecified(OptionalAmount::NotSpecified);
     HasAlternativeForm = 0;
     HasLeadingZeroes = 0;
@@ -783,9 +433,9 @@ bool FormatSpecifier::fixType(QualType QT) {
   return true;
 }
 
-void FormatSpecifier::toString(llvm::raw_ostream &os) const {
+void PrintfSpecifier::toString(llvm::raw_ostream &os) const {
   // Whilst some features have no defined order, we are using the order
-  // appearing in the C99 standard (ISO/IEC 9899:1999 (E) ¤7.19.6.1)
+  // appearing in the C99 standard (ISO/IEC 9899:1999 (E) ¤7.19.6.1)
   os << "%";
 
   // Positional args
@@ -810,7 +460,7 @@ void FormatSpecifier::toString(llvm::raw_ostream &os) const {
   os << CS.toString();
 }
 
-bool FormatSpecifier::hasValidPlusPrefix() const {
+bool PrintfSpecifier::hasValidPlusPrefix() const {
   if (!HasPlusPrefix)
     return true;
 
@@ -833,7 +483,7 @@ bool FormatSpecifier::hasValidPlusPrefix() const {
   }
 }
 
-bool FormatSpecifier::hasValidAlternativeForm() const {
+bool PrintfSpecifier::hasValidAlternativeForm() const {
   if (!HasAlternativeForm)
     return true;
 
@@ -856,7 +506,7 @@ bool FormatSpecifier::hasValidAlternativeForm() const {
   }
 }
 
-bool FormatSpecifier::hasValidLeadingZeros() const {
+bool PrintfSpecifier::hasValidLeadingZeros() const {
   if (!HasLeadingZeroes)
     return true;
 
@@ -883,7 +533,7 @@ bool FormatSpecifier::hasValidLeadingZeros() const {
   }
 }
 
-bool FormatSpecifier::hasValidSpacePrefix() const {
+bool PrintfSpecifier::hasValidSpacePrefix() const {
   if (!HasSpacePrefix)
     return true;
 
@@ -906,13 +556,13 @@ bool FormatSpecifier::hasValidSpacePrefix() const {
   }
 }
 
-bool FormatSpecifier::hasValidLeftJustified() const {
+bool PrintfSpecifier::hasValidLeftJustified() const {
   if (!IsLeftJustified)
     return true;
 
   // The left justified flag is valid for all conversions except n
   switch (CS.getKind()) {
-  case ConversionSpecifier::OutIntPtrArg:
+  case ConversionSpecifier::nArg:
     return false;
 
   default:
@@ -920,75 +570,7 @@ bool FormatSpecifier::hasValidLeftJustified() const {
   }
 }
 
-bool FormatSpecifier::hasValidLengthModifier() const {
-  switch (LM.getKind()) {
-  case LengthModifier::None:
-    return true;
-
-  // Handle most integer flags
-  case LengthModifier::AsChar:
-  case LengthModifier::AsShort:
-  case LengthModifier::AsLongLong:
-  case LengthModifier::AsIntMax:
-  case LengthModifier::AsSizeT:
-  case LengthModifier::AsPtrDiff:
-    switch (CS.getKind()) {
-    case ConversionSpecifier::dArg:
-    case ConversionSpecifier::iArg:
-    case ConversionSpecifier::oArg:
-    case ConversionSpecifier::uArg:
-    case ConversionSpecifier::xArg:
-    case ConversionSpecifier::XArg:
-    case ConversionSpecifier::OutIntPtrArg:
-      return true;
-    default:
-      return false;
-    }
-
-  // Handle 'l' flag
-  case LengthModifier::AsLong:
-    switch (CS.getKind()) {
-    case ConversionSpecifier::dArg:
-    case ConversionSpecifier::iArg:
-    case ConversionSpecifier::oArg:
-    case ConversionSpecifier::uArg:
-    case ConversionSpecifier::xArg:
-    case ConversionSpecifier::XArg:
-    case ConversionSpecifier::aArg:
-    case ConversionSpecifier::AArg:
-    case ConversionSpecifier::fArg:
-    case ConversionSpecifier::FArg:
-    case ConversionSpecifier::eArg:
-    case ConversionSpecifier::EArg:
-    case ConversionSpecifier::gArg:
-    case ConversionSpecifier::GArg:
-    case ConversionSpecifier::OutIntPtrArg:
-    case ConversionSpecifier::IntAsCharArg:
-    case ConversionSpecifier::CStrArg:
-      return true;
-    default:
-      return false;
-    }
-
-  case LengthModifier::AsLongDouble:
-    switch (CS.getKind()) {
-    case ConversionSpecifier::aArg:
-    case ConversionSpecifier::AArg:
-    case ConversionSpecifier::fArg:
-    case ConversionSpecifier::FArg:
-    case ConversionSpecifier::eArg:
-    case ConversionSpecifier::EArg:
-    case ConversionSpecifier::gArg:
-    case ConversionSpecifier::GArg:
-      return true;
-    default:
-      return false;
-    }
-  }
-  return false;
-}
-
-bool FormatSpecifier::hasValidPrecision() const {
+bool PrintfSpecifier::hasValidPrecision() const {
   if (Precision.getHowSpecified() == OptionalAmount::NotSpecified)
     return true;
 
@@ -1008,20 +590,20 @@ bool FormatSpecifier::hasValidPrecision() const {
   case ConversionSpecifier::FArg:
   case ConversionSpecifier::gArg:
   case ConversionSpecifier::GArg:
-  case ConversionSpecifier::CStrArg:
+  case ConversionSpecifier::sArg:
     return true;
 
   default:
     return false;
   }
 }
-bool FormatSpecifier::hasValidFieldWidth() const {
+bool PrintfSpecifier::hasValidFieldWidth() const {
   if (FieldWidth.getHowSpecified() == OptionalAmount::NotSpecified)
       return true;
 
   // The field width is valid for all conversions except n
   switch (CS.getKind()) {
-  case ConversionSpecifier::OutIntPtrArg:
+  case ConversionSpecifier::nArg:
     return false;
 
   default:
diff --git a/lib/Analysis/PseudoConstantAnalysis.cpp b/lib/Analysis/PseudoConstantAnalysis.cpp
new file mode 100644
index 000000000000..ff43fc252aa3
--- /dev/null
+++ b/lib/Analysis/PseudoConstantAnalysis.cpp
@@ -0,0 +1,238 @@
+//== PseudoConstantAnalysis.cpp - Find Pseudoconstants in the AST-*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file tracks the usage of variables in a Decl body to see if they are
+// never written to, implying that they constant. This is useful in static
+// analysis to see if a developer might have intended a variable to be const.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/Analyses/PseudoConstantAnalysis.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/Stmt.h"
+#include <deque>
+
+using namespace clang;
+
+// The number of ValueDecls we want to keep track of by default (per-function)
+#define VARDECL_SET_SIZE 256
+typedef llvm::SmallPtrSet<const VarDecl*, VARDECL_SET_SIZE> VarDeclSet;
+
+PseudoConstantAnalysis::PseudoConstantAnalysis(const Stmt *DeclBody) :
+      DeclBody(DeclBody), Analyzed(false) {
+  NonConstantsImpl = new VarDeclSet;
+  UsedVarsImpl = new VarDeclSet;
+}
+
+PseudoConstantAnalysis::~PseudoConstantAnalysis() {
+  delete (VarDeclSet*)NonConstantsImpl;
+  delete (VarDeclSet*)UsedVarsImpl;
+}
+
+// Returns true if the given ValueDecl is never written to in the given DeclBody
+bool PseudoConstantAnalysis::isPseudoConstant(const VarDecl *VD) {
+  // Only local and static variables can be pseudoconstants
+  if (!VD->hasLocalStorage() && !VD->isStaticLocal())
+    return false;
+
+  if (!Analyzed) {
+    RunAnalysis();
+    Analyzed = true;
+  }
+
+  VarDeclSet *NonConstants = (VarDeclSet*)NonConstantsImpl;
+
+  return !NonConstants->count(VD);
+}
+
+// Returns true if the variable was used (self assignments don't count)
+bool PseudoConstantAnalysis::wasReferenced(const VarDecl *VD) {
+  if (!Analyzed) {
+    RunAnalysis();
+    Analyzed = true;
+  }
+
+  VarDeclSet *UsedVars = (VarDeclSet*)UsedVarsImpl;
+
+  return UsedVars->count(VD);
+}
+
+// Returns a Decl from a (Block)DeclRefExpr (if any)
+const Decl *PseudoConstantAnalysis::getDecl(const Expr *E) {
+  if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E))
+    return DR->getDecl();
+  else if (const BlockDeclRefExpr *BDR = dyn_cast<BlockDeclRefExpr>(E))
+    return BDR->getDecl();
+  else
+    return 0;
+}
+
+void PseudoConstantAnalysis::RunAnalysis() {
+  std::deque<const Stmt *> WorkList;
+  VarDeclSet *NonConstants = (VarDeclSet*)NonConstantsImpl;
+  VarDeclSet *UsedVars = (VarDeclSet*)UsedVarsImpl;
+
+  // Start with the top level statement of the function
+  WorkList.push_back(DeclBody);
+
+  while (!WorkList.empty()) {
+    const Stmt* Head = WorkList.front();
+    WorkList.pop_front();
+
+    switch (Head->getStmtClass()) {
+    // Case 1: Assignment operators modifying VarDecls
+    case Stmt::BinaryOperatorClass: {
+      const BinaryOperator *BO = cast<BinaryOperator>(Head);
+      // Look for a Decl on the LHS
+      const Decl *LHSDecl = getDecl(BO->getLHS()->IgnoreParenCasts());
+      if (!LHSDecl)
+        break;
+
+      // We found a binary operator with a DeclRefExpr on the LHS. We now check
+      // for any of the assignment operators, implying that this Decl is being
+      // written to.
+      switch (BO->getOpcode()) {
+      // Self-assignments don't count as use of a variable
+      case BO_Assign: {
+        // Look for a DeclRef on the RHS
+        const Decl *RHSDecl = getDecl(BO->getRHS()->IgnoreParenCasts());
+
+        // If the Decls match, we have self-assignment
+        if (LHSDecl == RHSDecl)
+          // Do not visit the children
+          continue;
+
+      }
+      case BO_AddAssign:
+      case BO_SubAssign:
+      case BO_MulAssign:
+      case BO_DivAssign:
+      case BO_AndAssign:
+      case BO_OrAssign:
+      case BO_XorAssign:
+      case BO_ShlAssign:
+      case BO_ShrAssign: {
+        const VarDecl *VD = dyn_cast<VarDecl>(LHSDecl);
+        // The DeclRefExpr is being assigned to - mark it as non-constant
+        if (VD)
+          NonConstants->insert(VD);
+        break;
+      }
+
+      default:
+        break;
+      }
+      break;
+    }
+
+    // Case 2: Pre/post increment/decrement and address of
+    case Stmt::UnaryOperatorClass: {
+      const UnaryOperator *UO = cast<UnaryOperator>(Head);
+
+      // Look for a DeclRef in the subexpression
+      const Decl *D = getDecl(UO->getSubExpr()->IgnoreParenCasts());
+      if (!D)
+        break;
+
+      // We found a unary operator with a DeclRef as a subexpression. We now
+      // check for any of the increment/decrement operators, as well as
+      // addressOf.
+      switch (UO->getOpcode()) {
+      case UO_PostDec:
+      case UO_PostInc:
+      case UO_PreDec:
+      case UO_PreInc:
+        // The DeclRef is being changed - mark it as non-constant
+      case UO_AddrOf: {
+        // If we are taking the address of the DeclRefExpr, assume it is
+        // non-constant.
+        const VarDecl *VD = dyn_cast<VarDecl>(D);
+        if (VD)
+          NonConstants->insert(VD);
+        break;
+      }
+
+      default:
+        break;
+      }
+      break;
+    }
+
+    // Case 3: Reference Declarations
+    case Stmt::DeclStmtClass: {
+      const DeclStmt *DS = cast<DeclStmt>(Head);
+      // Iterate over each decl and see if any of them contain reference decls
+      for (DeclStmt::const_decl_iterator I = DS->decl_begin(),
+          E = DS->decl_end(); I != E; ++I) {
+        // We only care about VarDecls
+        const VarDecl *VD = dyn_cast<VarDecl>(*I);
+        if (!VD)
+          continue;
+
+        // We found a VarDecl; make sure it is a reference type
+        if (!VD->getType().getTypePtr()->isReferenceType())
+          continue;
+
+        // Try to find a Decl in the initializer
+        const Decl *D = getDecl(VD->getInit()->IgnoreParenCasts());
+        if (!D)
+          break;
+
+        // If the reference is to another var, add the var to the non-constant
+        // list
+        if (const VarDecl *RefVD = dyn_cast<VarDecl>(D)) {
+          NonConstants->insert(RefVD);
+          continue;
+        }
+      }
+      break;
+    }
+
+    // Case 4: Block variable references
+    case Stmt::BlockDeclRefExprClass: {
+      const BlockDeclRefExpr *BDR = cast<BlockDeclRefExpr>(Head);
+      if (const VarDecl *VD = dyn_cast<VarDecl>(BDR->getDecl())) {
+        // Add the Decl to the used list
+        UsedVars->insert(VD);
+        continue;
+      }
+      break;
+    }
+
+    // Case 5: Variable references
+    case Stmt::DeclRefExprClass: {
+      const DeclRefExpr *DR = cast<DeclRefExpr>(Head);
+      if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
+        // Add the Decl to the used list
+        UsedVars->insert(VD);
+        continue;
+      }
+      break;
+    }
+
+    // Case 6: Block expressions
+    case Stmt::BlockExprClass: {
+      const BlockExpr *B = cast<BlockExpr>(Head);
+      // Add the body of the block to the list
+      WorkList.push_back(B->getBody());
+      continue;
+    }
+
+      default:
+        break;
+    } // switch (head->getStmtClass())
+
+    // Add all substatements to the worklist
+    for (Stmt::const_child_iterator I = Head->child_begin(),
+        E = Head->child_end(); I != E; ++I)
+      if (*I)
+        WorkList.push_back(*I);
+  } // while (!WorkList.empty())
+}
diff --git a/lib/Analysis/ReachableCode.cpp b/lib/Analysis/ReachableCode.cpp
index f959e5cd43e1..05439392f916 100644
--- a/lib/Analysis/ReachableCode.cpp
+++ b/lib/Analysis/ReachableCode.cpp
@@ -41,7 +41,7 @@ top:
   switch (S->getStmtClass()) {
     case Expr::BinaryOperatorClass: {
       const BinaryOperator *BO = cast<BinaryOperator>(S);
-      if (BO->getOpcode() == BinaryOperator::Comma) {
+      if (BO->getOpcode() == BO_Comma) {
         if (sn+1 < b.size())
           return b[sn+1].getStmt()->getLocStart();
         const CFGBlock *n = &b;
diff --git a/lib/Analysis/ScanfFormatString.cpp b/lib/Analysis/ScanfFormatString.cpp
new file mode 100644
index 000000000000..6a8673ab55ca
--- /dev/null
+++ b/lib/Analysis/ScanfFormatString.cpp
@@ -0,0 +1,221 @@
+//= ScanfFormatString.cpp - Analysis of printf format strings --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Handling of format string in scanf and friends.  The structure of format
+// strings for fscanf() are described in C99 7.19.6.2.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/Analyses/FormatString.h"
+#include "FormatStringParsing.h"
+
+using clang::analyze_format_string::ArgTypeResult;
+using clang::analyze_format_string::FormatStringHandler;
+using clang::analyze_format_string::LengthModifier;
+using clang::analyze_format_string::OptionalAmount;
+using clang::analyze_format_string::ConversionSpecifier;
+using clang::analyze_scanf::ScanfConversionSpecifier;
+using clang::analyze_scanf::ScanfSpecifier;
+using clang::UpdateOnReturn;
+
+typedef clang::analyze_format_string::SpecifierResult<ScanfSpecifier>
+        ScanfSpecifierResult;
+
+static bool ParseScanList(FormatStringHandler &H,
+                          ScanfConversionSpecifier &CS,
+                          const char *&Beg, const char *E) {
+  const char *I = Beg;
+  const char *start = I - 1;
+  UpdateOnReturn <const char*> UpdateBeg(Beg, I);
+
+  // No more characters?
+  if (I == E) {
+    H.HandleIncompleteScanList(start, I);
+    return true;
+  }
+  
+  // Special case: ']' is the first character.
+  if (*I == ']') {
+    if (++I == E) {
+      H.HandleIncompleteScanList(start, I - 1);
+      return true;
+    }
+  }
+
+  // Look for a ']' character which denotes the end of the scan list.
+  while (*I != ']') {
+    if (++I == E) {
+      H.HandleIncompleteScanList(start, I - 1);
+      return true;
+    }
+  }    
+
+  CS.setEndScanList(I);
+  return false;
+}
+
+// FIXME: Much of this is copy-paste from ParsePrintfSpecifier.
+// We can possibly refactor.
+static ScanfSpecifierResult ParseScanfSpecifier(FormatStringHandler &H,
+                                                const char *&Beg,
+                                                const char *E,
+                                                unsigned &argIndex) {
+  
+  using namespace clang::analyze_scanf;
+  const char *I = Beg;
+  const char *Start = 0;
+  UpdateOnReturn <const char*> UpdateBeg(Beg, I);
+
+    // Look for a '%' character that indicates the start of a format specifier.
+  for ( ; I != E ; ++I) {
+    char c = *I;
+    if (c == '\0') {
+        // Detect spurious null characters, which are likely errors.
+      H.HandleNullChar(I);
+      return true;
+    }
+    if (c == '%') {
+      Start = I++;  // Record the start of the format specifier.
+      break;
+    }
+  }
+  
+    // No format specifier found?
+  if (!Start)
+    return false;
+  
+  if (I == E) {
+      // No more characters left?
+    H.HandleIncompleteSpecifier(Start, E - Start);
+    return true;
+  }
+  
+  ScanfSpecifier FS;
+  if (ParseArgPosition(H, FS, Start, I, E))
+    return true;
+
+  if (I == E) {
+      // No more characters left?
+    H.HandleIncompleteSpecifier(Start, E - Start);
+    return true;
+  }
+  
+  // Look for '*' flag if it is present.
+  if (*I == '*') {
+    FS.setSuppressAssignment(I);
+    if (++I == E) {
+      H.HandleIncompleteSpecifier(Start, E - Start);
+      return true;
+    }
+  }
+  
+  // Look for the field width (if any).  Unlike printf, this is either
+  // a fixed integer or isn't present.
+  const OptionalAmount &Amt = clang::analyze_format_string::ParseAmount(I, E);
+  if (Amt.getHowSpecified() != OptionalAmount::NotSpecified) {
+    assert(Amt.getHowSpecified() == OptionalAmount::Constant);
+    FS.setFieldWidth(Amt);
+
+    if (I == E) {
+      // No more characters left?
+      H.HandleIncompleteSpecifier(Start, E - Start);
+      return true;
+    }
+  }
+  
+  // Look for the length modifier.
+  if (ParseLengthModifier(FS, I, E) && I == E) {
+      // No more characters left?
+    H.HandleIncompleteSpecifier(Start, E - Start);
+    return true;
+  }
+  
+  // Detect spurious null characters, which are likely errors.
+  if (*I == '\0') {
+    H.HandleNullChar(I);
+    return true;
+  }
+  
+  // Finally, look for the conversion specifier.
+  const char *conversionPosition = I++;
+  ScanfConversionSpecifier::Kind k = ScanfConversionSpecifier::InvalidSpecifier;
+  switch (*conversionPosition) {
+    default:
+      break;
+    case '%': k = ConversionSpecifier::PercentArg;   break;
+    case 'A': k = ConversionSpecifier::AArg; break;
+    case 'E': k = ConversionSpecifier::EArg; break;
+    case 'F': k = ConversionSpecifier::FArg; break;
+    case 'G': k = ConversionSpecifier::GArg; break;
+    case 'X': k = ConversionSpecifier::XArg; break;
+    case 'a': k = ConversionSpecifier::aArg; break;
+    case 'd': k = ConversionSpecifier::dArg; break;
+    case 'e': k = ConversionSpecifier::eArg; break;
+    case 'f': k = ConversionSpecifier::fArg; break;
+    case 'g': k = ConversionSpecifier::gArg; break;
+    case 'i': k = ConversionSpecifier::iArg; break;
+    case 'n': k = ConversionSpecifier::nArg; break;
+    case 'c': k = ConversionSpecifier::cArg; break;
+    case 'C': k = ConversionSpecifier::CArg; break;
+    case 'S': k = ConversionSpecifier::SArg; break;
+    case '[': k = ConversionSpecifier::ScanListArg; break;
+    case 'u': k = ConversionSpecifier::uArg; break;
+    case 'x': k = ConversionSpecifier::xArg; break;
+    case 'o': k = ConversionSpecifier::oArg; break;
+    case 's': k = ConversionSpecifier::sArg; break;
+    case 'p': k = ConversionSpecifier::pArg; break;
+  }
+  ScanfConversionSpecifier CS(conversionPosition, k);
+  if (k == ScanfConversionSpecifier::ScanListArg) {
+    if (!ParseScanList(H, CS, I, E))
+      return true;
+  }
+  FS.setConversionSpecifier(CS);
+  if (CS.consumesDataArgument() && !FS.getSuppressAssignment()
+      && !FS.usesPositionalArg())
+    FS.setArgIndex(argIndex++);
+  
+  // FIXME: '%' and '*' doesn't make sense.  Issue a warning.
+  // FIXME: 'ConsumedSoFar' and '*' doesn't make sense.
+  
+  if (k == ScanfConversionSpecifier::InvalidSpecifier) {
+    // Assume the conversion takes one argument.
+    return !H.HandleInvalidScanfConversionSpecifier(FS, Beg, I - Beg);
+  }
+  return ScanfSpecifierResult(Start, FS);
+}
+  
+bool clang::analyze_format_string::ParseScanfString(FormatStringHandler &H,
+                                                    const char *I,
+                                                    const char *E) {
+  
+  unsigned argIndex = 0;
+  
+  // Keep looking for a format specifier until we have exhausted the string.
+  while (I != E) {
+    const ScanfSpecifierResult &FSR = ParseScanfSpecifier(H, I, E, argIndex);
+    // Did a fail-stop error of any kind occur when parsing the specifier?
+    // If so, don't do any more processing.
+    if (FSR.shouldStop())
+      return true;;
+      // Did we exhaust the string or encounter an error that
+      // we can recover from?
+    if (!FSR.hasValue())
+      continue;
+      // We have a format specifier.  Pass it to the callback.
+    if (!H.HandleScanfSpecifier(FSR.getValue(), FSR.getStart(),
+                                I - FSR.getStart())) {
+      return true;
+    }
+  }
+  assert(I == E && "Format string not exhausted");
+  return false;
+}
+
+
diff --git a/lib/Analysis/UninitializedValues.cpp b/lib/Analysis/UninitializedValues.cpp
index 7a628642dc99..0f43efa58cc0 100644
--- a/lib/Analysis/UninitializedValues.cpp
+++ b/lib/Analysis/UninitializedValues.cpp
@@ -121,7 +121,7 @@ bool TransferFuncs::VisitBinaryOperator(BinaryOperator* B) {
 
   if (VarDecl* VD = FindBlockVarDecl(B->getLHS()))
     if (B->isAssignmentOp()) {
-      if (B->getOpcode() == BinaryOperator::Assign)
+      if (B->getOpcode() == BO_Assign)
         return V(VD,AD) = Visit(B->getRHS());
       else // Handle +=, -=, *=, etc.  We do want '&', not '&&'.
         return V(VD,AD) = Visit(B->getLHS()) & Visit(B->getRHS());
@@ -168,7 +168,7 @@ bool TransferFuncs::VisitCallExpr(CallExpr* C) {
 
 bool TransferFuncs::VisitUnaryOperator(UnaryOperator* U) {
   switch (U->getOpcode()) {
-    case UnaryOperator::AddrOf: {
+    case UO_AddrOf: {
       VarDecl* VD = FindBlockVarDecl(U->getSubExpr());
       if (VD && VD->isBlockVarDecl())
         return V(VD,AD) = Initialized;
diff --git a/lib/Basic/Builtins.cpp b/lib/Basic/Builtins.cpp
index 1a3293775ed6..040cdb5d55f3 100644
--- a/lib/Basic/Builtins.cpp
+++ b/lib/Basic/Builtins.cpp
@@ -93,3 +93,23 @@ Builtin::Context::isPrintfLike(unsigned ID, unsigned &FormatIdx,
   return true;
 }
 
+// FIXME: Refactor with isPrintfLike.
+bool
+Builtin::Context::isScanfLike(unsigned ID, unsigned &FormatIdx,
+                              bool &HasVAListArg) {
+  const char *Scanf = strpbrk(GetRecord(ID).Attributes, "sS");
+  if (!Scanf)
+    return false;
+
+  HasVAListArg = (*Scanf == 'S');
+
+  ++Scanf;
+  assert(*Scanf == ':' && "s or S specifier must have be followed by a ':'");
+  ++Scanf;
+
+  assert(strchr(Scanf, ':') && "printf specifier must end with a ':'");
+  FormatIdx = strtol(Scanf, 0, 10);
+  return true;
+}
+
+
diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp
index 641d87bb9afa..d8095f4f6d54 100644
--- a/lib/Basic/Diagnostic.cpp
+++ b/lib/Basic/Diagnostic.cpp
@@ -38,6 +38,8 @@ using namespace clang;
 // Builtin Diagnostic information
 //===----------------------------------------------------------------------===//
 
+namespace {
+
 // Diagnostic classes.
 enum {
   CLASS_NOTE       = 0x01,
@@ -59,11 +61,10 @@ struct StaticDiagInfoRec {
   bool operator<(const StaticDiagInfoRec &RHS) const {
     return DiagID < RHS.DiagID;
   }
-  bool operator>(const StaticDiagInfoRec &RHS) const {
-    return DiagID > RHS.DiagID;
-  }
 };
 
+}
+
 static const StaticDiagInfoRec StaticDiagInfo[] = {
 #define DIAG(ENUM,CLASS,DEFAULT_MAPPING,DESC,GROUP,SFINAE, CATEGORY)    \
   { diag::ENUM, DEFAULT_MAPPING, CLASS, SFINAE, CATEGORY, DESC, GROUP },
@@ -244,6 +245,9 @@ static void DummyArgToStringFn(Diagnostic::ArgumentKind AK, intptr_t QT,
 
 
 Diagnostic::Diagnostic(DiagnosticClient *client) : Client(client) {
+  ArgToStringFn = DummyArgToStringFn;
+  ArgToStringCookie = 0;
+
   AllExtensionsSilenced = 0;
   IgnoreAllWarnings = false;
   WarningsAsErrors = false;
@@ -253,26 +257,15 @@ Diagnostic::Diagnostic(DiagnosticClient *client) : Client(client) {
   ShowOverloads = Ovl_All;
   ExtBehavior = Ext_Ignore;
 
-  ErrorOccurred = false;
-  FatalErrorOccurred = false;
   ErrorLimit = 0;
   TemplateBacktraceLimit = 0;
-
-  NumWarnings = 0;
-  NumErrors = 0;
-  NumErrorsSuppressed = 0;
   CustomDiagInfo = 0;
-  CurDiagID = ~0U;
-  LastDiagLevel = Ignored;
-
-  ArgToStringFn = DummyArgToStringFn;
-  ArgToStringCookie = 0;
-
-  DelayedDiagID = 0;
 
   // Set all mappings to 'unset'.
-  DiagMappings BlankDiags(diag::DIAG_UPPER_LIMIT/2, 0);
-  DiagMappingsStack.push_back(BlankDiags);
+  DiagMappingsStack.clear();
+  DiagMappingsStack.push_back(DiagMappings());
+
+  Reset();
 }
 
 Diagnostic::~Diagnostic() {
@@ -331,10 +324,21 @@ bool Diagnostic::isBuiltinExtensionDiag(unsigned DiagID,
       getBuiltinDiagClass(DiagID) != CLASS_EXTENSION)
     return false;
   
-  EnabledByDefault = StaticDiagInfo[DiagID].Mapping != diag::MAP_IGNORE;
+  EnabledByDefault = GetDefaultDiagMapping(DiagID) != diag::MAP_IGNORE;
   return true;
 }
 
+void Diagnostic::Reset() {
+  ErrorOccurred = false;
+  FatalErrorOccurred = false;
+  
+  NumWarnings = 0;
+  NumErrors = 0;
+  NumErrorsSuppressed = 0;
+  CurDiagID = ~0U;
+  LastDiagLevel = Ignored;
+  DelayedDiagID = 0;
+}
 
 /// getDescription - Given a diagnostic ID, return a description of the
 /// issue.
@@ -572,11 +576,11 @@ bool Diagnostic::ProcessDiag() {
   // If a fatal error has already been emitted, silence all subsequent
   // diagnostics.
   if (FatalErrorOccurred) {
-    if (DiagLevel >= Diagnostic::Error) {
+    if (DiagLevel >= Diagnostic::Error && Client->IncludeInDiagnosticCounts()) {
       ++NumErrors;
       ++NumErrorsSuppressed;
     }
-    
+
     return false;
   }
 
@@ -597,9 +601,11 @@ bool Diagnostic::ProcessDiag() {
   }
 
   if (DiagLevel >= Diagnostic::Error) {
-    ErrorOccurred = true;
-    ++NumErrors;
-    
+    if (Client->IncludeInDiagnosticCounts()) {
+      ErrorOccurred = true;
+      ++NumErrors;
+    }
+
     // If we've emitted a lot of errors, emit a fatal error after it to stop a
     // flood of bogus errors.
     if (ErrorLimit && NumErrors >= ErrorLimit &&
@@ -1146,11 +1152,6 @@ void StoredDiagnostic::Serialize(llvm::raw_ostream &OS) const {
       break;
     }
 
-    if (F->InsertionLoc.isValid() && F->InsertionLoc.isMacroID()) {
-      NumFixIts = 0;
-      break;
-    }
-
     ++NumFixIts;
   }
 
@@ -1160,7 +1161,6 @@ void StoredDiagnostic::Serialize(llvm::raw_ostream &OS) const {
     WriteSourceLocation(OS, SM, F->RemoveRange.getBegin());
     WriteSourceLocation(OS, SM, F->RemoveRange.getEnd());
     WriteUnsigned(OS, F->RemoveRange.isTokenRange());
-    WriteSourceLocation(OS, SM, F->InsertionLoc);
     WriteString(OS, F->CodeToInsert);
   }
 }
@@ -1288,12 +1288,11 @@ StoredDiagnostic::Deserialize(FileManager &FM, SourceManager &SM,
   if (ReadUnsigned(Memory, MemoryEnd, NumFixIts))
     return Diag;
   for (unsigned I = 0; I != NumFixIts; ++I) {
-    SourceLocation RemoveBegin, RemoveEnd, InsertionLoc;
+    SourceLocation RemoveBegin, RemoveEnd;
     unsigned InsertLen = 0, RemoveIsTokenRange;
     if (ReadSourceLocation(FM, SM, Memory, MemoryEnd, RemoveBegin) ||
         ReadSourceLocation(FM, SM, Memory, MemoryEnd, RemoveEnd) ||
         ReadUnsigned(Memory, MemoryEnd, RemoveIsTokenRange) ||
-        ReadSourceLocation(FM, SM, Memory, MemoryEnd, InsertionLoc) ||
         ReadUnsigned(Memory, MemoryEnd, InsertLen) ||
         Memory + InsertLen > MemoryEnd) {
       Diag.FixIts.clear();
@@ -1303,7 +1302,6 @@ StoredDiagnostic::Deserialize(FileManager &FM, SourceManager &SM,
     FixItHint Hint;
     Hint.RemoveRange = CharSourceRange(SourceRange(RemoveBegin, RemoveEnd),
                                        RemoveIsTokenRange);
-    Hint.InsertionLoc = InsertionLoc;
     Hint.CodeToInsert.assign(Memory, Memory + InsertLen);
     Memory += InsertLen;
     Diag.FixIts.push_back(Hint);
diff --git a/lib/Basic/FileManager.cpp b/lib/Basic/FileManager.cpp
index 3c91a0f875cc..565f8a61dee6 100644
--- a/lib/Basic/FileManager.cpp
+++ b/lib/Basic/FileManager.cpp
@@ -19,6 +19,7 @@
 
 #include "clang/Basic/FileManager.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/System/Path.h"
 #include "llvm/Config/config.h"
@@ -83,6 +84,9 @@ class FileManager::UniqueFileContainer {
 public:
   FileEntry &getFile(const char *Name, struct stat &StatBuf) {
     std::string FullPath(GetFullPath(Name));
+    
+    // LowercaseString because Windows filesystem is case insensitive.
+    FullPath = llvm::LowercaseString(FullPath);
     return UniqueFiles.GetOrCreateValue(
                                FullPath.c_str(),
                                FullPath.c_str() + FullPath.size()
@@ -365,6 +369,18 @@ FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
   UFE->ModTime = ModificationTime;
   UFE->Dir     = DirInfo;
   UFE->UID     = NextFileUID++;
+  
+  // If this virtual file resolves to a file, also map that file to the 
+  // newly-created file entry.
+  const char *InterndFileName = NamedFileEnt.getKeyData();
+  struct stat StatBuf;
+  if (!stat_cached(InterndFileName, &StatBuf) &&
+      !S_ISDIR(StatBuf.st_mode)) {
+    llvm::sys::Path FilePath(InterndFileName);
+    FilePath.makeAbsolute();
+    FileEntries[FilePath.str()] = UFE;
+  }
+  
   return UFE;
 }
 
diff --git a/lib/Basic/IdentifierTable.cpp b/lib/Basic/IdentifierTable.cpp
index 8993e6713fbe..6b673e39d365 100644
--- a/lib/Basic/IdentifierTable.cpp
+++ b/lib/Basic/IdentifierTable.cpp
@@ -34,6 +34,8 @@ IdentifierInfo::IdentifierInfo() {
   IsPoisoned = false;
   IsCPPOperatorKeyword = false;
   NeedsHandleIdentifier = false;
+  IsFromAST = false;
+  RevertedTokenID = false;
   FETokenInfo = 0;
   Entry = 0;
 }
@@ -71,7 +73,8 @@ namespace {
     KEYMS = 32,
     BOOLSUPPORT = 64,
     KEYALTIVEC = 128,
-    KEYNOMS = 256
+    KEYNOMS = 256,
+    KEYBORLAND = 512
   };
 }
 
@@ -93,6 +96,7 @@ static void AddKeyword(llvm::StringRef Keyword,
   else if (LangOpts.C99 && (Flags & KEYC99)) AddResult = 2;
   else if (LangOpts.GNUKeywords && (Flags & KEYGNU)) AddResult = 1;
   else if (LangOpts.Microsoft && (Flags & KEYMS)) AddResult = 1;
+  else if (LangOpts.Borland && (Flags & KEYBORLAND)) AddResult = 1;
   else if (LangOpts.Bool && (Flags & BOOLSUPPORT)) AddResult = 2;
   else if (LangOpts.AltiVec && (Flags & KEYALTIVEC)) AddResult = 2;
   else if (!LangOpts.Microsoft && (Flags & KEYNOMS)) AddResult = 2;
@@ -100,8 +104,7 @@ static void AddKeyword(llvm::StringRef Keyword,
   // Don't add this keyword if disabled in this language.
   if (AddResult == 0) return;
 
-  IdentifierInfo &Info = Table.get(Keyword);
-  Info.setTokenID(TokenCode);
+  IdentifierInfo &Info = Table.get(Keyword, TokenCode);
   Info.setIsExtensionToken(AddResult == 1);
 }
 
@@ -110,8 +113,7 @@ static void AddKeyword(llvm::StringRef Keyword,
 static void AddCXXOperatorKeyword(llvm::StringRef Keyword,
                                   tok::TokenKind TokenCode,
                                   IdentifierTable &Table) {
-  IdentifierInfo &Info = Table.get(Keyword);
-  Info.setTokenID(TokenCode);
+  IdentifierInfo &Info = Table.get(Keyword, TokenCode);
   Info.setIsCPlusPlusOperatorKeyword();
 }
 
diff --git a/lib/Basic/Makefile b/lib/Basic/Makefile
index 51b8ac1b4569..c15630459567 100644
--- a/lib/Basic/Makefile
+++ b/lib/Basic/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangBasic
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Basic/SourceManager.cpp b/lib/Basic/SourceManager.cpp
index e6d9785e1505..633d86c1eb8a 100644
--- a/lib/Basic/SourceManager.cpp
+++ b/lib/Basic/SourceManager.cpp
@@ -32,7 +32,8 @@ using llvm::MemoryBuffer;
 //===----------------------------------------------------------------------===//
 
 ContentCache::~ContentCache() {
-  delete Buffer.getPointer();
+  if (shouldFreeBuffer())
+    delete Buffer.getPointer();
 }
 
 /// getSizeBytesMapped - Returns the number of bytes actually mapped for
@@ -51,12 +52,14 @@ unsigned ContentCache::getSize() const {
                              : (unsigned) Entry->getSize();
 }
 
-void ContentCache::replaceBuffer(const llvm::MemoryBuffer *B) {
+void ContentCache::replaceBuffer(const llvm::MemoryBuffer *B,
+                                 bool DoNotFree) {
   assert(B != Buffer.getPointer());
   
-  delete Buffer.getPointer();
+  if (shouldFreeBuffer())
+    delete Buffer.getPointer();
   Buffer.setPointer(B);
-  Buffer.setInt(false);
+  Buffer.setInt(DoNotFree? DoNotFreeFlag : 0);
 }
 
 const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
@@ -72,7 +75,6 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
     struct stat FileInfo;
     Buffer.setPointer(MemoryBuffer::getFile(Entry->getName(), &ErrorStr,
                                             Entry->getSize(), &FileInfo));
-    Buffer.setInt(false);
     
     // If we were unable to open the file, then we are in an inconsistent
     // situation where the content cache referenced a file which no longer
@@ -99,7 +101,7 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
         Diag.Report(FullSourceLoc(Loc, SM), diag::err_cannot_open_file)
           << Entry->getName() << ErrorStr;
 
-      Buffer.setInt(true);
+      Buffer.setInt(Buffer.getInt() | InvalidFlag);
 
     // FIXME: This conditionalization is horrible, but we see spurious failures
     // in the test suite due to this warning and no one has had time to hunt it
@@ -119,14 +121,14 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
         Diag.Report(FullSourceLoc(Loc, SM), diag::err_file_modified)
           << Entry->getName();
 
-      Buffer.setInt(true);
+      Buffer.setInt(Buffer.getInt() | InvalidFlag);
 #endif
     }
     
     // If the buffer is valid, check to see if it has a UTF Byte Order Mark
     // (BOM).  We only support UTF-8 without a BOM right now.  See
     // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
-    if (!Buffer.getInt()) {
+    if (!isBufferInvalid()) {
       llvm::StringRef BufStr = Buffer.getPointer()->getBuffer();
       const char *BOM = 0;
       if (BufStr.startswith("\xFE\xBB\xBF"))
@@ -161,7 +163,7 @@ const llvm::MemoryBuffer *ContentCache::getBuffer(Diagnostic &Diag,
   }
   
   if (Invalid)
-    *Invalid = Buffer.getInt();
+    *Invalid = isBufferInvalid();
   
   return Buffer.getPointer();
 }
@@ -422,9 +424,12 @@ void SourceManager::PreallocateSLocEntries(ExternalSLocEntrySource *Source,
                                            unsigned NextOffset) {
   ExternalSLocEntries = Source;
   this->NextOffset = NextOffset;
+  unsigned CurPrealloc = SLocEntryLoaded.size();
+  // If we've ever preallocated, we must not count the dummy entry.
+  if (CurPrealloc) --CurPrealloc;
   SLocEntryLoaded.resize(NumSLocEntries + 1);
   SLocEntryLoaded[0] = true;
-  SLocEntryTable.resize(SLocEntryTable.size() + NumSLocEntries);
+  SLocEntryTable.resize(SLocEntryTable.size() + NumSLocEntries - CurPrealloc);
 }
 
 void SourceManager::ClearPreallocatedSLocEntries() {
@@ -448,7 +453,7 @@ void SourceManager::ClearPreallocatedSLocEntries() {
 // Methods to create new FileID's and instantiations.
 //===----------------------------------------------------------------------===//
 
-/// createFileID - Create a new fileID for the specified ContentCache and
+/// createFileID - Create a new FileID for the specified ContentCache and
 /// include position.  This works regardless of whether the ContentCache
 /// corresponds to a file or some other input source.
 FileID SourceManager::createFileID(const ContentCache *File,
@@ -521,12 +526,13 @@ SourceManager::getMemoryBufferForFile(const FileEntry *File,
 }
 
 bool SourceManager::overrideFileContents(const FileEntry *SourceFile,
-                                         const llvm::MemoryBuffer *Buffer) {
+                                         const llvm::MemoryBuffer *Buffer,
+                                         bool DoNotFree) {
   const SrcMgr::ContentCache *IR = getOrCreateContentCache(SourceFile);
   if (IR == 0)
     return true;
 
-  const_cast<SrcMgr::ContentCache *>(IR)->replaceBuffer(Buffer);
+  const_cast<SrcMgr::ContentCache *>(IR)->replaceBuffer(Buffer, DoNotFree);
   return false;
 }
 
@@ -1241,7 +1247,7 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   }
 
   // There is no common ancestor, most probably because one location is in the
-  // predefines buffer or a PCH file.
+  // predefines buffer or an AST file.
   // FIXME: We should rearrange the external interface so this simply never
   // happens; it can't conceptually happen. Also see PR5662.
   IsBeforeInTUCache.setQueryFIDs(FileID(), FileID()); // Don't try caching.
diff --git a/lib/Basic/TargetInfo.cpp b/lib/Basic/TargetInfo.cpp
index 7fcf372a368c..6d42883cd13c 100644
--- a/lib/Basic/TargetInfo.cpp
+++ b/lib/Basic/TargetInfo.cpp
@@ -58,6 +58,9 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
 
   // Default to no types using fpret.
   RealTypeUsesObjCFPRet = 0;
+
+  // Default to using the Itanium ABI.
+  CXXABI = CXXABI_Itanium;
 }
 
 // Out of line virtual dtor for TargetInfo.
@@ -287,8 +290,15 @@ bool TargetInfo::validateOutputConstraint(ConstraintInfo &Info) const {
       Info.setAllowsRegister();
       Info.setAllowsMemory();
       break;
-    case ',': // FIXME: Until we handle multiple alternative constraints,
-      return true;  // ignore everything after the first comma.
+    case ',': // multiple alternative constraint.  Pass it.
+      Name++;
+      // Handle additional optional '=' or '+' modifiers.
+      if (*Name == '=' || *Name == '+')
+        Name++;
+      break;
+    case '?': // Disparage slightly code.
+    case '!': // Disparage severly.
+      break;  // Pass them.
     }
 
     Name++;
@@ -352,6 +362,7 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
       if (!resolveSymbolicName(Name, OutputConstraints, NumOutputs, Index))
         return false;
 
+      Info.setTiedOperand(Index, OutputConstraints[Index]);
       break;
     }
     case '%': // commutative
@@ -382,8 +393,11 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
       Info.setAllowsRegister();
       Info.setAllowsMemory();
       break;
-    case ',': // FIXME: Until we handle multiple alternative constraints,
-      return true;  // ignore everything after the first comma.
+    case ',': // multiple alternative constraint.  Ignore comma.
+      break;
+    case '?': // Disparage slightly code.
+    case '!': // Disparage severly.
+      break;  // Pass them.
     }
 
     Name++;
diff --git a/lib/Basic/Targets.cpp b/lib/Basic/Targets.cpp
index fdf63e738b53..df20defa0fc0 100644
--- a/lib/Basic/Targets.cpp
+++ b/lib/Basic/Targets.cpp
@@ -502,7 +502,7 @@ public:
               // is therefore only safe to use `m' in an asm statement
               // if that asm statement accesses the operand exactly once.
               // The asm statement must also use `%U<opno>' as a
-              // placeholder for the “update” flag in the corresponding
+              // placeholder for the "update" flag in the corresponding
               // load or store instruction. For example: 
               // asm ("st%U0 %1,%0" : "=m" (mem) : "r" (val));
               // is correct but: 
@@ -512,7 +512,7 @@ public:
     case 'e': 
       if (Name[1] != 's')
           return false;
-              // es: A “stable” memory operand; that is, one which does not
+              // es: A "stable" memory operand; that is, one which does not
               // include any automodification of the base register. Unlike
               // `m', this constraint can be used in asm statements that
               // might access the operand several times, or that might not
@@ -912,11 +912,12 @@ class X86TargetInfo : public TargetInfo {
   } AMD3DNowLevel;
 
   bool HasAES;
-  
+  bool HasAVX;
+
 public:
   X86TargetInfo(const std::string& triple)
     : TargetInfo(triple), SSELevel(NoMMXSSE), AMD3DNowLevel(NoAMD3DNow),
-      HasAES(false) {
+      HasAES(false), HasAVX(false) {
     LongDoubleFormat = &llvm::APFloat::x87DoubleExtended;
   }
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
@@ -963,6 +964,7 @@ void X86TargetInfo::getDefaultFeatures(const std::string &CPU,
   Features["sse41"] = false;
   Features["sse42"] = false;
   Features["aes"] = false;
+  Features["avx"] = false;
 
   // LLVM does not currently recognize this.
   // Features["sse4a"] = false;
@@ -1046,6 +1048,8 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
       Features["3dnow"] = Features["3dnowa"] = true;
     else if (Name == "aes")
       Features["aes"] = true;
+    else if (Name == "avx")
+      Features["avx"] = true;
   } else {
     if (Name == "mmx")
       Features["mmx"] = Features["sse"] = Features["sse2"] = Features["sse3"] =
@@ -1073,6 +1077,8 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
       Features["3dnowa"] = false;
     else if (Name == "aes")
       Features["aes"] = false;
+    else if (Name == "avx")
+      Features["avx"] = false;
   }
 
   return true;
@@ -1092,6 +1098,13 @@ void X86TargetInfo::HandleTargetFeatures(std::vector<std::string> &Features) {
       continue;
     }
 
+    // FIXME: Not sure yet how to treat AVX in regard to SSE levels.
+    // For now let it be enabled together with other SSE levels.
+    if (Features[i].substr(1) == "avx") {
+      HasAVX = true;
+      continue;
+    }
+
     assert(Features[i][0] == '+' && "Invalid target feature!");
     X86SSEEnum Level = llvm::StringSwitch<X86SSEEnum>(Features[i].substr(1))
       .Case("sse42", SSE42)
@@ -1133,6 +1146,9 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasAES)
     Builder.defineMacro("__AES__");
 
+  if (HasAVX)
+    Builder.defineMacro("__AVX__");
+
   // Target properties.
   Builder.defineMacro("__LITTLE_ENDIAN__");
 
@@ -1186,6 +1202,15 @@ X86TargetInfo::validateAsmConstraint(const char *&Name,
                                      TargetInfo::ConstraintInfo &Info) const {
   switch (*Name) {
   default: return false;
+  case 'Y': // first letter of a pair:
+    switch (*(Name+1)) {
+    default: return false;
+    case '0':  // First SSE register.
+    case 't':  // Any SSE register, when SSE2 is enabled.
+    case 'i':  // Any SSE register, when SSE2 and inter-unit moves enabled.
+    case 'm':  // any MMX register, when inter-unit moves enabled.
+      break;   // falls through to setAllowsRegister.
+  }
   case 'a': // eax.
   case 'b': // ebx.
   case 'c': // ecx.
@@ -1193,22 +1218,27 @@ X86TargetInfo::validateAsmConstraint(const char *&Name,
   case 'S': // esi.
   case 'D': // edi.
   case 'A': // edx:eax.
+  case 'f': // any x87 floating point stack register.
   case 't': // top of floating point stack.
   case 'u': // second from top of floating point stack.
   case 'q': // Any register accessible as [r]l: a, b, c, and d.
   case 'y': // Any MMX register.
   case 'x': // Any SSE register.
   case 'Q': // Any register accessible as [r]h: a, b, c, and d.
+  case 'R': // "Legacy" registers: ax, bx, cx, dx, di, si, sp, bp.
+  case 'l': // "Index" registers: any general register that can be used as an
+            // index in a base+index memory access.
+    Info.setAllowsRegister();
+    return true;
+  case 'C': // SSE floating point constant.
+  case 'G': // x87 floating point constant.
   case 'e': // 32-bit signed integer constant for use with zero-extending
             // x86_64 instructions.
   case 'Z': // 32-bit unsigned integer constant for use with zero-extending
             // x86_64 instructions.
-  case 'N': // unsigned 8-bit integer constant for use with in and out
-            // instructions.
-  case 'R': // "legacy" registers: ax, bx, cx, dx, di, si, sp, bp.
-    Info.setAllowsRegister();
     return true;
   }
+  return false;
 }
 
 std::string
@@ -1333,6 +1363,8 @@ public:
     // 300=386, 400=486, 500=Pentium, 600=Blend (default)
     // We lost the original triple, so we use the default.
     Builder.defineMacro("_M_IX86", "600");
+    Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
+    Builder.defineMacro("_STDCALL_SUPPORTED");
   }
 };
 } // end anonymous namespace
@@ -1388,7 +1420,7 @@ public:
     SizeType = UnsignedLong;
     IntPtrType = SignedLong;
     PtrDiffType = SignedLong;
-  }                                       	
+  }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
     X86_32TargetInfo::getTargetDefines(Opts, Builder);
@@ -1447,7 +1479,10 @@ public:
     TLSSupported = false;
     WCharType = UnsignedShort;
     LongWidth = LongAlign = 32;
-    DoubleAlign = LongLongAlign = 64;
+    DoubleAlign = LongLongAlign = 64;      
+    IntMaxType = SignedLongLong;
+    UIntMaxType = UnsignedLongLong;
+    Int64Type = SignedLongLong;
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
@@ -1469,9 +1504,10 @@ public:
                                 MacroBuilder &Builder) const {
     WindowsX86_64TargetInfo::getTargetDefines(Opts, Builder);
     Builder.defineMacro("_M_X64");
+    Builder.defineMacro("_INTEGRAL_MAX_BITS", "64");
   }
   virtual const char *getVAListDeclaration() const {
-    return "typedef char* va_list;";
+    return "typedef char* __builtin_va_list;";
   }
 };
 } // end anonymous namespace
@@ -1566,6 +1602,9 @@ public:
                            "i64:64:64-f32:32:32-f64:64:64-"
                            "v64:64:64-v128:128:128-a0:0:64-n32");
     }
+
+    // ARM targets default to using the ARM C++ ABI.
+    CXXABI = CXXABI_ARM;
   }
   virtual const char *getABI() const { return ABI.c_str(); }
   virtual bool setABI(const std::string &Name) {
@@ -1769,18 +1808,27 @@ public:
 };
 
 const char * const ARMTargetInfo::GCCRegNames[] = {
+  // Integer registers
   "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
-  "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
+  "r8", "r9", "r10", "r11", "r12", "sp", "lr", "pc",
+
+  // Float registers
+  "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+  "s8", "s9", "s10", "s11", "s12", "s13", "s14", "s15",
+  "s16", "s17", "s18", "s19", "s20", "s21", "s22", "s23",
+  "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31"
+
+  // FIXME: Need double and NEON registers, but we need support for aliasing
+  // multiple registers for that.
 };
 
 void ARMTargetInfo::getGCCRegNames(const char * const *&Names,
-                                       unsigned &NumNames) const {
+                                   unsigned &NumNames) const {
   Names = GCCRegNames;
   NumNames = llvm::array_lengthof(GCCRegNames);
 }
 
 const TargetInfo::GCCRegAlias ARMTargetInfo::GCCRegAliases[] = {
-
   { { "a1" }, "r0" },
   { { "a2" }, "r1" },
   { { "a3" }, "r2" },
@@ -1794,9 +1842,9 @@ const TargetInfo::GCCRegAlias ARMTargetInfo::GCCRegAliases[] = {
   { { "sl" }, "r10" },
   { { "fp" }, "r11" },
   { { "ip" }, "r12" },
-  { { "sp" }, "r13" },
-  { { "lr" }, "r14" },
-  { { "pc" }, "r15" },
+  { { "r13" }, "sp" },
+  { { "r14" }, "lr" },
+  { { "r15" }, "pc" },
 };
 
 void ARMTargetInfo::getGCCRegAliases(const GCCRegAlias *&Aliases,
@@ -2603,7 +2651,7 @@ TargetInfo *TargetInfo::CreateTargetInfo(Diagnostic &Diags,
   }
 
   // Set the target C++ ABI.
-  if (!Target->setCXXABI(Opts.CXXABI)) {
+  if (!Opts.CXXABI.empty() && !Target->setCXXABI(Opts.CXXABI)) {
     Diags.Report(diag::err_target_unknown_cxxabi) << Opts.CXXABI;
     return 0;
   }
diff --git a/lib/Basic/Version.cpp b/lib/Basic/Version.cpp
index e0c23362b930..3e6d222b621f 100644
--- a/lib/Basic/Version.cpp
+++ b/lib/Basic/Version.cpp
@@ -21,7 +21,7 @@ using namespace std;
 namespace clang {
   
 llvm::StringRef getClangRepositoryPath() {
-  static const char URL[] = "$URL: http://llvm.org/svn/llvm-project/cfe/trunk/lib/Basic/Version.cpp $";
+  static const char URL[] = "$URL: http://llvm.org/svn/llvm-project/cfe/branches/release_28/lib/Basic/Version.cpp $";
   const char *URLEnd = URL + strlen(URL);
 
   const char *End = strstr(URL, "/lib/Basic");
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index bc2cd460d92b..bd5e342e028f 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -8,6 +8,8 @@ add_subdirectory(CodeGen)
 add_subdirectory(Analysis)
 add_subdirectory(Rewrite)
 add_subdirectory(Driver)
+add_subdirectory(Serialization)
 add_subdirectory(Frontend)
+add_subdirectory(FrontendTool)
 add_subdirectory(Index)
 add_subdirectory(Checker)
diff --git a/lib/Checker/AdjustedReturnValueChecker.cpp b/lib/Checker/AdjustedReturnValueChecker.cpp
index b92f2e705625..0ed04fb14aba 100644
--- a/lib/Checker/AdjustedReturnValueChecker.cpp
+++ b/lib/Checker/AdjustedReturnValueChecker.cpp
@@ -70,8 +70,7 @@ void AdjustedReturnValueChecker::PostVisitCallExpr(CheckerContext &C,
   }
   else if (const BlockDataRegion *BD = dyn_cast<BlockDataRegion>(callee)) {
     const BlockTextRegion *BR = BD->getCodeRegion();
-    const BlockPointerType *BT =
-      BR->getLocationType(C.getASTContext())->getAs<BlockPointerType>();
+    const BlockPointerType *BT=BR->getLocationType()->getAs<BlockPointerType>();
     const FunctionType *FT = BT->getPointeeType()->getAs<FunctionType>();
     actualResultTy = FT->getResultType();
   }
diff --git a/lib/Checker/AggExprVisitor.cpp b/lib/Checker/AggExprVisitor.cpp
index 343afec18d21..6d472f46b45a 100644
--- a/lib/Checker/AggExprVisitor.cpp
+++ b/lib/Checker/AggExprVisitor.cpp
@@ -18,6 +18,13 @@
 using namespace clang;
 
 namespace {
+/// AggExprVisitor is designed after AggExprEmitter of the CodeGen module.  It
+/// is used for evaluating exprs of C++ object type. Evaluating such exprs
+/// requires a destination pointer pointing to the object being evaluated
+/// into. Passing such a pointer around would pollute the Visit* interface of
+/// GRExprEngine. AggExprVisitor encapsulates code that goes through various
+/// cast and construct exprs (and others), and at the final point, dispatches
+/// back to the GRExprEngine to let the real evaluation logic happen.
 class AggExprVisitor : public StmtVisitor<AggExprVisitor> {
   SVal DestPtr;
   ExplodedNode *Pred;
@@ -38,8 +45,8 @@ void AggExprVisitor::VisitCastExpr(CastExpr *E) {
   switch (E->getCastKind()) {
   default: 
     assert(0 && "Unhandled cast kind");
-  case CastExpr::CK_NoOp:
-  case CastExpr::CK_ConstructorConversion:
+  case CK_NoOp:
+  case CK_ConstructorConversion:
     Visit(E->getSubExpr());
     break;
   }
diff --git a/lib/Checker/AnalysisConsumer.cpp b/lib/Checker/AnalysisConsumer.cpp
index 524f37e39662..ad5ccb503b24 100644
--- a/lib/Checker/AnalysisConsumer.cpp
+++ b/lib/Checker/AnalysisConsumer.cpp
@@ -29,6 +29,7 @@
 #include "clang/Checker/PathSensitive/GRTransferFuncs.h"
 #include "clang/Checker/PathDiagnosticClients.h"
 #include "GRExprEngineExperimentalChecks.h"
+#include "GRExprEngineInternalChecks.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/AnalyzerOptions.h"
@@ -173,10 +174,12 @@ public:
     Mgr.reset(new AnalysisManager(*Ctx, PP.getDiagnostics(),
                                   PP.getLangOptions(), PD,
                                   CreateStoreMgr, CreateConstraintMgr,
+                                  /* Indexer */ 0, 
                                   Opts.MaxNodes, Opts.MaxLoop,
                                   Opts.VisualizeEGDot, Opts.VisualizeEGUbi,
                                   Opts.PurgeDead, Opts.EagerlyAssume,
-                                  Opts.TrimGraph, Opts.InlineCall));
+                                  Opts.TrimGraph, Opts.InlineCall,
+                                  Opts.UnoptimizedCFG));
   }
 
   virtual void HandleTranslationUnit(ASTContext &C);
@@ -341,7 +344,10 @@ static void ActionGRExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
   if (C.Opts.EnableExperimentalChecks)
     RegisterExperimentalChecks(Eng);
 
-  if (C.Opts.EnableIdempotentOperationChecker)
+  // Enable idempotent operation checking if it was explicitly turned on, or if
+  // we are running experimental checks (i.e. everything)
+  if (C.Opts.IdempotentOps || C.Opts.EnableExperimentalChecks
+      || C.Opts.EnableExperimentalInternalChecks)
     RegisterIdempotentOperationChecker(Eng);
 
   // Set the graph auditor.
@@ -352,7 +358,7 @@ static void ActionGRExprEngine(AnalysisConsumer &C, AnalysisManager& mgr,
   }
 
   // Execute the worklist algorithm.
-  Eng.ExecuteWorkList(mgr.getStackFrame(D), mgr.getMaxNodes());
+  Eng.ExecuteWorkList(mgr.getStackFrame(D, 0), mgr.getMaxNodes());
 
   // Release the auditor (if any) so that it doesn't monitor the graph
   // created BugReporter.
diff --git a/lib/Checker/AnalysisManager.cpp b/lib/Checker/AnalysisManager.cpp
new file mode 100644
index 000000000000..339cdab80bdb
--- /dev/null
+++ b/lib/Checker/AnalysisManager.cpp
@@ -0,0 +1,31 @@
+//===-- AnalysisManager.cpp -------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Checker/PathSensitive/AnalysisManager.h"
+#include "clang/Index/Entity.h"
+#include "clang/Index/Indexer.h"
+
+using namespace clang;
+
+const AnalysisContext *
+AnalysisManager::getAnalysisContextInAnotherTU(const Decl *D) {
+  idx::Entity Ent = idx::Entity::get(const_cast<Decl *>(D), 
+                                     Idxer->getProgram());
+  FunctionDecl *FuncDef;
+  idx::TranslationUnit *TU;
+  llvm::tie(FuncDef, TU) = Idxer->getDefinitionFor(Ent);
+
+  if (FuncDef == 0)
+    return 0;
+
+  // This AnalysisContext wraps function definition in another translation unit.
+  // But it is still owned by the AnalysisManager associated with the current
+  // translation unit.
+  return AnaCtxMgr.getContext(FuncDef, TU);
+}
diff --git a/lib/Checker/ArrayBoundChecker.cpp b/lib/Checker/ArrayBoundChecker.cpp
index 746b3f95d41e..98345bd70375 100644
--- a/lib/Checker/ArrayBoundChecker.cpp
+++ b/lib/Checker/ArrayBoundChecker.cpp
@@ -58,7 +58,7 @@ void ArrayBoundChecker::VisitLocation(CheckerContext &C, const Stmt *S, SVal l){
   // Get the size of the array.
   DefinedOrUnknownSVal NumElements 
     = C.getStoreManager().getSizeInElements(state, ER->getSuperRegion(), 
-                                         ER->getValueType(C.getASTContext()));
+                                            ER->getValueType());
 
   const GRState *StInBound = state->AssumeInBound(Idx, NumElements, true);
   const GRState *StOutBound = state->AssumeInBound(Idx, NumElements, false);
diff --git a/lib/Checker/BasicObjCFoundationChecks.cpp b/lib/Checker/BasicObjCFoundationChecks.cpp
index ecb2d1c4e34b..3c1a6d1c8282 100644
--- a/lib/Checker/BasicObjCFoundationChecks.cpp
+++ b/lib/Checker/BasicObjCFoundationChecks.cpp
@@ -73,9 +73,6 @@ class BasicObjCFoundationChecks : public GRSimpleAPICheck {
   bool isNSString(const ObjCInterfaceType *T, llvm::StringRef suffix);
   bool AuditNSString(ExplodedNode* N, const ObjCMessageExpr* ME);
 
-  void Warn(ExplodedNode* N, const Expr* E, const std::string& s);
-  void WarnNilArg(ExplodedNode* N, const Expr* E);
-
   bool CheckNilArg(ExplodedNode* N, unsigned Arg);
 
 public:
@@ -358,7 +355,7 @@ bool AuditCFNumberCreate::Audit(ExplodedNode* N,GRStateManager&){
   if (!R)
     return false;
 
-  QualType T = Ctx.getCanonicalType(R->getValueType(Ctx));
+  QualType T = Ctx.getCanonicalType(R->getValueType());
 
   // FIXME: If the pointee isn't an integer type, should we flag a warning?
   //  People can do weird stuff with pointers.
diff --git a/lib/Checker/BasicStore.cpp b/lib/Checker/BasicStore.cpp
index 62c8d9c248ae..f82e1b20be9b 100644
--- a/lib/Checker/BasicStore.cpp
+++ b/lib/Checker/BasicStore.cpp
@@ -52,7 +52,7 @@ public:
   Store InvalidateRegions(Store store, const MemRegion * const *Begin,
                           const MemRegion * const *End, const Expr *E,
                           unsigned Count, InvalidatedSymbols *IS,
-                          bool invalidateGlobals);
+                          bool invalidateGlobals, InvalidatedRegions *Regions);
 
   Store scanForIvars(Stmt *B, const Decl* SelfDecl,
                      const MemRegion *SelfRegion, Store St);
@@ -61,11 +61,6 @@ public:
   Store Remove(Store St, Loc loc);
   Store getInitialStore(const LocationContext *InitLoc);
 
-  // FIXME: Investigate what is using this. This method should be removed.
-  virtual Loc getLoc(const VarDecl* VD, const LocationContext *LC) {
-    return ValMgr.makeLoc(MRMgr.getVarRegion(VD, LC));
-  }
-
   Store BindCompoundLiteral(Store store, const CompoundLiteralExpr*,
                             const LocationContext*, SVal val) {
     return store;
@@ -77,9 +72,8 @@ public:
 
   /// RemoveDeadBindings - Scans a BasicStore of 'state' for dead values.
   ///  It updatees the GRState object in place with the values removed.
-  const GRState *RemoveDeadBindings(GRState &state,
-                                    const StackFrameContext *LCtx,
-                                    SymbolReaper& SymReaper,
+  Store RemoveDeadBindings(Store store, const StackFrameContext *LCtx,
+                           SymbolReaper& SymReaper,
                           llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
 
   void iterBindings(Store store, BindingsHandler& f);
@@ -103,8 +97,6 @@ public:
 
 private:
   SVal LazyRetrieve(Store store, const TypedRegion *R);
-
-  ASTContext& getContext() { return StateMgr.getContext(); }
 };
 
 } // end anonymous namespace
@@ -228,17 +220,15 @@ Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
     return VBFactory.Add(B, R, V).getRoot();
   }
 
-  ASTContext &C = StateMgr.getContext();
-
   // Special case: handle store of pointer values (Loc) to pointers via
   // a cast to intXX_t*, void*, etc.  This is needed to handle
   // OSCompareAndSwap32Barrier/OSCompareAndSwap64Barrier.
   if (isa<Loc>(V) || isa<nonloc::LocAsInteger>(V))
     if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
       // FIXME: Should check for index 0.
-      QualType T = ER->getLocationType(C);
+      QualType T = ER->getLocationType();
 
-      if (isHigherOrderRawPtr(T, C))
+      if (isHigherOrderRawPtr(T, Ctx))
         R = ER->getSuperRegion();
     }
 
@@ -249,7 +239,7 @@ Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
 
   // Do not bind to arrays.  We need to explicitly check for this so that
   // we do not encounter any weirdness of trying to load/store from arrays.
-  if (TyR->isBoundable() && TyR->getValueType(C)->isArrayType())
+  if (TyR->isBoundable() && TyR->getValueType()->isArrayType())
     return store;
 
   if (nonloc::LocAsInteger *X = dyn_cast<nonloc::LocAsInteger>(&V)) {
@@ -259,7 +249,7 @@ Store BasicStoreManager::Bind(Store store, Loc loc, SVal V) {
     // a pointer.  We may wish to flag a type error here if the types
     // are incompatible.  This may also cause lots of breakage
     // elsewhere. Food for thought.
-    if (TyR->isBoundable() && Loc::IsLocType(TyR->getValueType(C)))
+    if (TyR->isBoundable() && Loc::IsLocType(TyR->getValueType()))
       V = X->getLoc();
   }
 
@@ -285,12 +275,11 @@ Store BasicStoreManager::Remove(Store store, Loc loc) {
   }
 }
 
-const GRState *BasicStoreManager::RemoveDeadBindings(GRState &state,
+Store BasicStoreManager::RemoveDeadBindings(Store store,
                                             const StackFrameContext *LCtx,
                                             SymbolReaper& SymReaper,
                            llvm::SmallVectorImpl<const MemRegion*>& RegionRoots)
 {
-  Store store = state.getStore();
   BindingsTy B = GetBindings(store);
   typedef SVal::symbol_iterator symbol_iterator;
 
@@ -365,8 +354,7 @@ const GRState *BasicStoreManager::RemoveDeadBindings(GRState &state,
     }
   }
 
-  state.setStore(store);
-  return StateMgr.getPersistentState(state);
+  return store;
 }
 
 Store BasicStoreManager::scanForIvars(Stmt *B, const Decl* SelfDecl,
@@ -406,10 +394,10 @@ Store BasicStoreManager::getInitialStore(const LocationContext *InitLoc) {
   Store St = VBFactory.GetEmptyMap().getRoot();
 
   for (LVDataTy::decl_iterator I=D.begin_decl(), E=D.end_decl(); I != E; ++I) {
-    NamedDecl* ND = const_cast<NamedDecl*>(I->first);
+    const NamedDecl* ND = I->first;
 
     // Handle implicit parameters.
-    if (ImplicitParamDecl* PD = dyn_cast<ImplicitParamDecl>(ND)) {
+    if (const ImplicitParamDecl* PD = dyn_cast<ImplicitParamDecl>(ND)) {
       const Decl& CD = *InitLoc->getDecl();
       if (const ObjCMethodDecl* MD = dyn_cast<ObjCMethodDecl>(&CD)) {
         if (MD->getSelfDecl() == PD) {
@@ -449,11 +437,11 @@ Store BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
     // will not be called more than once.
 
     // Static global variables should not be visited here.
-    assert(!(VD->getStorageClass() == VarDecl::Static &&
+    assert(!(VD->getStorageClass() == SC_Static &&
              VD->isFileVarDecl()));
 
     // Process static variables.
-    if (VD->getStorageClass() == VarDecl::Static) {
+    if (VD->getStorageClass() == SC_Static) {
       // C99: 6.7.8 Initialization
       //  If an object that has static storage duration is not initialized
       //  explicitly, then:
@@ -465,12 +453,9 @@ Store BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
         if (Loc::IsLocType(T))
           store = Bind(store, loc::MemRegionVal(VR),
                        loc::ConcreteInt(BasicVals.getValue(0, T)));
-        else if (T->isIntegerType())
+        else if (T->isIntegerType() && T->isScalarType())
           store = Bind(store, loc::MemRegionVal(VR),
                        nonloc::ConcreteInt(BasicVals.getValue(0, T)));
-        else {
-          // assert(0 && "ignore other types of variables");
-        }
       } else {
         store = Bind(store, loc::MemRegionVal(VR), *InitVal);
       }
@@ -478,7 +463,8 @@ Store BasicStoreManager::BindDeclInternal(Store store, const VarRegion* VR,
   } else {
     // Process local scalar variables.
     QualType T = VD->getType();
-    if (ValMgr.getSymbolManager().canSymbolicate(T)) {
+    // BasicStore only supports scalars.
+    if (T->isScalarType() && ValMgr.getSymbolManager().canSymbolicate(T)) {
       SVal V = InitVal ? *InitVal : UndefinedVal();
       store = Bind(store, loc::MemRegionVal(VR), V);
     }
@@ -523,11 +509,12 @@ StoreManager::BindingsHandler::~BindingsHandler() {}
 
 
 Store BasicStoreManager::InvalidateRegions(Store store,
-                                      const MemRegion * const *I,
-                                      const MemRegion * const *End,
-                                      const Expr *E, unsigned Count,
-                                      InvalidatedSymbols *IS,
-                                      bool invalidateGlobals) {
+                                           const MemRegion * const *I,
+                                           const MemRegion * const *End,
+                                           const Expr *E, unsigned Count,
+                                           InvalidatedSymbols *IS,
+                                           bool invalidateGlobals,
+                                           InvalidatedRegions *Regions) {
   if (invalidateGlobals) {
     BindingsTy B = GetBindings(store);
     for (BindingsTy::iterator I=B.begin(), End=B.end(); I != End; ++I) {
@@ -545,6 +532,8 @@ Store BasicStoreManager::InvalidateRegions(Store store,
         continue;
     }
     store = InvalidateRegion(store, *I, E, Count, IS);
+    if (Regions)
+      Regions->push_back(R);
   }
 
   // FIXME: This is copy-and-paste from RegionStore.cpp.
@@ -558,6 +547,8 @@ Store BasicStoreManager::InvalidateRegions(Store store,
                                   Count);
 
     store = Bind(store, loc::MemRegionVal(GS), V);
+    if (Regions)
+      Regions->push_back(GS);
   }
 
   return store;
@@ -582,7 +573,7 @@ Store BasicStoreManager::InvalidateRegion(Store store,
     }
   }
 
-  QualType T = cast<TypedRegion>(R)->getValueType(R->getContext());
+  QualType T = cast<TypedRegion>(R)->getValueType();
   SVal V = ValMgr.getConjuredSymbolVal(R, E, T, Count);
   return Bind(store, loc::MemRegionVal(R), V);
 }
diff --git a/lib/Checker/BasicValueFactory.cpp b/lib/Checker/BasicValueFactory.cpp
index 246beead1208..4c9b109c8823 100644
--- a/lib/Checker/BasicValueFactory.cpp
+++ b/lib/Checker/BasicValueFactory.cpp
@@ -149,22 +149,22 @@ BasicValueFactory::EvaluateAPSInt(BinaryOperator::Opcode Op,
     default:
       assert (false && "Invalid Opcode.");
 
-    case BinaryOperator::Mul:
+    case BO_Mul:
       return &getValue( V1 * V2 );
 
-    case BinaryOperator::Div:
+    case BO_Div:
       return &getValue( V1 / V2 );
 
-    case BinaryOperator::Rem:
+    case BO_Rem:
       return &getValue( V1 % V2 );
 
-    case BinaryOperator::Add:
+    case BO_Add:
       return &getValue( V1 + V2 );
 
-    case BinaryOperator::Sub:
+    case BO_Sub:
       return &getValue( V1 - V2 );
 
-    case BinaryOperator::Shl: {
+    case BO_Shl: {
 
       // FIXME: This logic should probably go higher up, where we can
       // test these conditions symbolically.
@@ -182,7 +182,7 @@ BasicValueFactory::EvaluateAPSInt(BinaryOperator::Opcode Op,
       return &getValue( V1.operator<<( (unsigned) Amt ));
     }
 
-    case BinaryOperator::Shr: {
+    case BO_Shr: {
 
       // FIXME: This logic should probably go higher up, where we can
       // test these conditions symbolically.
@@ -200,33 +200,33 @@ BasicValueFactory::EvaluateAPSInt(BinaryOperator::Opcode Op,
       return &getValue( V1.operator>>( (unsigned) Amt ));
     }
 
-    case BinaryOperator::LT:
+    case BO_LT:
       return &getTruthValue( V1 < V2 );
 
-    case BinaryOperator::GT:
+    case BO_GT:
       return &getTruthValue( V1 > V2 );
 
-    case BinaryOperator::LE:
+    case BO_LE:
       return &getTruthValue( V1 <= V2 );
 
-    case BinaryOperator::GE:
+    case BO_GE:
       return &getTruthValue( V1 >= V2 );
 
-    case BinaryOperator::EQ:
+    case BO_EQ:
       return &getTruthValue( V1 == V2 );
 
-    case BinaryOperator::NE:
+    case BO_NE:
       return &getTruthValue( V1 != V2 );
 
       // Note: LAnd, LOr, Comma are handled specially by higher-level logic.
 
-    case BinaryOperator::And:
+    case BO_And:
       return &getValue( V1 & V2 );
 
-    case BinaryOperator::Or:
+    case BO_Or:
       return &getValue( V1 | V2 );
 
-    case BinaryOperator::Xor:
+    case BO_Xor:
       return &getValue( V1 ^ V2 );
   }
 }
diff --git a/lib/Checker/BugReporter.cpp b/lib/Checker/BugReporter.cpp
index 0422d80ae26d..bffbd52b7d88 100644
--- a/lib/Checker/BugReporter.cpp
+++ b/lib/Checker/BugReporter.cpp
@@ -94,8 +94,8 @@ static const Stmt* GetNextStmt(const ExplodedNode* N) {
         case Stmt::ChooseExprClass:
         case Stmt::ConditionalOperatorClass: continue;
         case Stmt::BinaryOperatorClass: {
-          BinaryOperator::Opcode Op = cast<BinaryOperator>(S)->getOpcode();
-          if (Op == BinaryOperator::LAnd || Op == BinaryOperator::LOr)
+          BinaryOperatorKind Op = cast<BinaryOperator>(S)->getOpcode();
+          if (Op == BO_LAnd || Op == BO_LOr)
             continue;
           break;
         }
@@ -177,18 +177,9 @@ public:
   }
 
   virtual NodeMapClosure& getNodeResolver() { return NMC; }
-  BugReport& getReport() { return *R; }
 
   PathDiagnosticLocation getEnclosingStmtLocation(const Stmt *S);
 
-  PathDiagnosticLocation
-  getEnclosingStmtLocation(const PathDiagnosticLocation &L) {
-    if (const Stmt *S = L.asStmt())
-      return getEnclosingStmtLocation(S);
-
-    return L;
-  }
-
   PathDiagnosticClient::PathGenerationScheme getGenerationScheme() const {
     return PDC ? PDC->getGenerationScheme() : PathDiagnosticClient::Extensive;
   }
@@ -541,9 +532,9 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
     ProgramPoint P = N->getLocation();
 
     if (const BlockEdge* BE = dyn_cast<BlockEdge>(&P)) {
-      CFGBlock* Src = BE->getSrc();
-      CFGBlock* Dst = BE->getDst();
-      Stmt* T = Src->getTerminator();
+      const CFGBlock* Src = BE->getSrc();
+      const CFGBlock* Dst = BE->getDst();
+      const Stmt* T = Src->getTerminator();
 
       if (!T)
         continue;
@@ -577,7 +568,7 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           std::string sbuf;
           llvm::raw_string_ostream os(sbuf);
 
-          if (Stmt* S = Dst->getLabel()) {
+          if (const Stmt* S = Dst->getLabel()) {
             PathDiagnosticLocation End(S, SMgr);
 
             switch (S->getStmtClass()) {
@@ -593,17 +584,17 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
 
               case Stmt::CaseStmtClass: {
                 os << "Control jumps to 'case ";
-                CaseStmt* Case = cast<CaseStmt>(S);
-                Expr* LHS = Case->getLHS()->IgnoreParenCasts();
+                const CaseStmt* Case = cast<CaseStmt>(S);
+                const Expr* LHS = Case->getLHS()->IgnoreParenCasts();
 
                 // Determine if it is an enum.
                 bool GetRawInt = true;
 
-                if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(LHS)) {
+                if (const DeclRefExpr* DR = dyn_cast<DeclRefExpr>(LHS)) {
                   // FIXME: Maybe this should be an assertion.  Are there cases
                   // were it is not an EnumConstantDecl?
-                  EnumConstantDecl* D =
-                  dyn_cast<EnumConstantDecl>(DR->getDecl());
+                  const EnumConstantDecl* D =
+                    dyn_cast<EnumConstantDecl>(DR->getDecl());
 
                   if (D) {
                     GetRawInt = false;
@@ -668,12 +659,12 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           if (!PDB.supportsLogicalOpControlFlow())
             break;
 
-          BinaryOperator *B = cast<BinaryOperator>(T);
+          const BinaryOperator *B = cast<BinaryOperator>(T);
           std::string sbuf;
           llvm::raw_string_ostream os(sbuf);
           os << "Left side of '";
 
-          if (B->getOpcode() == BinaryOperator::LAnd) {
+          if (B->getOpcode() == BO_LAnd) {
             os << "&&" << "' is ";
 
             if (*(Src->succ_begin()+1) == Dst) {
@@ -692,7 +683,7 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
             }
           }
           else {
-            assert(B->getOpcode() == BinaryOperator::LOr);
+            assert(B->getOpcode() == BO_LOr);
             os << "||" << "' is ";
 
             if (*(Src->succ_begin()+1) == Dst) {
@@ -902,8 +893,6 @@ class EdgeBuilder {
     CLocs.pop_back();
   }
 
-  PathDiagnosticLocation IgnoreParens(const PathDiagnosticLocation &L);
-
 public:
   EdgeBuilder(PathDiagnostic &pd, PathDiagnosticBuilder &pdb)
     : PD(pd), PDB(pdb) {
@@ -935,10 +924,6 @@ public:
 
   void addEdge(PathDiagnosticLocation NewLoc, bool alwaysAdd = false);
 
-  void addEdge(const Stmt *S, bool alwaysAdd = false) {
-    addEdge(PathDiagnosticLocation(S, PDB.getSourceManager()), alwaysAdd);
-  }
-
   void rawAddEdge(PathDiagnosticLocation NewLoc);
 
   void addContext(const Stmt *S);
@@ -1006,14 +991,6 @@ bool EdgeBuilder::containsLocation(const PathDiagnosticLocation &Container,
            SM.getInstantiationColumnNumber(ContainerREnd)));
 }
 
-PathDiagnosticLocation
-EdgeBuilder::IgnoreParens(const PathDiagnosticLocation &L) {
-  if (const Expr* E = dyn_cast_or_null<Expr>(L.asStmt()))
-      return PathDiagnosticLocation(E->IgnoreParenCasts(),
-                                    PDB.getSourceManager());
-  return L;
-}
-
 void EdgeBuilder::rawAddEdge(PathDiagnosticLocation NewLoc) {
   if (!PrevLoc.isValid()) {
     PrevLoc = NewLoc;
diff --git a/lib/Checker/BugReporterVisitors.cpp b/lib/Checker/BugReporterVisitors.cpp
index 776e12bd2ae4..91cf349107ca 100644
--- a/lib/Checker/BugReporterVisitors.cpp
+++ b/lib/Checker/BugReporterVisitors.cpp
@@ -31,7 +31,7 @@ const Stmt *clang::bugreporter::GetDerefExpr(const ExplodedNode *N) {
   const Stmt *S = N->getLocationAs<PostStmt>()->getStmt();
 
   if (const UnaryOperator *U = dyn_cast<UnaryOperator>(S)) {
-    if (U->getOpcode() == UnaryOperator::Deref)
+    if (U->getOpcode() == UO_Deref)
       return U->getSubExpr()->IgnoreParenCasts();
   }
   else if (const MemberExpr *ME = dyn_cast<MemberExpr>(S)) {
@@ -143,10 +143,9 @@ public:
 
         if (isa<loc::ConcreteInt>(V)) {
           bool b = false;
-          ASTContext &C = BRC.getASTContext();
           if (R->isBoundable()) {
             if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
-              if (TR->getValueType(C)->isObjCObjectPointerType()) {
+              if (TR->getValueType()->isObjCObjectPointerType()) {
                 os << "initialized to nil";
                 b = true;
               }
@@ -174,10 +173,9 @@ public:
     if (os.str().empty()) {
       if (isa<loc::ConcreteInt>(V)) {
         bool b = false;
-        ASTContext &C = BRC.getASTContext();
         if (R->isBoundable()) {
           if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
-            if (TR->getValueType(C)->isObjCObjectPointerType()) {
+            if (TR->getValueType()->isObjCObjectPointerType()) {
               os << "nil object reference stored to ";
               b = true;
             }
@@ -209,7 +207,7 @@ public:
     ProgramPoint P = N->getLocation();
 
     if (BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-      CFGBlock *BSrc = BE->getSrc();
+      const CFGBlock *BSrc = BE->getSrc();
       S = BSrc->getTerminatorCondition();
     }
     else if (PostStmt *PS = dyn_cast<PostStmt>(&P)) {
@@ -282,7 +280,7 @@ public:
       ProgramPoint P = N->getLocation();
 
       if (BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-        CFGBlock *BSrc = BE->getSrc();
+        const CFGBlock *BSrc = BE->getSrc();
         S = BSrc->getTerminatorCondition();
       }
       else if (PostStmt *PS = dyn_cast<PostStmt>(&P)) {
@@ -421,3 +419,40 @@ public:
 void clang::bugreporter::registerNilReceiverVisitor(BugReporterContext &BRC) {
   BRC.addVisitor(new NilReceiverVisitor());
 }
+
+// Registers every VarDecl inside a Stmt with a last store vistor.
+void clang::bugreporter::registerVarDeclsLastStore(BugReporterContext &BRC,
+                                                   const void *stmt,
+                                                   const ExplodedNode *N) {
+  const Stmt *S = static_cast<const Stmt *>(stmt);
+
+  std::deque<const Stmt *> WorkList;
+
+  WorkList.push_back(S);
+
+  while (!WorkList.empty()) {
+    const Stmt *Head = WorkList.front();
+    WorkList.pop_front();
+
+    GRStateManager &StateMgr = BRC.getStateManager();
+    const GRState *state = N->getState();
+
+    if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Head)) {
+      if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
+        const VarRegion *R =
+        StateMgr.getRegionManager().getVarRegion(VD, N->getLocationContext());
+
+        // What did we load?
+        SVal V = state->getSVal(S);
+
+        if (isa<loc::ConcreteInt>(V) || isa<nonloc::ConcreteInt>(V)) {
+          ::registerFindLastStore(BRC, R, V);
+        }
+      }
+    }
+
+    for (Stmt::const_child_iterator I = Head->child_begin();
+        I != Head->child_end(); ++I)
+      WorkList.push_back(*I);
+  }
+}
diff --git a/lib/Checker/CFRefCount.cpp b/lib/Checker/CFRefCount.cpp
index 3c74cd8f9b27..6fa48b2923fe 100644
--- a/lib/Checker/CFRefCount.cpp
+++ b/lib/Checker/CFRefCount.cpp
@@ -82,8 +82,7 @@ public:
 
 static const ObjCMethodDecl*
 ResolveToInterfaceMethodDecl(const ObjCMethodDecl *MD) {
-  ObjCInterfaceDecl *ID =
-    const_cast<ObjCInterfaceDecl*>(MD->getClassInterface());
+  const ObjCInterfaceDecl *ID = MD->getClassInterface();
 
   return MD->isInstanceMethod()
          ? ID->lookupInstanceMethod(MD->getSelector())
@@ -93,11 +92,11 @@ ResolveToInterfaceMethodDecl(const ObjCMethodDecl *MD) {
 namespace {
 class GenericNodeBuilder {
   GRStmtNodeBuilder *SNB;
-  Stmt *S;
+  const Stmt *S;
   const void *tag;
   GREndPathNodeBuilder *ENB;
 public:
-  GenericNodeBuilder(GRStmtNodeBuilder &snb, Stmt *s,
+  GenericNodeBuilder(GRStmtNodeBuilder &snb, const Stmt *s,
                      const void *t)
   : SNB(&snb), S(s), tag(t), ENB(0) {}
 
@@ -195,12 +194,6 @@ public:
   static RetEffect MakeNoRet() {
     return RetEffect(NoRet);
   }
-
-  void Profile(llvm::FoldingSetNodeID& ID) const {
-    ID.AddInteger((unsigned)K);
-    ID.AddInteger((unsigned)O);
-    ID.AddInteger(index);
-  }
 };
 
 //===----------------------------------------------------------------------===//
@@ -239,9 +232,6 @@ private:
   RefVal(Kind k, RetEffect::ObjKind o, unsigned cnt, unsigned acnt, QualType t)
   : kind(k), okind(o), Cnt(cnt), ACnt(acnt), T(t) {}
 
-  RefVal(Kind k, unsigned cnt = 0)
-  : kind(k), okind(RetEffect::AnyObj), Cnt(cnt), ACnt(0) {}
-
 public:
   Kind getKind() const { return kind; }
 
@@ -256,12 +246,6 @@ public:
 
   QualType getType() const { return T; }
 
-  // Useful predicates.
-
-  static bool isError(Kind k) { return k >= ERROR_START; }
-
-  static bool isLeak(Kind k) { return k >= ERROR_LEAK_START; }
-
   bool isOwned() const {
     return getKind() == Owned;
   }
@@ -278,11 +262,6 @@ public:
     return getKind() == ReturnedNotOwned;
   }
 
-  bool isNonLeakError() const {
-    Kind k = getKind();
-    return isError(k) && !isLeak(k);
-  }
-
   static RefVal makeOwned(RetEffect::ObjKind o, QualType t,
                           unsigned Count = 1) {
     return RefVal(Owned, o, Count, 0, t);
@@ -474,11 +453,6 @@ public:
     DefaultArgEffect = E;
   }
 
-  /// setArg - Set the argument effect on the argument specified by idx.
-  void setArgEffect(ArgEffects::Factory& AF, unsigned idx, ArgEffect E) {
-    Args = AF.Add(Args, idx, E);
-  }
-
   /// getRetEffect - Returns the effect on the return value of the call.
   RetEffect getRetEffect() const { return Ret; }
 
@@ -492,28 +466,6 @@ public:
   /// getReceiverEffect - Returns the effect on the receiver of the call.
   ///  This is only meaningful if the summary applies to an ObjCMessageExpr*.
   ArgEffect getReceiverEffect() const { return Receiver; }
-
-  /// setReceiverEffect - Set the effect on the receiver of the call.
-  void setReceiverEffect(ArgEffect E) { Receiver = E; }
-
-  typedef ArgEffects::iterator ExprIterator;
-
-  ExprIterator begin_args() const { return Args.begin(); }
-  ExprIterator end_args()   const { return Args.end(); }
-
-  static void Profile(llvm::FoldingSetNodeID& ID, ArgEffects A,
-                      RetEffect RetEff, ArgEffect DefaultEff,
-                      ArgEffect ReceiverEff, bool EndPath) {
-    ID.Add(A);
-    ID.Add(RetEff);
-    ID.AddInteger((unsigned) DefaultEff);
-    ID.AddInteger((unsigned) ReceiverEff);
-    ID.AddInteger((unsigned) EndPath);
-  }
-
-  void Profile(llvm::FoldingSetNodeID& ID) const {
-    Profile(ID, Args, Ret, DefaultArgEffect, Receiver, EndPath);
-  }
 };
 } // end anonymous namespace
 
@@ -618,11 +570,6 @@ public:
     return Summ;
   }
 
-
-  RetainSummary* find(Expr* Receiver, Selector S) {
-    return find(getReceiverDecl(Receiver), S);
-  }
-
   RetainSummary* find(IdentifierInfo* II, Selector S) {
     // FIXME: Class method lookup.  Right now we dont' have a good way
     // of going between IdentifierInfo* and the class hierarchy.
@@ -634,47 +581,6 @@ public:
     return I == M.end() ? NULL : I->second;
   }
 
-  const ObjCInterfaceDecl* getReceiverDecl(Expr* E) {
-    if (const ObjCObjectPointerType* PT =
-        E->getType()->getAs<ObjCObjectPointerType>())
-      return PT->getInterfaceDecl();
-
-    return NULL;
-  }
-
-  RetainSummary*& operator[](ObjCMessageExpr* ME) {
-
-    Selector S = ME->getSelector();
-
-    const ObjCInterfaceDecl* OD = 0;
-    bool IsInstanceMessage = false;
-    switch (ME->getReceiverKind()) {
-    case ObjCMessageExpr::Instance:
-      OD = getReceiverDecl(ME->getInstanceReceiver());
-      IsInstanceMessage = true;
-      break;
-
-    case ObjCMessageExpr::SuperInstance:
-      IsInstanceMessage = true;
-      OD = ME->getSuperType()->getAs<ObjCObjectPointerType>()
-                                                        ->getInterfaceDecl();
-      break;
-
-    case ObjCMessageExpr::Class:
-      OD = ME->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
-      break;
-
-    case ObjCMessageExpr::SuperClass:
-      OD = ME->getSuperType()->getAs<ObjCObjectType>()->getInterface();
-      break;
-    }
-
-    if (IsInstanceMessage)
-      return OD ? M[ObjCSummaryKey(OD->getIdentifier(), S)] : M[S];
-
-    return M[ObjCSummaryKey(OD->getIdentifier(), S)];
-  }
-
   RetainSummary*& operator[](ObjCSummaryKey K) {
     return M[K];
   }
@@ -696,7 +602,7 @@ class RetainSummaryManager {
   //  Typedefs.
   //==-----------------------------------------------------------------==//
 
-  typedef llvm::DenseMap<FunctionDecl*, RetainSummary*>
+  typedef llvm::DenseMap<const FunctionDecl*, RetainSummary*>
           FuncSummariesTy;
 
   typedef ObjCSummaryCache ObjCMethodSummariesTy;
@@ -766,9 +672,10 @@ public:
 
   RetainSummary* getUnarySummary(const FunctionType* FT, UnaryFuncKind func);
 
-  RetainSummary* getCFSummaryCreateRule(FunctionDecl* FD);
-  RetainSummary* getCFSummaryGetRule(FunctionDecl* FD);
-  RetainSummary* getCFCreateGetRuleSummary(FunctionDecl* FD, StringRef FName);
+  RetainSummary* getCFSummaryCreateRule(const FunctionDecl* FD);
+  RetainSummary* getCFSummaryGetRule(const FunctionDecl* FD);
+  RetainSummary* getCFCreateGetRuleSummary(const FunctionDecl* FD, 
+                                           StringRef FName);
 
   RetainSummary* getPersistentSummary(ArgEffects AE, RetEffect RetEff,
                                       ArgEffect ReceiverEff = DoNothing,
@@ -796,12 +703,6 @@ public:
   void InitializeClassMethodSummaries();
   void InitializeMethodSummaries();
 private:
-
-  void addClsMethSummary(IdentifierInfo* ClsII, Selector S,
-                         RetainSummary* Summ) {
-    ObjCClassMethodSummaries[ObjCSummaryKey(ClsII, S)] = Summ;
-  }
-
   void addNSObjectClsMethSummary(Selector S, RetainSummary *Summ) {
     ObjCClassMethodSummaries[S] = Summ;
   }
@@ -892,7 +793,7 @@ public:
 
   ~RetainSummaryManager();
 
-  RetainSummary* getSummary(FunctionDecl* FD);
+  RetainSummary* getSummary(const FunctionDecl* FD);
 
   RetainSummary *getInstanceMethodSummary(const ObjCMessageExpr *ME,
                                           const GRState *state,
@@ -999,15 +900,15 @@ RetainSummaryManager::getPersistentSummary(ArgEffects AE, RetEffect RetEff,
 // Summary creation for functions (largely uses of Core Foundation).
 //===----------------------------------------------------------------------===//
 
-static bool isRetain(FunctionDecl* FD, StringRef FName) {
+static bool isRetain(const FunctionDecl* FD, StringRef FName) {
   return FName.endswith("Retain");
 }
 
-static bool isRelease(FunctionDecl* FD, StringRef FName) {
+static bool isRelease(const FunctionDecl* FD, StringRef FName) {
   return FName.endswith("Release");
 }
 
-RetainSummary* RetainSummaryManager::getSummary(FunctionDecl* FD) {
+RetainSummary* RetainSummaryManager::getSummary(const FunctionDecl* FD) {
   // Look up a summary in our cache of FunctionDecls -> Summaries.
   FuncSummariesTy::iterator I = FuncSummaries.find(FD);
   if (I != FuncSummaries.end())
@@ -1201,7 +1102,7 @@ RetainSummary* RetainSummaryManager::getSummary(FunctionDecl* FD) {
 }
 
 RetainSummary*
-RetainSummaryManager::getCFCreateGetRuleSummary(FunctionDecl* FD,
+RetainSummaryManager::getCFCreateGetRuleSummary(const FunctionDecl* FD,
                                                 StringRef FName) {
 
   if (FName.find("Create") != StringRef::npos ||
@@ -1250,7 +1151,8 @@ RetainSummaryManager::getUnarySummary(const FunctionType* FT,
   }
 }
 
-RetainSummary* RetainSummaryManager::getCFSummaryCreateRule(FunctionDecl* FD) {
+RetainSummary* 
+RetainSummaryManager::getCFSummaryCreateRule(const FunctionDecl* FD) {
   assert (ScratchArgs.isEmpty());
 
   if (FD->getIdentifier() == CFDictionaryCreateII) {
@@ -1261,7 +1163,8 @@ RetainSummary* RetainSummaryManager::getCFSummaryCreateRule(FunctionDecl* FD) {
   return getPersistentSummary(RetEffect::MakeOwned(RetEffect::CF, true));
 }
 
-RetainSummary* RetainSummaryManager::getCFSummaryGetRule(FunctionDecl* FD) {
+RetainSummary* 
+RetainSummaryManager::getCFSummaryGetRule(const FunctionDecl* FD) {
   assert (ScratchArgs.isEmpty());
   return getPersistentSummary(RetEffect::MakeNotOwned(RetEffect::CF),
                               DoNothing, DoNothing);
@@ -1767,7 +1670,7 @@ private:
 
   void ProcessNonLeakError(ExplodedNodeSet& Dst,
                            GRStmtNodeBuilder& Builder,
-                           Expr* NodeExpr, SourceRange ErrorRange,
+                           const Expr* NodeExpr, SourceRange ErrorRange,
                            ExplodedNode* Pred,
                            const GRState* St,
                            RefVal::Kind hasErr, SymbolRef Sym);
@@ -1810,33 +1713,26 @@ public:
   void EvalSummary(ExplodedNodeSet& Dst,
                    GRExprEngine& Eng,
                    GRStmtNodeBuilder& Builder,
-                   Expr* Ex,
+                   const Expr* Ex,
                    InstanceReceiver Receiver,
                    const RetainSummary& Summ,
                    const MemRegion *Callee,
-                   ExprIterator arg_beg, ExprIterator arg_end,
+                   ConstExprIterator arg_beg, ConstExprIterator arg_end,
                    ExplodedNode* Pred, const GRState *state);
 
   virtual void EvalCall(ExplodedNodeSet& Dst,
                         GRExprEngine& Eng,
                         GRStmtNodeBuilder& Builder,
-                        CallExpr* CE, SVal L,
+                        const CallExpr* CE, SVal L,
                         ExplodedNode* Pred);
 
 
   virtual void EvalObjCMessageExpr(ExplodedNodeSet& Dst,
                                    GRExprEngine& Engine,
                                    GRStmtNodeBuilder& Builder,
-                                   ObjCMessageExpr* ME,
+                                   const ObjCMessageExpr* ME,
                                    ExplodedNode* Pred,
                                    const GRState *state);
-
-  bool EvalObjCMessageExprAux(ExplodedNodeSet& Dst,
-                              GRExprEngine& Engine,
-                              GRStmtNodeBuilder& Builder,
-                              ObjCMessageExpr* ME,
-                              ExplodedNode* Pred);
-
   // Stores.
   virtual void EvalBind(GRStmtNodeBuilderRef& B, SVal location, SVal val);
 
@@ -1861,7 +1757,7 @@ public:
   virtual void EvalReturn(ExplodedNodeSet& Dst,
                           GRExprEngine& Engine,
                           GRStmtNodeBuilder& Builder,
-                          ReturnStmt* S,
+                          const ReturnStmt* S,
                           ExplodedNode* Pred);
 
   // Assumptions.
@@ -1934,7 +1830,6 @@ namespace {
   public:
 
     CFRefCount& getTF() { return TF; }
-    const CFRefCount& getTF() const { return TF; }
 
     // FIXME: Eventually remove.
     virtual const char* getDescription() const = 0;
@@ -2049,9 +1944,6 @@ namespace {
     CFRefBug& getBugType() {
       return (CFRefBug&) RangedBugReport::getBugType();
     }
-    const CFRefBug& getBugType() const {
-      return (const CFRefBug&) RangedBugReport::getBugType();
-    }
 
     virtual void getRanges(const SourceRange*& beg, const SourceRange*& end) {
       if (!getBugType().isLeak())
@@ -2605,11 +2497,12 @@ static QualType GetReturnType(const Expr* RetE, ASTContext& Ctx) {
 void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
                              GRExprEngine& Eng,
                              GRStmtNodeBuilder& Builder,
-                             Expr* Ex,
+                             const Expr* Ex,
                              InstanceReceiver Receiver,
                              const RetainSummary& Summ,
                              const MemRegion *Callee,
-                             ExprIterator arg_beg, ExprIterator arg_end,
+                             ConstExprIterator arg_beg, 
+                             ConstExprIterator arg_end,
                              ExplodedNode* Pred, const GRState *state) {
 
   // Evaluate the effect of the arguments.
@@ -2620,19 +2513,25 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
 
   llvm::SmallVector<const MemRegion*, 10> RegionsToInvalidate;
 
-  for (ExprIterator I = arg_beg; I != arg_end; ++I, ++idx) {
+  // HACK: Symbols that have ref-count state that are referenced directly
+  //  (not as structure or array elements, or via bindings) by an argument
+  //  should not have their ref-count state stripped after we have
+  //  done an invalidation pass.
+  llvm::DenseSet<SymbolRef> WhitelistedSymbols;
+
+  for (ConstExprIterator I = arg_beg; I != arg_end; ++I, ++idx) {
     SVal V = state->getSValAsScalarOrLoc(*I);
     SymbolRef Sym = V.getAsLocSymbol();
 
     if (Sym)
       if (RefBindings::data_type* T = state->get<RefBindings>(Sym)) {
+        WhitelistedSymbols.insert(Sym);
         state = Update(state, Sym, *T, Summ.getArg(idx), hasErr);
         if (hasErr) {
           ErrorRange = (*I)->getSourceRange();
           ErrorSym = Sym;
           break;
         }
-        continue;
       }
 
   tryAgain:
@@ -2703,22 +2602,22 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
   //  expression (the context) and the expression itself.  This should
   //  disambiguate conjured symbols.
   unsigned Count = Builder.getCurrentBlockCount();
-  StoreManager& StoreMgr = Eng.getStateManager().getStoreManager();
   StoreManager::InvalidatedSymbols IS;
-  Store store = state->getStore();
 
   // NOTE: Even if RegionsToInvalidate is empty, we must still invalidate
   //  global variables.
-  store = StoreMgr.InvalidateRegions(store, RegionsToInvalidate.data(),
-                                     RegionsToInvalidate.data() +
-                                     RegionsToInvalidate.size(),
-                                     Ex, Count, &IS,
-                                     /* invalidateGlobals = */ true);
+  state = state->InvalidateRegions(RegionsToInvalidate.data(),
+                                   RegionsToInvalidate.data() +
+                                   RegionsToInvalidate.size(),
+                                   Ex, Count, &IS,
+                                   /* invalidateGlobals = */ true);
 
-  state = state->makeWithStore(store);
   for (StoreManager::InvalidatedSymbols::iterator I = IS.begin(),
        E = IS.end(); I!=E; ++I) {
-      // Remove any existing reference-count binding.
+    SymbolRef sym = *I;
+    if (WhitelistedSymbols.count(sym))
+      continue;
+    // Remove any existing reference-count binding.
     state = state->remove<RefBindings>(*I);
   }
 
@@ -2860,7 +2759,7 @@ void CFRefCount::EvalSummary(ExplodedNodeSet& Dst,
 void CFRefCount::EvalCall(ExplodedNodeSet& Dst,
                           GRExprEngine& Eng,
                           GRStmtNodeBuilder& Builder,
-                          CallExpr* CE, SVal L,
+                          const CallExpr* CE, SVal L,
                           ExplodedNode* Pred) {
 
   RetainSummary *Summ = 0;
@@ -2874,7 +2773,7 @@ void CFRefCount::EvalCall(ExplodedNodeSet& Dst,
   else {
     const FunctionDecl* FD = L.getAsFunctionDecl();
     Summ = !FD ? Summaries.getDefaultSummary() :
-                 Summaries.getSummary(const_cast<FunctionDecl*>(FD));
+                 Summaries.getSummary(FD);
   }
 
   assert(Summ);
@@ -2885,7 +2784,7 @@ void CFRefCount::EvalCall(ExplodedNodeSet& Dst,
 void CFRefCount::EvalObjCMessageExpr(ExplodedNodeSet& Dst,
                                      GRExprEngine& Eng,
                                      GRStmtNodeBuilder& Builder,
-                                     ObjCMessageExpr* ME,
+                                     const ObjCMessageExpr* ME,
                                      ExplodedNode* Pred,
                                      const GRState *state) {
   RetainSummary *Summ =
@@ -2956,10 +2855,10 @@ void CFRefCount::EvalBind(GRStmtNodeBuilderRef& B, SVal location, SVal val) {
 void CFRefCount::EvalReturn(ExplodedNodeSet& Dst,
                             GRExprEngine& Eng,
                             GRStmtNodeBuilder& Builder,
-                            ReturnStmt* S,
+                            const ReturnStmt* S,
                             ExplodedNode* Pred) {
 
-  Expr* RetE = S->getRetValue();
+  const Expr* RetE = S->getRetValue();
   if (!RetE)
     return;
 
@@ -3404,7 +3303,7 @@ void CFRefCount::EvalDeadSymbols(ExplodedNodeSet& Dst,
                                  ExplodedNode* Pred,
                                  const GRState* state,
                                  SymbolReaper& SymReaper) {
-  Stmt *S = Builder.getStmt();
+  const Stmt *S = Builder.getStmt();
   RefBindings B = state->get<RefBindings>();
 
   // Update counts from autorelease pools
@@ -3454,7 +3353,8 @@ void CFRefCount::EvalDeadSymbols(ExplodedNodeSet& Dst,
 
 void CFRefCount::ProcessNonLeakError(ExplodedNodeSet& Dst,
                                      GRStmtNodeBuilder& Builder,
-                                     Expr* NodeExpr, SourceRange ErrorRange,
+                                     const Expr* NodeExpr, 
+                                     SourceRange ErrorRange,
                                      ExplodedNode* Pred,
                                      const GRState* St,
                                      RefVal::Kind hasErr, SymbolRef Sym) {
diff --git a/lib/Checker/CMakeLists.txt b/lib/Checker/CMakeLists.txt
index 259346a97068..5b54f0d12d5d 100644
--- a/lib/Checker/CMakeLists.txt
+++ b/lib/Checker/CMakeLists.txt
@@ -4,6 +4,7 @@ add_clang_library(clangChecker
   AdjustedReturnValueChecker.cpp
   AggExprVisitor.cpp
   AnalysisConsumer.cpp
+  AnalysisManager.cpp
   ArrayBoundChecker.cpp
   AttrNonNullChecker.cpp
   BasicConstraintManager.cpp
@@ -15,7 +16,6 @@ add_clang_library(clangChecker
   BuiltinFunctionChecker.cpp
   CFRefCount.cpp
   CallAndMessageChecker.cpp
-  CallInliner.cpp
   CastSizeChecker.cpp
   CastToStructChecker.cpp
   CheckDeadStores.cpp
@@ -24,6 +24,7 @@ add_clang_library(clangChecker
   CheckSecuritySyntaxOnly.cpp
   CheckSizeofPointer.cpp
   Checker.cpp
+  CheckerHelpers.cpp
   CocoaConventions.cpp
   CStringChecker.cpp
   DereferenceChecker.cpp
@@ -74,6 +75,7 @@ add_clang_library(clangChecker
   UndefinedArraySubscriptChecker.cpp
   UndefinedAssignmentChecker.cpp
   UnixAPIChecker.cpp
+  UnreachableCodeChecker.cpp
   VLASizeChecker.cpp
   ValueManager.cpp
   )
diff --git a/lib/Checker/CStringChecker.cpp b/lib/Checker/CStringChecker.cpp
index a92d409703a3..9ea572f90dfb 100644
--- a/lib/Checker/CStringChecker.cpp
+++ b/lib/Checker/CStringChecker.cpp
@@ -15,19 +15,30 @@
 #include "GRExprEngineExperimentalChecks.h"
 #include "clang/Checker/BugReporter/BugType.h"
 #include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "clang/Checker/PathSensitive/GRStateTrait.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace clang;
 
 namespace {
 class CStringChecker : public CheckerVisitor<CStringChecker> {
-  BugType *BT_Null, *BT_Bounds, *BT_Overlap;
+  BugType *BT_Null, *BT_Bounds, *BT_BoundsWrite, *BT_Overlap, *BT_NotCString;
 public:
   CStringChecker()
-  : BT_Null(0), BT_Bounds(0), BT_Overlap(0) {}
+  : BT_Null(0), BT_Bounds(0), BT_BoundsWrite(0), BT_Overlap(0), BT_NotCString(0)
+  {}
   static void *getTag() { static int tag; return &tag; }
 
   bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
+  void PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS);
+  void MarkLiveSymbols(const GRState *state, SymbolReaper &SR);
+  void EvalDeadSymbols(CheckerContext &C, SymbolReaper &SR);
+  bool WantsRegionChangeUpdate(const GRState *state);
+
+  const GRState *EvalRegionChanges(const GRState *state,
+                                   const MemRegion * const *Begin,
+                                   const MemRegion * const *End,
+                                   bool*);
 
   typedef void (CStringChecker::*FnCheck)(CheckerContext &, const CallExpr *);
 
@@ -40,26 +51,61 @@ public:
 
   void EvalMemcmp(CheckerContext &C, const CallExpr *CE);
 
+  void EvalStrlen(CheckerContext &C, const CallExpr *CE);
+
+  void EvalStrcpy(CheckerContext &C, const CallExpr *CE);
+  void EvalStpcpy(CheckerContext &C, const CallExpr *CE);
+  void EvalStrcpyCommon(CheckerContext &C, const CallExpr *CE, bool ReturnEnd);
+
   // Utility methods
   std::pair<const GRState*, const GRState*>
   AssumeZero(CheckerContext &C, const GRState *state, SVal V, QualType Ty);
 
+  const GRState *SetCStringLength(const GRState *state, const MemRegion *MR,
+                                  SVal StrLen);
+  SVal GetCStringLengthForRegion(CheckerContext &C, const GRState *&state,
+                                 const Expr *Ex, const MemRegion *MR);
+  SVal GetCStringLength(CheckerContext &C, const GRState *&state,
+                        const Expr *Ex, SVal Buf);
+
+  const GRState *InvalidateBuffer(CheckerContext &C, const GRState *state,
+                                  const Expr *Ex, SVal V);
+
+  bool SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
+                       const MemRegion *MR);
+
+  // Re-usable checks
   const GRState *CheckNonNull(CheckerContext &C, const GRState *state,
                                const Expr *S, SVal l);
   const GRState *CheckLocation(CheckerContext &C, const GRState *state,
-                               const Expr *S, SVal l);
+                               const Expr *S, SVal l,
+                               bool IsDestination = false);
   const GRState *CheckBufferAccess(CheckerContext &C, const GRState *state,
                                    const Expr *Size,
                                    const Expr *FirstBuf,
-                                   const Expr *SecondBuf = NULL);
+                                   const Expr *SecondBuf = NULL,
+                                   bool FirstIsDestination = false);
   const GRState *CheckOverlap(CheckerContext &C, const GRState *state,
                               const Expr *Size, const Expr *First,
                               const Expr *Second);
   void EmitOverlapBug(CheckerContext &C, const GRState *state,
                       const Stmt *First, const Stmt *Second);
 };
+
+class CStringLength {
+public:
+  typedef llvm::ImmutableMap<const MemRegion *, SVal> EntryMap;
+};
 } //end anonymous namespace
 
+namespace clang {
+  template <>
+  struct GRStateTrait<CStringLength> 
+    : public GRStatePartialTrait<CStringLength::EntryMap> {
+    static void *GDMIndex() { return CStringChecker::getTag(); }
+  };
+}
+
 void clang::RegisterCStringChecker(GRExprEngine &Eng) {
   Eng.registerCheck(new CStringChecker());
 }
@@ -122,7 +168,8 @@ const GRState *CStringChecker::CheckNonNull(CheckerContext &C,
 // FIXME: This was originally copied from ArrayBoundChecker.cpp. Refactor?
 const GRState *CStringChecker::CheckLocation(CheckerContext &C,
                                              const GRState *state,
-                                             const Expr *S, SVal l) {
+                                             const Expr *S, SVal l,
+                                             bool IsDestination) {
   // If a previous check has failed, propagate the failure.
   if (!state)
     return NULL;
@@ -136,7 +183,7 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
   if (!ER)
     return state;
 
-  assert(ER->getValueType(C.getASTContext()) == C.getASTContext().CharTy &&
+  assert(ER->getValueType() == C.getASTContext().CharTy &&
     "CheckLocation should only be called with char* ElementRegions");
 
   // Get the size of the array.
@@ -155,17 +202,26 @@ const GRState *CStringChecker::CheckLocation(CheckerContext &C,
     if (!N)
       return NULL;
 
-    if (!BT_Bounds)
-      BT_Bounds = new BuiltinBug("Out-of-bound array access",
-        "Byte string function accesses out-of-bound array element "
-        "(buffer overflow)");
+    BuiltinBug *BT;
+    if (IsDestination) {
+      if (!BT_BoundsWrite) {
+        BT_BoundsWrite = new BuiltinBug("Out-of-bound array access",
+          "Byte string function overflows destination buffer");
+      }
+      BT = static_cast<BuiltinBug*>(BT_BoundsWrite);
+    } else {
+      if (!BT_Bounds) {
+        BT_Bounds = new BuiltinBug("Out-of-bound array access",
+          "Byte string function accesses out-of-bound array element");
+      }
+      BT = static_cast<BuiltinBug*>(BT_Bounds);
+    }
 
     // FIXME: It would be nice to eventually make this diagnostic more clear,
     // e.g., by referencing the original declaration or by saying *why* this
     // reference is outside the range.
 
     // Generate a report for this bug.
-    BuiltinBug *BT = static_cast<BuiltinBug*>(BT_Bounds);
     RangedBugReport *report = new RangedBugReport(*BT, BT->getDescription(), N);
 
     report->addRange(S->getSourceRange());
@@ -182,7 +238,8 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
                                                  const GRState *state,
                                                  const Expr *Size,
                                                  const Expr *FirstBuf,
-                                                 const Expr *SecondBuf) {
+                                                 const Expr *SecondBuf,
+                                                 bool FirstIsDestination) {
   // If a previous check has failed, propagate the failure.
   if (!state)
     return NULL;
@@ -191,7 +248,7 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
   SValuator &SV = VM.getSValuator();
   ASTContext &Ctx = C.getASTContext();
 
-  QualType SizeTy = Ctx.getSizeType();
+  QualType SizeTy = Size->getType();
   QualType PtrTy = Ctx.getPointerType(Ctx.CharTy);
 
   // Check that the first buffer is non-null.
@@ -208,18 +265,20 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
 
   // Compute the offset of the last element to be accessed: size-1.
   NonLoc One = cast<NonLoc>(VM.makeIntVal(1, SizeTy));
-  NonLoc LastOffset = cast<NonLoc>(SV.EvalBinOpNN(state, BinaryOperator::Sub,
+  NonLoc LastOffset = cast<NonLoc>(SV.EvalBinOpNN(state, BO_Sub,
                                                   *Length, One, SizeTy));
 
   // Check that the first buffer is sufficently long.
-  Loc BufStart = cast<Loc>(SV.EvalCast(BufVal, PtrTy, FirstBuf->getType()));
-  SVal BufEnd
-    = SV.EvalBinOpLN(state, BinaryOperator::Add, BufStart, LastOffset, PtrTy);
-  state = CheckLocation(C, state, FirstBuf, BufEnd);
+  SVal BufStart = SV.EvalCast(BufVal, PtrTy, FirstBuf->getType());
+  if (Loc *BufLoc = dyn_cast<Loc>(&BufStart)) {
+    SVal BufEnd = SV.EvalBinOpLN(state, BO_Add, *BufLoc,
+                                 LastOffset, PtrTy);
+    state = CheckLocation(C, state, FirstBuf, BufEnd, FirstIsDestination);
 
-  // If the buffer isn't large enough, abort.
-  if (!state)
-    return NULL;
+    // If the buffer isn't large enough, abort.
+    if (!state)
+      return NULL;
+  }
 
   // If there's a second buffer, check it as well.
   if (SecondBuf) {
@@ -228,10 +287,12 @@ const GRState *CStringChecker::CheckBufferAccess(CheckerContext &C,
     if (!state)
       return NULL;
 
-    BufStart = cast<Loc>(SV.EvalCast(BufVal, PtrTy, SecondBuf->getType()));
-    BufEnd
-      = SV.EvalBinOpLN(state, BinaryOperator::Add, BufStart, LastOffset, PtrTy);
-    state = CheckLocation(C, state, SecondBuf, BufEnd);
+    BufStart = SV.EvalCast(BufVal, PtrTy, SecondBuf->getType());
+    if (Loc *BufLoc = dyn_cast<Loc>(&BufStart)) {
+      SVal BufEnd = SV.EvalBinOpLN(state, BO_Add, *BufLoc,
+                                   LastOffset, PtrTy);
+      state = CheckLocation(C, state, SecondBuf, BufEnd);
+    }
   }
 
   // Large enough or not, return this state!
@@ -284,7 +345,7 @@ const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
 
   // Which value comes first?
   QualType CmpTy = Ctx.IntTy;
-  SVal Reverse = SV.EvalBinOpLL(state, BinaryOperator::GT,
+  SVal Reverse = SV.EvalBinOpLL(state, BO_GT,
                                 *FirstLoc, *SecondLoc, CmpTy);
   DefinedOrUnknownSVal *ReverseTest = dyn_cast<DefinedOrUnknownSVal>(&Reverse);
   if (!ReverseTest)
@@ -324,14 +385,14 @@ const GRState *CStringChecker::CheckOverlap(CheckerContext &C,
     return state;
 
   // Compute the end of the first buffer. Bail out if THAT fails.
-  SVal FirstEnd = SV.EvalBinOpLN(state, BinaryOperator::Add,
+  SVal FirstEnd = SV.EvalBinOpLN(state, BO_Add,
                                  *FirstStartLoc, *Length, CharPtrTy);
   Loc *FirstEndLoc = dyn_cast<Loc>(&FirstEnd);
   if (!FirstEndLoc)
     return state;
 
   // Is the end of the first buffer past the start of the second buffer?
-  SVal Overlap = SV.EvalBinOpLL(state, BinaryOperator::GT,
+  SVal Overlap = SV.EvalBinOpLL(state, BO_GT,
                                 *FirstEndLoc, *SecondLoc, CmpTy);
   DefinedOrUnknownSVal *OverlapTest = dyn_cast<DefinedOrUnknownSVal>(&Overlap);
   if (!OverlapTest)
@@ -369,6 +430,222 @@ void CStringChecker::EmitOverlapBug(CheckerContext &C, const GRState *state,
   C.EmitReport(report);
 }
 
+const GRState *CStringChecker::SetCStringLength(const GRState *state,
+                                                const MemRegion *MR,
+                                                SVal StrLen) {
+  assert(!StrLen.isUndef() && "Attempt to set an undefined string length");
+  if (StrLen.isUnknown())
+    return state;
+
+  MR = MR->StripCasts();
+
+  switch (MR->getKind()) {
+  case MemRegion::StringRegionKind:
+    // FIXME: This can happen if we strcpy() into a string region. This is
+    // undefined [C99 6.4.5p6], but we should still warn about it.
+    return state;
+
+  case MemRegion::SymbolicRegionKind:
+  case MemRegion::AllocaRegionKind:
+  case MemRegion::VarRegionKind:
+  case MemRegion::FieldRegionKind:
+  case MemRegion::ObjCIvarRegionKind:
+    return state->set<CStringLength>(MR, StrLen);
+
+  case MemRegion::ElementRegionKind:
+    // FIXME: Handle element regions by upper-bounding the parent region's
+    // string length.
+    return state;
+
+  default:
+    // Other regions (mostly non-data) can't have a reliable C string length.
+    // For now, just ignore the change.
+    // FIXME: These are rare but not impossible. We should output some kind of
+    // warning for things like strcpy((char[]){'a', 0}, "b");
+    return state;
+  }
+}
+
+SVal CStringChecker::GetCStringLengthForRegion(CheckerContext &C,
+                                               const GRState *&state,
+                                               const Expr *Ex,
+                                               const MemRegion *MR) {
+  // If there's a recorded length, go ahead and return it.
+  const SVal *Recorded = state->get<CStringLength>(MR);
+  if (Recorded)
+    return *Recorded;
+  
+  // Otherwise, get a new symbol and update the state.
+  unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+  ValueManager &ValMgr = C.getValueManager();
+  QualType SizeTy = ValMgr.getContext().getSizeType();
+  SVal Strlen = ValMgr.getMetadataSymbolVal(getTag(), MR, Ex, SizeTy, Count);
+  
+  state = state->set<CStringLength>(MR, Strlen);
+  return Strlen;
+}
+
+SVal CStringChecker::GetCStringLength(CheckerContext &C, const GRState *&state,
+                                      const Expr *Ex, SVal Buf) {
+  const MemRegion *MR = Buf.getAsRegion();
+  if (!MR) {
+    // If we can't get a region, see if it's something we /know/ isn't a
+    // C string. In the context of locations, the only time we can issue such
+    // a warning is for labels.
+    if (loc::GotoLabel *Label = dyn_cast<loc::GotoLabel>(&Buf)) {
+      if (ExplodedNode *N = C.GenerateNode(state)) {
+        if (!BT_NotCString)
+          BT_NotCString = new BuiltinBug("API",
+            "Argument is not a null-terminated string.");
+
+        llvm::SmallString<120> buf;
+        llvm::raw_svector_ostream os(buf);
+        os << "Argument to byte string function is the address of the label '"
+           << Label->getLabel()->getID()->getName()
+           << "', which is not a null-terminated string";
+
+        // Generate a report for this bug.
+        EnhancedBugReport *report = new EnhancedBugReport(*BT_NotCString,
+                                                          os.str(), N);
+
+        report->addRange(Ex->getSourceRange());
+        C.EmitReport(report);        
+      }
+
+      return UndefinedVal();
+    }
+
+    // If it's not a region and not a label, give up.
+    return UnknownVal();
+  }
+
+  // If we have a region, strip casts from it and see if we can figure out
+  // its length. For anything we can't figure out, just return UnknownVal.
+  MR = MR->StripCasts();
+
+  switch (MR->getKind()) {
+  case MemRegion::StringRegionKind: {
+    // Modifying the contents of string regions is undefined [C99 6.4.5p6],
+    // so we can assume that the byte length is the correct C string length.
+    ValueManager &ValMgr = C.getValueManager();
+    QualType SizeTy = ValMgr.getContext().getSizeType();
+    const StringLiteral *Str = cast<StringRegion>(MR)->getStringLiteral();
+    return ValMgr.makeIntVal(Str->getByteLength(), SizeTy);
+  }
+  case MemRegion::SymbolicRegionKind:
+  case MemRegion::AllocaRegionKind:
+  case MemRegion::VarRegionKind:
+  case MemRegion::FieldRegionKind:
+  case MemRegion::ObjCIvarRegionKind:
+    return GetCStringLengthForRegion(C, state, Ex, MR);
+  case MemRegion::CompoundLiteralRegionKind:
+    // FIXME: Can we track this? Is it necessary?
+    return UnknownVal();
+  case MemRegion::ElementRegionKind:
+    // FIXME: How can we handle this? It's not good enough to subtract the
+    // offset from the base string length; consider "123\x00567" and &a[5].
+    return UnknownVal();
+  default:
+    // Other regions (mostly non-data) can't have a reliable C string length.
+    // In this case, an error is emitted and UndefinedVal is returned.
+    // The caller should always be prepared to handle this case.
+    if (ExplodedNode *N = C.GenerateNode(state)) {
+      if (!BT_NotCString)
+        BT_NotCString = new BuiltinBug("API",
+          "Argument is not a null-terminated string.");
+
+      llvm::SmallString<120> buf;
+      llvm::raw_svector_ostream os(buf);
+
+      os << "Argument to byte string function is ";
+
+      if (SummarizeRegion(os, C.getASTContext(), MR))
+        os << ", which is not a null-terminated string";
+      else
+        os << "not a null-terminated string";
+
+      // Generate a report for this bug.
+      EnhancedBugReport *report = new EnhancedBugReport(*BT_NotCString,
+                                                        os.str(), N);
+
+      report->addRange(Ex->getSourceRange());
+      C.EmitReport(report);        
+    }
+
+    return UndefinedVal();
+  }
+}
+
+const GRState *CStringChecker::InvalidateBuffer(CheckerContext &C,
+                                                const GRState *state,
+                                                const Expr *E, SVal V) {
+  Loc *L = dyn_cast<Loc>(&V);
+  if (!L)
+    return state;
+
+  // FIXME: This is a simplified version of what's in CFRefCount.cpp -- it makes
+  // some assumptions about the value that CFRefCount can't. Even so, it should
+  // probably be refactored.
+  if (loc::MemRegionVal* MR = dyn_cast<loc::MemRegionVal>(L)) {
+    const MemRegion *R = MR->getRegion()->StripCasts();
+
+    // Are we dealing with an ElementRegion?  If so, we should be invalidating
+    // the super-region.
+    if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
+      R = ER->getSuperRegion();
+      // FIXME: What about layers of ElementRegions?
+    }
+
+    // Invalidate this region.
+    unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+    return state->InvalidateRegion(R, E, Count, NULL);
+  }
+
+  // If we have a non-region value by chance, just remove the binding.
+  // FIXME: is this necessary or correct? This handles the non-Region
+  //  cases.  Is it ever valid to store to these?
+  return state->unbindLoc(*L);
+}
+
+bool CStringChecker::SummarizeRegion(llvm::raw_ostream& os, ASTContext& Ctx,
+                                     const MemRegion *MR) {
+  const TypedRegion *TR = dyn_cast<TypedRegion>(MR);
+  if (!TR)
+    return false;
+
+  switch (TR->getKind()) {
+  case MemRegion::FunctionTextRegionKind: {
+    const FunctionDecl *FD = cast<FunctionTextRegion>(TR)->getDecl();
+    if (FD)
+      os << "the address of the function '" << FD << "'";
+    else
+      os << "the address of a function";
+    return true;
+  }
+  case MemRegion::BlockTextRegionKind:
+    os << "block text";
+    return true;
+  case MemRegion::BlockDataRegionKind:
+    os << "a block";
+    return true;
+  case MemRegion::CXXThisRegionKind:
+  case MemRegion::CXXObjectRegionKind:
+    os << "a C++ object of type " << TR->getValueType().getAsString();
+    return true;
+  case MemRegion::VarRegionKind:
+    os << "a variable of type" << TR->getValueType().getAsString();
+    return true;
+  case MemRegion::FieldRegionKind:
+    os << "a field of type " << TR->getValueType().getAsString();
+    return true;
+  case MemRegion::ObjCIvarRegionKind:
+    os << "an instance variable of type " << TR->getValueType().getAsString();
+    return true;
+  default:
+    return false;
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Evaluation of individual function calls.
 //===----------------------------------------------------------------------===//
@@ -390,11 +667,20 @@ void CStringChecker::EvalCopyCommon(CheckerContext &C, const GRState *state,
   // If the size can be nonzero, we have to check the other arguments.
   if (StNonZeroSize) {
     state = StNonZeroSize;
-    state = CheckBufferAccess(C, state, Size, Dest, Source);
+    state = CheckBufferAccess(C, state, Size, Dest, Source,
+                              /* FirstIsDst = */ true);
     if (Restricted)
       state = CheckOverlap(C, state, Size, Dest, Source);
-    if (state)
+
+    if (state) {
+      // Invalidate the destination.
+      // FIXME: Even if we can't perfectly model the copy, we should see if we
+      // can use LazyCompoundVals to copy the source values into the destination.
+      // This would probably remove any existing bindings past the end of the
+      // copied region, but that's still an improvement over blank invalidation.
+      state = InvalidateBuffer(C, state, Dest, state->getSVal(Dest));
       C.addTransition(state);
+    }
   }
 }
 
@@ -481,7 +767,7 @@ void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
       if (state) {
         // The return value is the comparison result, which we don't know.
         unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
-        SVal CmpV = ValMgr.getConjuredSymbolVal(NULL, CE, CE->getType(), Count);
+        SVal CmpV = ValMgr.getConjuredSymbolVal(NULL, CE, Count);
         state = state->BindExpr(CE, CmpV);
         C.addTransition(state);
       }
@@ -489,8 +775,123 @@ void CStringChecker::EvalMemcmp(CheckerContext &C, const CallExpr *CE) {
   }
 }
 
+void CStringChecker::EvalStrlen(CheckerContext &C, const CallExpr *CE) {
+  // size_t strlen(const char *s);
+  const GRState *state = C.getState();
+  const Expr *Arg = CE->getArg(0);
+  SVal ArgVal = state->getSVal(Arg);
+
+  // Check that the argument is non-null.
+  state = CheckNonNull(C, state, Arg, ArgVal);
+
+  if (state) {
+    SVal StrLen = GetCStringLength(C, state, Arg, ArgVal);
+
+    // If the argument isn't a valid C string, there's no valid state to
+    // transition to.
+    if (StrLen.isUndef())
+      return;
+
+    // If GetCStringLength couldn't figure out the length, conjure a return
+    // value, so it can be used in constraints, at least.
+    if (StrLen.isUnknown()) {
+      ValueManager &ValMgr = C.getValueManager();
+      unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+      StrLen = ValMgr.getConjuredSymbolVal(NULL, CE, Count);
+    }
+
+    // Bind the return value.
+    state = state->BindExpr(CE, StrLen);
+    C.addTransition(state);
+  }
+}
+
+void CStringChecker::EvalStrcpy(CheckerContext &C, const CallExpr *CE) {
+  // char *strcpy(char *restrict dst, const char *restrict src);
+  EvalStrcpyCommon(C, CE, /* ReturnEnd = */ false);
+}
+
+void CStringChecker::EvalStpcpy(CheckerContext &C, const CallExpr *CE) {
+  // char *stpcpy(char *restrict dst, const char *restrict src);
+  EvalStrcpyCommon(C, CE, /* ReturnEnd = */ true);
+}
+
+void CStringChecker::EvalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
+                                      bool ReturnEnd) {
+  const GRState *state = C.getState();
+
+  // Check that the destination is non-null
+  const Expr *Dst = CE->getArg(0);
+  SVal DstVal = state->getSVal(Dst);
+
+  state = CheckNonNull(C, state, Dst, DstVal);
+  if (!state)
+    return;
+
+  // Check that the source is non-null.
+  const Expr *Src = CE->getArg(1);
+  SVal SrcVal = state->getSVal(Src);
+
+  state = CheckNonNull(C, state, Src, SrcVal);
+  if (!state)
+    return;
+
+  // Get the string length of the source.
+  SVal StrLen = GetCStringLength(C, state, Src, SrcVal);
+
+  // If the source isn't a valid C string, give up.
+  if (StrLen.isUndef())
+    return;
+
+  SVal Result = (ReturnEnd ? UnknownVal() : DstVal);
+
+  // If the destination is a MemRegion, try to check for a buffer overflow and
+  // record the new string length.
+  if (loc::MemRegionVal *DstRegVal = dyn_cast<loc::MemRegionVal>(&DstVal)) {
+    // If the length is known, we can check for an overflow.
+    if (NonLoc *KnownStrLen = dyn_cast<NonLoc>(&StrLen)) {
+      SValuator &SV = C.getSValuator();
+
+      SVal LastElement = SV.EvalBinOpLN(state, BO_Add,
+                                        *DstRegVal, *KnownStrLen,
+                                        Dst->getType());
+
+      state = CheckLocation(C, state, Dst, LastElement, /* IsDst = */ true);
+      if (!state)
+        return;
+
+      // If this is a stpcpy-style copy, the last element is the return value.
+      if (ReturnEnd)
+        Result = LastElement;
+    }
+
+    // Invalidate the destination. This must happen before we set the C string
+    // length because invalidation will clear the length.
+    // FIXME: Even if we can't perfectly model the copy, we should see if we
+    // can use LazyCompoundVals to copy the source values into the destination.
+    // This would probably remove any existing bindings past the end of the
+    // string, but that's still an improvement over blank invalidation.
+    state = InvalidateBuffer(C, state, Dst, *DstRegVal);
+
+    // Set the C string length of the destination.
+    state = SetCStringLength(state, DstRegVal->getRegion(), StrLen);
+  }
+
+  // If this is a stpcpy-style copy, but we were unable to check for a buffer
+  // overflow, we still need a result. Conjure a return value.
+  if (ReturnEnd && Result.isUnknown()) {
+    ValueManager &ValMgr = C.getValueManager();
+    unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
+    StrLen = ValMgr.getConjuredSymbolVal(NULL, CE, Count);
+  }
+
+  // Set the return value.
+  state = state->BindExpr(CE, Result);
+  C.addTransition(state);
+}
+
 //===----------------------------------------------------------------------===//
-// The driver method.
+// The driver method, and other Checker callbacks.
 //===----------------------------------------------------------------------===//
 
 bool CStringChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
@@ -512,6 +913,9 @@ bool CStringChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
     .Cases("memcpy", "__memcpy_chk", &CStringChecker::EvalMemcpy)
     .Cases("memcmp", "bcmp", &CStringChecker::EvalMemcmp)
     .Cases("memmove", "__memmove_chk", &CStringChecker::EvalMemmove)
+    .Cases("strcpy", "__strcpy_chk", &CStringChecker::EvalStrcpy)
+    .Cases("stpcpy", "__stpcpy_chk", &CStringChecker::EvalStpcpy)
+    .Case("strlen", &CStringChecker::EvalStrlen)
     .Case("bcopy", &CStringChecker::EvalBcopy)
     .Default(NULL);
 
@@ -523,3 +927,129 @@ bool CStringChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
   (this->*EvalFunction)(C, CE);
   return true;
 }
+
+void CStringChecker::PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS) {
+  // Record string length for char a[] = "abc";
+  const GRState *state = C.getState();
+
+  for (DeclStmt::const_decl_iterator I = DS->decl_begin(), E = DS->decl_end();
+       I != E; ++I) {
+    const VarDecl *D = dyn_cast<VarDecl>(*I);
+    if (!D)
+      continue;
+
+    // FIXME: Handle array fields of structs.
+    if (!D->getType()->isArrayType())
+      continue;
+
+    const Expr *Init = D->getInit();
+    if (!Init)
+      continue;
+    if (!isa<StringLiteral>(Init))
+      continue;
+
+    Loc VarLoc = state->getLValue(D, C.getPredecessor()->getLocationContext());
+    const MemRegion *MR = VarLoc.getAsRegion();
+    if (!MR)
+      continue;
+
+    SVal StrVal = state->getSVal(Init);
+    assert(StrVal.isValid() && "Initializer string is unknown or undefined");
+    DefinedOrUnknownSVal StrLen
+      = cast<DefinedOrUnknownSVal>(GetCStringLength(C, state, Init, StrVal));
+
+    state = state->set<CStringLength>(MR, StrLen);
+  }
+
+  C.addTransition(state);
+}
+
+bool CStringChecker::WantsRegionChangeUpdate(const GRState *state) {
+  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  return !Entries.isEmpty();
+}
+
+const GRState *CStringChecker::EvalRegionChanges(const GRState *state,
+                                                 const MemRegion * const *Begin,
+                                                 const MemRegion * const *End,
+                                                 bool *) {
+  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  if (Entries.isEmpty())
+    return state;
+
+  llvm::SmallPtrSet<const MemRegion *, 8> Invalidated;
+  llvm::SmallPtrSet<const MemRegion *, 32> SuperRegions;
+
+  // First build sets for the changed regions and their super-regions.
+  for ( ; Begin != End; ++Begin) {
+    const MemRegion *MR = *Begin;
+    Invalidated.insert(MR);
+
+    SuperRegions.insert(MR);
+    while (const SubRegion *SR = dyn_cast<SubRegion>(MR)) {
+      MR = SR->getSuperRegion();
+      SuperRegions.insert(MR);
+    }
+  }
+
+  CStringLength::EntryMap::Factory &F = state->get_context<CStringLength>();
+
+  // Then loop over the entries in the current state.
+  for (CStringLength::EntryMap::iterator I = Entries.begin(),
+       E = Entries.end(); I != E; ++I) {
+    const MemRegion *MR = I.getKey();
+
+    // Is this entry for a super-region of a changed region?
+    if (SuperRegions.count(MR)) {
+      Entries = F.Remove(Entries, MR);
+      continue;
+    }
+
+    // Is this entry for a sub-region of a changed region?
+    const MemRegion *Super = MR;
+    while (const SubRegion *SR = dyn_cast<SubRegion>(Super)) {
+      Super = SR->getSuperRegion();
+      if (Invalidated.count(Super)) {
+        Entries = F.Remove(Entries, MR);
+        break;
+      }
+    }
+  }
+
+  return state->set<CStringLength>(Entries);
+}
+
+void CStringChecker::MarkLiveSymbols(const GRState *state, SymbolReaper &SR) {
+  // Mark all symbols in our string length map as valid.
+  CStringLength::EntryMap Entries = state->get<CStringLength>();
+
+  for (CStringLength::EntryMap::iterator I = Entries.begin(), E = Entries.end();
+       I != E; ++I) {
+    SVal Len = I.getData();
+    if (SymbolRef Sym = Len.getAsSymbol())
+      SR.markInUse(Sym);
+  }
+}
+
+void CStringChecker::EvalDeadSymbols(CheckerContext &C, SymbolReaper &SR) {
+  if (!SR.hasDeadSymbols())
+    return;
+
+  const GRState *state = C.getState();
+  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  if (Entries.isEmpty())
+    return;
+
+  CStringLength::EntryMap::Factory &F = state->get_context<CStringLength>();
+  for (CStringLength::EntryMap::iterator I = Entries.begin(), E = Entries.end();
+       I != E; ++I) {
+    SVal Len = I.getData();
+    if (SymbolRef Sym = Len.getAsSymbol()) {
+      if (SR.isDead(Sym))
+        Entries = F.Remove(Entries, I.getKey());
+    }
+  }
+
+  state = state->set<CStringLength>(Entries);
+  C.GenerateNode(state);
+}
diff --git a/lib/Checker/CallAndMessageChecker.cpp b/lib/Checker/CallAndMessageChecker.cpp
index c619d75479ed..3c9ddce9f664 100644
--- a/lib/Checker/CallAndMessageChecker.cpp
+++ b/lib/Checker/CallAndMessageChecker.cpp
@@ -114,7 +114,7 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
       : C(c), StoreMgr(storeMgr), MrMgr(mrMgr), store(s) {}
 
       bool Find(const TypedRegion *R) {
-        QualType T = R->getValueType(C);
+        QualType T = R->getValueType();
         if (const RecordType *RT = T->getAsStructureType()) {
           const RecordDecl *RD = RT->getDecl()->getDefinition();
           assert(RD && "Referred record has no definition");
diff --git a/lib/Checker/CallInliner.cpp b/lib/Checker/CallInliner.cpp
deleted file mode 100644
index c47e06c78fb2..000000000000
--- a/lib/Checker/CallInliner.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-//===--- CallInliner.cpp - Transfer function that inlines callee ----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file implements the callee inlining transfer function.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Checker/PathSensitive/CheckerVisitor.h"
-#include "clang/Checker/PathSensitive/GRState.h"
-#include "clang/Checker/Checkers/LocalCheckers.h"
-
-using namespace clang;
-
-namespace {
-class CallInliner : public Checker {
-public:
-  static void *getTag() {
-    static int x;
-    return &x;
-  }
-
-  virtual bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
-};
-}
-
-void clang::RegisterCallInliner(GRExprEngine &Eng) {
-  Eng.registerCheck(new CallInliner());
-}
-
-bool CallInliner::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
-  const GRState *state = C.getState();
-  const Expr *Callee = CE->getCallee();
-  SVal L = state->getSVal(Callee);
-  
-  const FunctionDecl *FD = L.getAsFunctionDecl();
-  if (!FD)
-    return false;
-
-  if (!FD->hasBody(FD))
-    return false;
-
-  // Now we have the definition of the callee, create a CallEnter node.
-  CallEnter Loc(CE, FD, C.getPredecessor()->getLocationContext());
-  C.addTransition(state, Loc);
-
-  return true;
-}
-
diff --git a/lib/Checker/CastSizeChecker.cpp b/lib/Checker/CastSizeChecker.cpp
index a502c10cac16..6676fe5e7a36 100644
--- a/lib/Checker/CastSizeChecker.cpp
+++ b/lib/Checker/CastSizeChecker.cpp
@@ -44,6 +44,10 @@ void CastSizeChecker::PreVisitCastExpr(CheckerContext &C, const CastExpr *CE) {
 
   QualType ToPointeeTy = ToPTy->getPointeeType();
 
+  // Only perform the check if 'ToPointeeTy' is a complete type.
+  if (ToPointeeTy->isIncompleteType())
+    return;
+
   const GRState *state = C.getState();
   const MemRegion *R = state->getSVal(E).getAsRegion();
   if (R == 0)
diff --git a/lib/Checker/CheckDeadStores.cpp b/lib/Checker/CheckDeadStores.cpp
index d6ea187957ce..38961000d3fd 100644
--- a/lib/Checker/CheckDeadStores.cpp
+++ b/lib/Checker/CheckDeadStores.cpp
@@ -217,7 +217,7 @@ public:
               // If x is EVER assigned a new value later, don't issue
               // a warning.  This is because such initialization can be
               // due to defensive programming.
-              if (E->isConstantInitializer(Ctx))
+              if (E->isConstantInitializer(Ctx, false))
                 return;
 
               if (DeclRefExpr *DRE=dyn_cast<DeclRefExpr>(E->IgnoreParenCasts()))
@@ -268,7 +268,7 @@ public:
     // Check for '&'.  Any VarDecl whose value has its address-taken we
     // treat as escaped.
     Expr* E = U->getSubExpr()->IgnoreParenCasts();
-    if (U->getOpcode() == UnaryOperator::AddrOf)
+    if (U->getOpcode() == UO_AddrOf)
       if (DeclRefExpr* DR = dyn_cast<DeclRefExpr>(E))
         if (VarDecl* VD = dyn_cast<VarDecl>(DR->getDecl())) {
           Escaped.insert(VD);
diff --git a/lib/Checker/CheckSecuritySyntaxOnly.cpp b/lib/Checker/CheckSecuritySyntaxOnly.cpp
index af85c2faee3a..9a2ac45fa2e1 100644
--- a/lib/Checker/CheckSecuritySyntaxOnly.cpp
+++ b/lib/Checker/CheckSecuritySyntaxOnly.cpp
@@ -41,8 +41,8 @@ class WalkAST : public StmtVisitor<WalkAST> {
 
 public:
   WalkAST(BugReporter &br) : BR(br),
-			     II_gets(0), II_getpw(0), II_mktemp(0),
-			     II_rand(), II_random(0), II_setid(),
+                             II_gets(0), II_getpw(0), II_mktemp(0),
+                             II_rand(), II_random(0), II_setid(),
                  CheckRand(isArc4RandomAvailable(BR.getContext())) {}
 
   // Statement visitor methods.
@@ -131,7 +131,7 @@ GetIncrementedVar(const Expr *expr, const VarDecl *x, const VarDecl *y) {
 
   if (const BinaryOperator *B = dyn_cast<BinaryOperator>(expr)) {
     if (!(B->isAssignmentOp() || B->isCompoundAssignmentOp() ||
-          B->getOpcode() == BinaryOperator::Comma))
+          B->getOpcode() == BO_Comma))
       return NULL;
 
     if (const DeclRefExpr *lhs = GetIncrementedVar(B->getLHS(), x, y))
@@ -217,7 +217,7 @@ void WalkAST::CheckLoopConditionForFloat(const ForStmt *FS) {
   llvm::SmallString<256> sbuf;
   llvm::raw_svector_ostream os(sbuf);
 
-  os << "Variable '" << drCond->getDecl()->getNameAsCString()
+  os << "Variable '" << drCond->getDecl()->getName()
      << "' with floating point type '" << drCond->getType().getAsString()
      << "' should not be used as a loop counter";
 
@@ -332,10 +332,10 @@ void WalkAST::CheckCall_mktemp(const CallExpr *CE, const FunctionDecl *FD) {
   // Issue a waring.
   SourceRange R = CE->getCallee()->getSourceRange();
   BR.EmitBasicReport("Potential insecure temporary file in call 'mktemp'",
-		     "Security",
-		     "Call to function 'mktemp' is insecure as it always "
-		     "creates or uses insecure temporary file.  Use 'mkstemp' instead",
-		     CE->getLocStart(), &R, 1);
+                     "Security",
+                     "Call to function 'mktemp' is insecure as it always "
+                     "creates or uses insecure temporary file.  Use 'mkstemp' instead",
+                     CE->getLocStart(), &R, 1);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Checker/CheckerHelpers.cpp b/lib/Checker/CheckerHelpers.cpp
new file mode 100644
index 000000000000..ece69435cac7
--- /dev/null
+++ b/lib/Checker/CheckerHelpers.cpp
@@ -0,0 +1,80 @@
+//===---- CheckerHelpers.cpp - Helper functions for checkers ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines several static functions for use in checkers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Checker/PathSensitive/CheckerHelpers.h"
+#include "clang/AST/Expr.h"
+
+// Recursively find any substatements containing macros
+bool clang::containsMacro(const Stmt *S) {
+  if (S->getLocStart().isMacroID())
+    return true;
+
+  if (S->getLocEnd().isMacroID())
+    return true;
+
+  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
+      ++I)
+    if (const Stmt *child = *I)
+      if (containsMacro(child))
+        return true;
+
+  return false;
+}
+
+// Recursively find any substatements containing enum constants
+bool clang::containsEnum(const Stmt *S) {
+  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S);
+
+  if (DR && isa<EnumConstantDecl>(DR->getDecl()))
+    return true;
+
+  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
+      ++I)
+    if (const Stmt *child = *I)
+      if (containsEnum(child))
+        return true;
+
+  return false;
+}
+
+// Recursively find any substatements containing static vars
+bool clang::containsStaticLocal(const Stmt *S) {
+  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S);
+
+  if (DR)
+    if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl()))
+      if (VD->isStaticLocal())
+        return true;
+
+  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
+      ++I)
+    if (const Stmt *child = *I)
+      if (containsStaticLocal(child))
+        return true;
+
+  return false;
+}
+
+// Recursively find any substatements containing __builtin_offsetof
+bool clang::containsBuiltinOffsetOf(const Stmt *S) {
+  if (isa<OffsetOfExpr>(S))
+    return true;
+
+  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
+      ++I)
+    if (const Stmt *child = *I)
+      if (containsBuiltinOffsetOf(child))
+        return true;
+
+  return false;
+}
diff --git a/lib/Checker/CocoaConventions.cpp b/lib/Checker/CocoaConventions.cpp
index 3ba887ccc7e3..b446a048d488 100644
--- a/lib/Checker/CocoaConventions.cpp
+++ b/lib/Checker/CocoaConventions.cpp
@@ -173,9 +173,10 @@ bool cocoa::isCocoaObjectRef(QualType Ty) {
   if (!PT)
     return true;
   
-  // We assume that id<..>, id, and "Class" all represent tracked objects.
+  // We assume that id<..>, id, Class, and Class<..> all represent tracked
+  // objects.
   if (PT->isObjCIdType() || PT->isObjCQualifiedIdType() ||
-      PT->isObjCClassType())
+      PT->isObjCClassType() || PT->isObjCQualifiedClassType())
     return true;
   
   // Does the interface subclass NSObject?
diff --git a/lib/Checker/DivZeroChecker.cpp b/lib/Checker/DivZeroChecker.cpp
index e09a87149f5c..32e2a1782d10 100644
--- a/lib/Checker/DivZeroChecker.cpp
+++ b/lib/Checker/DivZeroChecker.cpp
@@ -40,10 +40,10 @@ void *DivZeroChecker::getTag() {
 void DivZeroChecker::PreVisitBinaryOperator(CheckerContext &C,
                                             const BinaryOperator *B) {
   BinaryOperator::Opcode Op = B->getOpcode();
-  if (Op != BinaryOperator::Div &&
-      Op != BinaryOperator::Rem &&
-      Op != BinaryOperator::DivAssign &&
-      Op != BinaryOperator::RemAssign)
+  if (Op != BO_Div &&
+      Op != BO_Rem &&
+      Op != BO_DivAssign &&
+      Op != BO_RemAssign)
     return;
 
   if (!B->getRHS()->getType()->isIntegerType() ||
diff --git a/lib/Checker/Environment.cpp b/lib/Checker/Environment.cpp
index 48152ceb46f0..02291f43500c 100644
--- a/lib/Checker/Environment.cpp
+++ b/lib/Checker/Environment.cpp
@@ -80,7 +80,7 @@ SVal Environment::GetSVal(const Stmt *E, ValueManager& ValMgr) const {
   return LookupExpr(E);
 }
 
-Environment EnvironmentManager::BindExpr(Environment Env, const Stmt *S,
+Environment EnvironmentManager::bindExpr(Environment Env, const Stmt *S,
                                          SVal V, bool Invalidate) {
   assert(S);
 
@@ -94,6 +94,16 @@ Environment EnvironmentManager::BindExpr(Environment Env, const Stmt *S,
   return Environment(F.Add(Env.ExprBindings, S, V));
 }
 
+static inline const Stmt *MakeLocation(const Stmt *S) {
+  return (const Stmt*) (((uintptr_t) S) | 0x1);
+}
+
+Environment EnvironmentManager::bindExprAndLocation(Environment Env,
+                                                    const Stmt *S,
+                                                    SVal location, SVal V) {
+  return Environment(F.Add(F.Add(Env.ExprBindings, MakeLocation(S), V), S, V));
+}
+
 namespace {
 class MarkLiveCallback : public SymbolVisitor {
   SymbolReaper &SymReaper;
@@ -115,6 +125,12 @@ static bool isBlockExprInCallers(const Stmt *E, const LocationContext *LC) {
   return false;
 }
 
+// In addition to mapping from Stmt * - > SVals in the Environment, we also
+// maintain a mapping from Stmt * -> SVals (locations) that were used during
+// a load and store.
+static inline bool IsLocation(const Stmt *S) {
+  return (bool) (((uintptr_t) S) & 0x1);
+}
 
 // RemoveDeadBindings:
 //  - Remove subexpression bindings.
@@ -123,7 +139,6 @@ static bool isBlockExprInCallers(const Stmt *E, const LocationContext *LC) {
 //   - Mark their reachable symbols live in SymbolReaper,
 //     see ScanReachableSymbols.
 //   - Mark the region in DRoots if the binding is a loc::MemRegionVal.
-
 Environment
 EnvironmentManager::RemoveDeadBindings(Environment Env,
                                        SymbolReaper &SymReaper,
@@ -136,12 +151,25 @@ EnvironmentManager::RemoveDeadBindings(Environment Env,
   // individually removing all the subexpression bindings (which will greatly
   // outnumber block-level expression bindings).
   Environment NewEnv = getInitialEnvironment();
+  
+  llvm::SmallVector<std::pair<const Stmt*, SVal>, 10> deferredLocations;
 
   // Iterate over the block-expr bindings.
   for (Environment::iterator I = Env.begin(), E = Env.end();
        I != E; ++I) {
 
     const Stmt *BlkExpr = I.getKey();
+    
+    // For recorded locations (used when evaluating loads and stores), we
+    // consider them live only when their associated normal expression is
+    // also live.
+    // NOTE: This assumes that loads/stores that evaluated to UnknownVal
+    // still have an entry in the map.
+    if (IsLocation(BlkExpr)) {
+      deferredLocations.push_back(std::make_pair(BlkExpr, I.getData()));
+      continue;
+    }
+    
     const SVal &X = I.getData();
 
     // Block-level expressions in callers are assumed always live.
@@ -186,6 +214,15 @@ EnvironmentManager::RemoveDeadBindings(Environment Env,
     if (X.isUndef() && cast<UndefinedVal>(X).getData())
       NewEnv.ExprBindings = F.Add(NewEnv.ExprBindings, BlkExpr, X);
   }
+  
+  // Go through he deferred locations and add them to the new environment if
+  // the correspond Stmt* is in the map as well.
+  for (llvm::SmallVectorImpl<std::pair<const Stmt*, SVal> >::iterator
+      I = deferredLocations.begin(), E = deferredLocations.end(); I != E; ++I) {
+    const Stmt *S = (Stmt*) (((uintptr_t) I->first) & (uintptr_t) ~0x1);
+    if (NewEnv.ExprBindings.lookup(S))
+      NewEnv.ExprBindings = F.Add(NewEnv.ExprBindings, I->first, I->second);
+  }
 
   return NewEnv;
 }
diff --git a/lib/Checker/FixedAddressChecker.cpp b/lib/Checker/FixedAddressChecker.cpp
index 4fce45bd35e8..29a3c3ae3533 100644
--- a/lib/Checker/FixedAddressChecker.cpp
+++ b/lib/Checker/FixedAddressChecker.cpp
@@ -40,7 +40,7 @@ void FixedAddressChecker::PreVisitBinaryOperator(CheckerContext &C,
   // Using a fixed address is not portable because that address will probably
   // not be valid in all environments or platforms.
 
-  if (B->getOpcode() != BinaryOperator::Assign)
+  if (B->getOpcode() != BO_Assign)
     return;
 
   QualType T = B->getType();
diff --git a/lib/Checker/FlatStore.cpp b/lib/Checker/FlatStore.cpp
index 64575b3c9783..21fa422166f0 100644
--- a/lib/Checker/FlatStore.cpp
+++ b/lib/Checker/FlatStore.cpp
@@ -44,11 +44,10 @@ public:
   }
 
   SVal ArrayToPointer(Loc Array);
-  const GRState *RemoveDeadBindings(GRState &state, 
-                           const StackFrameContext *LCtx,
+  Store RemoveDeadBindings(Store store, const StackFrameContext *LCtx,
                            SymbolReaper& SymReaper,
                          llvm::SmallVectorImpl<const MemRegion*>& RegionRoots){
-    return StateMgr.getPersistentState(state);
+    return store;
   }
 
   Store BindDecl(Store store, const VarRegion *VR, SVal initVal);
@@ -57,13 +56,10 @@ public:
 
   typedef llvm::DenseSet<SymbolRef> InvalidatedSymbols;
   
-  Store InvalidateRegion(Store store, const MemRegion *R, const Expr *E, 
-                         unsigned Count, InvalidatedSymbols *IS);
-  
   Store InvalidateRegions(Store store, const MemRegion * const *I,
                           const MemRegion * const *E, const Expr *Ex,
                           unsigned Count, InvalidatedSymbols *IS,
-                          bool invalidateGlobals);
+                          bool invalidateGlobals, InvalidatedRegions *Regions);
 
   void print(Store store, llvm::raw_ostream& Out, const char* nl, 
              const char *sep);
@@ -74,7 +70,14 @@ private:
     return RegionBindings(static_cast<const RegionBindings::TreeTy*>(store));
   }
 
-  Interval RegionToInterval(const MemRegion *R);
+  class RegionInterval {
+  public:
+    const MemRegion *R;
+    Interval I;
+    RegionInterval(const MemRegion *r, int64_t s, int64_t e) : R(r), I(s, e){}
+  };
+
+  RegionInterval RegionToInterval(const MemRegion *R);
 
   SVal RetrieveRegionWithNoBinding(const MemRegion *R, QualType T);
 };
@@ -86,11 +89,15 @@ StoreManager *clang::CreateFlatStoreManager(GRStateManager &StMgr) {
 
 SVal FlatStoreManager::Retrieve(Store store, Loc L, QualType T) {
   const MemRegion *R = cast<loc::MemRegionVal>(L).getRegion();
-  Interval I = RegionToInterval(R);
+  RegionInterval RI = RegionToInterval(R);
+  // FIXME: FlatStore should handle regions with unknown intervals.
+  if (!RI.R)
+    return UnknownVal();
+
   RegionBindings B = getRegionBindings(store);
-  const BindingVal *BV = B.lookup(R);
+  const BindingVal *BV = B.lookup(RI.R);
   if (BV) {
-    const SVal *V = BVFactory.Lookup(*BV, I);
+    const SVal *V = BVFactory.Lookup(*BV, RI.I);
     if (V)
       return *V;
     else
@@ -116,9 +123,12 @@ Store FlatStoreManager::Bind(Store store, Loc L, SVal val) {
   if (V)
     BV = *V;
 
-  Interval I = RegionToInterval(R);
-  BV = BVFactory.Add(BV, I, val);
-  B = RBFactory.Add(B, R, BV);
+  RegionInterval RI = RegionToInterval(R);
+  // FIXME: FlatStore should handle regions with unknown intervals.
+  if (!RI.R)
+    return B.getRoot();
+  BV = BVFactory.Add(BV, RI.I, val);
+  B = RBFactory.Add(B, RI.R, BV);
   return B.getRoot();
 }
 
@@ -139,7 +149,7 @@ SVal FlatStoreManager::ArrayToPointer(Loc Array) {
 
 Store FlatStoreManager::BindDecl(Store store, const VarRegion *VR, 
                                  SVal initVal) {
-  return store;
+  return Bind(store, ValMgr.makeLoc(VR), initVal);
 }
 
 Store FlatStoreManager::BindDeclWithNoInit(Store store, const VarRegion *VR) {
@@ -147,18 +157,12 @@ Store FlatStoreManager::BindDeclWithNoInit(Store store, const VarRegion *VR) {
 }
 
 Store FlatStoreManager::InvalidateRegions(Store store,
-                                            const MemRegion * const *I,
-                                            const MemRegion * const *E,
-                                            const Expr *Ex, unsigned Count,
-                                            InvalidatedSymbols *IS,
-                                            bool invalidateGlobals) {
-  assert(false && "Not implemented");
-  return store;
-}
-
-Store FlatStoreManager::InvalidateRegion(Store store, const MemRegion *R,
-                                         const Expr *E, unsigned Count,
-                                         InvalidatedSymbols *IS) {
+                                          const MemRegion * const *I,
+                                          const MemRegion * const *E,
+                                          const Expr *Ex, unsigned Count,
+                                          InvalidatedSymbols *IS,
+                                          bool invalidateGlobals,
+                                          InvalidatedRegions *Regions) {
   assert(false && "Not implemented");
   return store;
 }
@@ -170,15 +174,29 @@ void FlatStoreManager::print(Store store, llvm::raw_ostream& Out,
 void FlatStoreManager::iterBindings(Store store, BindingsHandler& f) {
 }
 
-Interval FlatStoreManager::RegionToInterval(const MemRegion *R) { 
+FlatStoreManager::RegionInterval 
+FlatStoreManager::RegionToInterval(const MemRegion *R) { 
   switch (R->getKind()) {
   case MemRegion::VarRegionKind: {
-    QualType T = cast<VarRegion>(R)->getValueType(Ctx);
-    uint64_t Size = Ctx.getTypeSize(T);
-    return Interval(0, Size-1);
+    QualType T = cast<VarRegion>(R)->getValueType();
+    int64_t Size = Ctx.getTypeSize(T);
+    return RegionInterval(R, 0, Size-1);
   }
+
+  case MemRegion::ElementRegionKind: 
+  case MemRegion::FieldRegionKind: {
+    RegionOffset Offset = R->getAsOffset();
+    // We cannot compute offset for all regions, for example, elements
+    // with symbolic offsets.
+    if (!Offset.getRegion())
+      return RegionInterval(0, 0, 0);
+    int64_t Start = Offset.getOffset();
+    int64_t Size = Ctx.getTypeSize(cast<TypedRegion>(R)->getValueType());
+    return RegionInterval(Offset.getRegion(), Start, Start+Size);
+  }
+
   default:
     llvm_unreachable("Region kind unhandled.");
-    return Interval(0, 0);
+    return RegionInterval(0, 0, 0);
   }
 }
diff --git a/lib/Checker/GRCXXExprEngine.cpp b/lib/Checker/GRCXXExprEngine.cpp
index 18e112cc8d36..a49989b5fda5 100644
--- a/lib/Checker/GRCXXExprEngine.cpp
+++ b/lib/Checker/GRCXXExprEngine.cpp
@@ -17,7 +17,7 @@
 
 using namespace clang;
 
-void GRExprEngine::EvalArguments(ExprIterator AI, ExprIterator AE,
+void GRExprEngine::EvalArguments(ConstExprIterator AI, ConstExprIterator AE,
                                  const FunctionProtoType *FnType, 
                                  ExplodedNode *Pred, ExplodedNodeSet &Dst) {
   llvm::SmallVector<CallExprWLItem, 20> WorkList;
@@ -55,7 +55,7 @@ const CXXThisRegion *GRExprEngine::getCXXThisRegion(const CXXMethodDecl *D,
   return ValMgr.getRegionManager().getCXXThisRegion(PT, SFC);
 }
 
-void GRExprEngine::CreateCXXTemporaryObject(Expr *Ex, ExplodedNode *Pred,
+void GRExprEngine::CreateCXXTemporaryObject(const Expr *Ex, ExplodedNode *Pred,
                                             ExplodedNodeSet &Dst) {
   ExplodedNodeSet Tmp;
   Visit(Ex, Pred, Tmp);
@@ -94,9 +94,7 @@ void GRExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E, SVal Dest,
   // Evaluate other arguments.
   ExplodedNodeSet ArgsEvaluated;
   const FunctionProtoType *FnType = CD->getType()->getAs<FunctionProtoType>();
-  EvalArguments(const_cast<CXXConstructExpr*>(E)->arg_begin(),
-                const_cast<CXXConstructExpr*>(E)->arg_end(),
-                FnType, Pred, ArgsEvaluated);
+  EvalArguments(E->arg_begin(), E->arg_end(), FnType, Pred, ArgsEvaluated);
   // The callee stack frame context used to create the 'this' parameter region.
   const StackFrameContext *SFC = AMgr.getStackFrame(CD, 
                                                     Pred->getLocationContext(),
@@ -104,11 +102,12 @@ void GRExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *E, SVal Dest,
 
   const CXXThisRegion *ThisR = getCXXThisRegion(E->getConstructor(), SFC);
 
-  CallEnter Loc(E, CD, Pred->getLocationContext());
+  CallEnter Loc(E, SFC->getAnalysisContext(), Pred->getLocationContext());
   for (ExplodedNodeSet::iterator NI = ArgsEvaluated.begin(),
                                  NE = ArgsEvaluated.end(); NI != NE; ++NI) {
     const GRState *state = GetState(*NI);
-    // Setup 'this' region.
+    // Setup 'this' region, so that the ctor is evaluated on the object pointed
+    // by 'Dest'.
     state = state->bindLoc(loc::MemRegionVal(ThisR), Dest);
     ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
     if (N)
@@ -126,9 +125,7 @@ void GRExprEngine::VisitCXXMemberCallExpr(const CXXMemberCallExpr *MCE,
 
   // Evaluate explicit arguments with a worklist.
   ExplodedNodeSet ArgsEvaluated;
-  EvalArguments(const_cast<CXXMemberCallExpr*>(MCE)->arg_begin(),
-                const_cast<CXXMemberCallExpr*>(MCE)->arg_end(),
-                FnType, Pred, ArgsEvaluated);
+  EvalArguments(MCE->arg_begin(), MCE->arg_end(), FnType, Pred, ArgsEvaluated);
  
   // Evaluate the implicit object argument.
   ExplodedNodeSet AllArgsEvaluated;
@@ -157,7 +154,7 @@ void GRExprEngine::VisitCXXMemberCallExpr(const CXXMemberCallExpr *MCE,
                                                     Builder->getBlock(), 
                                                     Builder->getIndex());
   const CXXThisRegion *ThisR = getCXXThisRegion(MD, SFC);
-  CallEnter Loc(MCE, MD, Pred->getLocationContext());
+  CallEnter Loc(MCE, SFC->getAnalysisContext(), Pred->getLocationContext());
   for (ExplodedNodeSet::iterator I = AllArgsEvaluated.begin(),
          E = AllArgsEvaluated.end(); I != E; ++I) {
     // Set up 'this' region.
@@ -169,7 +166,7 @@ void GRExprEngine::VisitCXXMemberCallExpr(const CXXMemberCallExpr *MCE,
   }
 }
 
-void GRExprEngine::VisitCXXNewExpr(CXXNewExpr *CNE, ExplodedNode *Pred,
+void GRExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
                                    ExplodedNodeSet &Dst) {
   if (CNE->isArray()) {
     // FIXME: allocating an array has not been handled.
@@ -177,7 +174,7 @@ void GRExprEngine::VisitCXXNewExpr(CXXNewExpr *CNE, ExplodedNode *Pred,
   }
 
   unsigned Count = Builder->getCurrentBlockCount();
-  DefinedOrUnknownSVal SymVal = getValueManager().getConjuredSymbolVal(NULL,CNE, 
+  DefinedOrUnknownSVal SymVal = getValueManager().getConjuredSymbolVal(NULL,CNE,
                                                          CNE->getType(), Count);
   const MemRegion *NewReg = cast<loc::MemRegionVal>(SymVal).getRegion();
 
@@ -201,10 +198,7 @@ void GRExprEngine::VisitCXXNewExpr(CXXNewExpr *CNE, ExplodedNode *Pred,
     const GRState *state = GetState(*I);
 
     if (ObjTy->isRecordType()) {
-      Store store = state->getStore();
-      StoreManager::InvalidatedSymbols IS;
-      store = getStoreManager().InvalidateRegion(store, EleReg, CNE, Count, &IS);
-      state = state->makeWithStore(store);
+      state = state->InvalidateRegion(EleReg, CNE, Count);
     } else {
       if (CNE->hasInitializer()) {
         SVal V = state->getSVal(*CNE->constructor_arg_begin());
@@ -220,8 +214,8 @@ void GRExprEngine::VisitCXXNewExpr(CXXNewExpr *CNE, ExplodedNode *Pred,
   }
 }
 
-void GRExprEngine::VisitCXXDeleteExpr(CXXDeleteExpr *CDE, ExplodedNode *Pred,
-                                      ExplodedNodeSet &Dst) {
+void GRExprEngine::VisitCXXDeleteExpr(const CXXDeleteExpr *CDE, 
+                                      ExplodedNode *Pred,ExplodedNodeSet &Dst) {
   // Should do more checking.
   ExplodedNodeSet ArgEvaluated;
   Visit(CDE->getArgument(), Pred, ArgEvaluated);
@@ -232,7 +226,7 @@ void GRExprEngine::VisitCXXDeleteExpr(CXXDeleteExpr *CDE, ExplodedNode *Pred,
   }
 }
 
-void GRExprEngine::VisitCXXThisExpr(CXXThisExpr *TE, ExplodedNode *Pred,
+void GRExprEngine::VisitCXXThisExpr(const CXXThisExpr *TE, ExplodedNode *Pred,
                                     ExplodedNodeSet &Dst) {
   // Get the this object region from StoreManager.
   const MemRegion *R =
diff --git a/lib/Checker/GRCoreEngine.cpp b/lib/Checker/GRCoreEngine.cpp
index a816186a307d..5125f366de2a 100644
--- a/lib/Checker/GRCoreEngine.cpp
+++ b/lib/Checker/GRCoreEngine.cpp
@@ -12,8 +12,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Checker/PathSensitive/AnalysisManager.h"
 #include "clang/Checker/PathSensitive/GRCoreEngine.h"
 #include "clang/Checker/PathSensitive/GRExprEngine.h"
+#include "clang/Index/TranslationUnit.h"
 #include "clang/AST/Expr.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/ADT/DenseMap.h"
@@ -24,6 +26,12 @@ using llvm::cast;
 using llvm::isa;
 using namespace clang;
 
+// This should be removed in the future.
+namespace clang {
+GRTransferFuncs* MakeCFRefCountTF(ASTContext& Ctx, bool GCEnabled,
+                                  const LangOptions& lopts);
+}
+
 //===----------------------------------------------------------------------===//
 // Worklist classes for exploration of reachable states.
 //===----------------------------------------------------------------------===//
@@ -118,47 +126,15 @@ GRWorkList* GRWorkList::MakeBFSBlockDFSContents() {
 //===----------------------------------------------------------------------===//
 // Core analysis engine.
 //===----------------------------------------------------------------------===//
-void GRCoreEngine::ProcessEndPath(GREndPathNodeBuilder& Builder) {
-  SubEngine.ProcessEndPath(Builder);
-}
-
-void GRCoreEngine::ProcessStmt(CFGElement E, GRStmtNodeBuilder& Builder) {
-  SubEngine.ProcessStmt(E, Builder);
-}
-
-bool GRCoreEngine::ProcessBlockEntrance(CFGBlock* Blk, const ExplodedNode *Pred,
-                                        GRBlockCounter BC) {
-  return SubEngine.ProcessBlockEntrance(Blk, Pred, BC);
-}
-
-void GRCoreEngine::ProcessBranch(Stmt* Condition, Stmt* Terminator,
-                   GRBranchNodeBuilder& Builder) {
-  SubEngine.ProcessBranch(Condition, Terminator, Builder);
-}
-
-void GRCoreEngine::ProcessIndirectGoto(GRIndirectGotoNodeBuilder& Builder) {
-  SubEngine.ProcessIndirectGoto(Builder);
-}
-
-void GRCoreEngine::ProcessSwitch(GRSwitchNodeBuilder& Builder) {
-  SubEngine.ProcessSwitch(Builder);
-}
-
-void GRCoreEngine::ProcessCallEnter(GRCallEnterNodeBuilder &Builder) {
-  SubEngine.ProcessCallEnter(Builder);
-}
-
-void GRCoreEngine::ProcessCallExit(GRCallExitNodeBuilder &Builder) {
-  SubEngine.ProcessCallExit(Builder);
-}
 
 /// ExecuteWorkList - Run the worklist algorithm for a maximum number of steps.
-bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps) {
+bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps,
+                                   const GRState *InitState) {
 
   if (G->num_roots() == 0) { // Initialize the analysis by constructing
     // the root if none exists.
 
-    CFGBlock* Entry = &(L->getCFG()->getEntry());
+    const CFGBlock* Entry = &(L->getCFG()->getEntry());
 
     assert (Entry->empty() &&
             "Entry block must be empty.");
@@ -167,7 +143,7 @@ bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps) {
             "Entry block must have 1 successor.");
 
     // Get the solitary successor.
-    CFGBlock* Succ = *(Entry->succ_begin());
+    const CFGBlock* Succ = *(Entry->succ_begin());
 
     // Construct an edge representing the
     // starting location in the function.
@@ -176,8 +152,11 @@ bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps) {
     // Set the current block counter to being empty.
     WList->setBlockCounter(BCounterFactory.GetEmptyCounter());
 
-    // Generate the root.
-    GenerateNode(StartLoc, getInitialState(L), 0);
+    if (!InitState)
+      // Generate the root.
+      GenerateNode(StartLoc, getInitialState(L), 0);
+    else
+      GenerateNode(StartLoc, InitState, 0);
   }
 
   while (Steps && WList->hasWork()) {
@@ -221,14 +200,25 @@ bool GRCoreEngine::ExecuteWorkList(const LocationContext *L, unsigned Steps) {
     }
   }
 
-  SubEngine.ProcessEndWorklist(WList->hasWork() || BlockAborted);
+  SubEngine.ProcessEndWorklist(hasWorkRemaining());
   return WList->hasWork();
 }
 
+void GRCoreEngine::ExecuteWorkListWithInitialState(const LocationContext *L, 
+                                                   unsigned Steps,
+                                                   const GRState *InitState, 
+                                                   ExplodedNodeSet &Dst) {
+  ExecuteWorkList(L, Steps, InitState);
+  for (llvm::SmallVectorImpl<ExplodedNode*>::iterator I = G->EndNodes.begin(), 
+                                           E = G->EndNodes.end(); I != E; ++I) {
+    Dst.Add(*I);
+  }
+}
+
 void GRCoreEngine::HandleCallEnter(const CallEnter &L, const CFGBlock *Block,
                                    unsigned Index, ExplodedNode *Pred) {
-  GRCallEnterNodeBuilder Builder(*this, Pred, L.getCallExpr(), L.getCallee(), 
-                                 Block, Index);
+  GRCallEnterNodeBuilder Builder(*this, Pred, L.getCallExpr(), 
+                                 L.getCalleeContext(), Block, Index);
   ProcessCallEnter(Builder);
 }
 
@@ -239,7 +229,7 @@ void GRCoreEngine::HandleCallExit(const CallExit &L, ExplodedNode *Pred) {
 
 void GRCoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
 
-  CFGBlock* Blk = L.getDst();
+  const CFGBlock* Blk = L.getDst();
 
   // Check if we are entering the EXIT block.
   if (Blk == &(L.getLocationContext()->getCFG()->getExit())) {
@@ -260,8 +250,9 @@ void GRCoreEngine::HandleBlockEdge(const BlockEdge& L, ExplodedNode* Pred) {
   if (ProcessBlockEntrance(Blk, Pred, WList->getBlockCounter()))
     GenerateNode(BlockEntrance(Blk, Pred->getLocationContext()),
                  Pred->State, Pred);
-  else
-    BlockAborted = true;
+  else {
+    blocksAborted.push_back(std::make_pair(L, Pred));
+  }
 }
 
 void GRCoreEngine::HandleBlockEntrance(const BlockEntrance& L,
@@ -284,9 +275,9 @@ void GRCoreEngine::HandleBlockEntrance(const BlockEntrance& L,
     HandleBlockExit(L.getBlock(), Pred);
 }
 
-void GRCoreEngine::HandleBlockExit(CFGBlock * B, ExplodedNode* Pred) {
+void GRCoreEngine::HandleBlockExit(const CFGBlock * B, ExplodedNode* Pred) {
 
-  if (Stmt* Term = B->getTerminator()) {
+  if (const Stmt* Term = B->getTerminator()) {
     switch (Term->getStmtClass()) {
       default:
         assert(false && "Analysis for this terminator not implemented.");
@@ -372,8 +363,8 @@ void GRCoreEngine::HandleBlockExit(CFGBlock * B, ExplodedNode* Pred) {
                Pred->State, Pred);
 }
 
-void GRCoreEngine::HandleBranch(Stmt* Cond, Stmt* Term, CFGBlock * B,
-                                ExplodedNode* Pred) {
+void GRCoreEngine::HandleBranch(const Stmt* Cond, const Stmt* Term, 
+                                const CFGBlock * B, ExplodedNode* Pred) {
   assert (B->succ_size() == 2);
 
   GRBranchNodeBuilder Builder(B, *(B->succ_begin()), *(B->succ_begin()+1),
@@ -382,7 +373,7 @@ void GRCoreEngine::HandleBranch(Stmt* Cond, Stmt* Term, CFGBlock * B,
   ProcessBranch(Cond, Term, Builder);
 }
 
-void GRCoreEngine::HandlePostStmt(const PostStmt& L, CFGBlock* B,
+void GRCoreEngine::HandlePostStmt(const PostStmt& L, const CFGBlock* B,
                                   unsigned StmtIdx, ExplodedNode* Pred) {
 
   assert (!B->empty());
@@ -415,7 +406,7 @@ void GRCoreEngine::GenerateNode(const ProgramPoint& Loc,
   if (IsNew) WList->Enqueue(Node);
 }
 
-GRStmtNodeBuilder::GRStmtNodeBuilder(CFGBlock* b, unsigned idx,
+GRStmtNodeBuilder::GRStmtNodeBuilder(const CFGBlock* b, unsigned idx,
                                      ExplodedNode* N, GRCoreEngine* e,
                                      GRStateManager &mgr)
   : Eng(*e), B(*b), Idx(idx), Pred(N), Mgr(mgr), Auditor(0),
@@ -438,7 +429,7 @@ void GRStmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   if (isa<CallEnter>(N->getLocation())) {
     // Still use the index of the CallExpr. It's needed to create the callee
     // StackFrameContext.
-    Eng.WList->Enqueue(N, B, Idx);
+    Eng.WList->Enqueue(N, &B, Idx);
     return;
   }
 
@@ -447,7 +438,7 @@ void GRStmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   if (Loc == N->getLocation()) {
     // Note: 'N' should be a fresh node because otherwise it shouldn't be
     // a member of Deferred.
-    Eng.WList->Enqueue(N, B, Idx+1);
+    Eng.WList->Enqueue(N, &B, Idx+1);
     return;
   }
 
@@ -456,10 +447,10 @@ void GRStmtNodeBuilder::GenerateAutoTransition(ExplodedNode* N) {
   Succ->addPredecessor(N, *Eng.G);
 
   if (IsNew)
-    Eng.WList->Enqueue(Succ, B, Idx+1);
+    Eng.WList->Enqueue(Succ, &B, Idx+1);
 }
 
-ExplodedNode* GRStmtNodeBuilder::MakeNode(ExplodedNodeSet& Dst, Stmt* S, 
+ExplodedNode* GRStmtNodeBuilder::MakeNode(ExplodedNodeSet& Dst, const Stmt* S, 
                                           ExplodedNode* Pred, const GRState* St,
                                           ProgramPoint::Kind K) {
   const GRState* PredState = GetState(Pred);
@@ -692,6 +683,59 @@ void GREndPathNodeBuilder::GenerateCallExitNode(const GRState *state) {
 
 void GRCallEnterNodeBuilder::GenerateNode(const GRState *state,
                                           const LocationContext *LocCtx) {
+  // Check if the callee is in the same translation unit.
+  if (CalleeCtx->getTranslationUnit() != 
+      Pred->getLocationContext()->getTranslationUnit()) {
+    // Create a new engine. We must be careful that the new engine should not
+    // reference data structures owned by the old engine.
+
+    AnalysisManager &OldMgr = Eng.SubEngine.getAnalysisManager();
+    
+    // Get the callee's translation unit.
+    idx::TranslationUnit *TU = CalleeCtx->getTranslationUnit();
+
+    // Create a new AnalysisManager with components of the callee's
+    // TranslationUnit.
+    // The Diagnostic is actually shared when we create ASTUnits from AST files.
+    AnalysisManager AMgr(TU->getASTContext(), TU->getDiagnostic(), 
+                         OldMgr.getLangOptions(), 
+                         OldMgr.getPathDiagnosticClient(),
+                         OldMgr.getStoreManagerCreator(),
+                         OldMgr.getConstraintManagerCreator(),
+                         OldMgr.getIndexer(),
+                         OldMgr.getMaxNodes(), OldMgr.getMaxLoop(),
+                         OldMgr.shouldVisualizeGraphviz(),
+                         OldMgr.shouldVisualizeUbigraph(),
+                         OldMgr.shouldPurgeDead(),
+                         OldMgr.shouldEagerlyAssume(),
+                         OldMgr.shouldTrimGraph(),
+                         OldMgr.shouldInlineCall(),
+                     OldMgr.getAnalysisContextManager().getUseUnoptimizedCFG());
+    llvm::OwningPtr<GRTransferFuncs> TF(MakeCFRefCountTF(AMgr.getASTContext(),
+                                                         /* GCEnabled */ false,
+                                                        AMgr.getLangOptions()));
+    // Create the new engine.
+    GRExprEngine NewEng(AMgr, TF.take());
+
+    // Create the new LocationContext.
+    AnalysisContext *NewAnaCtx = AMgr.getAnalysisContext(CalleeCtx->getDecl(), 
+                                               CalleeCtx->getTranslationUnit());
+    const StackFrameContext *OldLocCtx = cast<StackFrameContext>(LocCtx);
+    const StackFrameContext *NewLocCtx = AMgr.getStackFrame(NewAnaCtx, 
+                                               OldLocCtx->getParent(),
+                                               OldLocCtx->getCallSite(),
+                                               OldLocCtx->getCallSiteBlock(), 
+                                               OldLocCtx->getIndex());
+
+    // Now create an initial state for the new engine.
+    const GRState *NewState = NewEng.getStateManager().MarshalState(state,
+                                                                    NewLocCtx);
+    ExplodedNodeSet ReturnNodes;
+    NewEng.ExecuteWorkListWithInitialState(NewLocCtx, AMgr.getMaxNodes(), 
+                                           NewState, ReturnNodes);
+    return;
+  }
+
   // Get the callee entry block.
   const CFGBlock *Entry = &(LocCtx->getCFG()->getEntry());
   assert(Entry->empty());
@@ -721,6 +765,6 @@ void GRCallExitNodeBuilder::GenerateNode(const GRState *state) {
   ExplodedNode *Node = Eng.G->getNode(Loc, state, &isNew);
   Node->addPredecessor(const_cast<ExplodedNode*>(Pred), *Eng.G);
   if (isNew)
-    Eng.WList->Enqueue(Node, *const_cast<CFGBlock*>(LocCtx->getCallSiteBlock()),
+    Eng.WList->Enqueue(Node, LocCtx->getCallSiteBlock(),
                        LocCtx->getIndex() + 1);
 }
diff --git a/lib/Checker/GRExprEngine.cpp b/lib/Checker/GRExprEngine.cpp
index 07fee9d39e49..feb826e4a640 100644
--- a/lib/Checker/GRExprEngine.cpp
+++ b/lib/Checker/GRExprEngine.cpp
@@ -169,17 +169,18 @@ public:
 // Checker worklist routines.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::CheckerVisit(Stmt *S, ExplodedNodeSet &Dst,
-                                ExplodedNodeSet &Src, bool isPrevisit) {
+void GRExprEngine::CheckerVisit(const Stmt *S, ExplodedNodeSet &Dst,
+                                ExplodedNodeSet &Src, CallbackKind Kind) {
 
   // Determine if we already have a cached 'CheckersOrdered' vector
-  // specifically tailored for the provided <Stmt kind, isPrevisit>.  This
+  // specifically tailored for the provided <CallbackKind, Stmt kind>.  This
   // can reduce the number of checkers actually called.
   CheckersOrdered *CO = &Checkers;
   llvm::OwningPtr<CheckersOrdered> NewCO;
-  
-  const std::pair<unsigned, unsigned> &K =
-    std::make_pair((unsigned)S->getStmtClass(), isPrevisit ? 1U : 0U);
+
+  // The cache key is made up of the and the callback kind (pre- or post-visit)
+  // and the statement kind.
+  CallbackTag K = GetCallbackTag(Kind, S->getStmtClass());
 
   CheckersOrdered *& CO_Ref = COCache[K];
   
@@ -204,7 +205,10 @@ void GRExprEngine::CheckerVisit(Stmt *S, ExplodedNodeSet &Dst,
   ExplodedNodeSet *PrevSet = &Src;
   unsigned checkersEvaluated = 0;
 
-  for (CheckersOrdered::iterator I=CO->begin(), E=CO->end(); I!=E; ++I){
+  for (CheckersOrdered::iterator I=CO->begin(), E=CO->end(); I!=E; ++I) {
+    // If all nodes are sunk, bail out early.
+    if (PrevSet->empty())
+      break;
     ExplodedNodeSet *CurrSet = 0;
     if (I+1 == E)
       CurrSet = &Dst;
@@ -219,8 +223,8 @@ void GRExprEngine::CheckerVisit(Stmt *S, ExplodedNodeSet &Dst,
     for (ExplodedNodeSet::iterator NI = PrevSet->begin(), NE = PrevSet->end();
          NI != NE; ++NI) {
 
-      checker->GR_Visit(*CurrSet, *Builder, *this, S, *NI, tag, isPrevisit,
-                        respondsToCallback);
+      checker->GR_Visit(*CurrSet, *Builder, *this, S, *NI, tag,
+                        Kind == PreVisitStmtCallback, respondsToCallback);
       
     }
 
@@ -235,7 +239,9 @@ void GRExprEngine::CheckerVisit(Stmt *S, ExplodedNodeSet &Dst,
   
   // If we built NewCO, check if we called all the checkers.  This is important
   // so that we know that we accurately determined the entire set of checkers
-  // that responds to this callback.
+  // that responds to this callback.  Note that 'checkersEvaluated' might
+  // not be the same as Checkers.size() if one of the Checkers generates
+  // a sink node.
   if (NewCO.get() && checkersEvaluated == Checkers.size())
     CO_Ref = NewCO.take();
 
@@ -300,10 +306,9 @@ bool GRExprEngine::CheckerEvalCall(const CallExpr *CE,
 
 // FIXME: This is largely copy-paste from CheckerVisit().  Need to
 // unify.
-void GRExprEngine::CheckerVisitBind(const Stmt *AssignE, const Stmt *StoreE,
-                                    ExplodedNodeSet &Dst,
-                                    ExplodedNodeSet &Src,
-                                    SVal location, SVal val, bool isPrevisit) {
+void GRExprEngine::CheckerVisitBind(const Stmt *StoreE, ExplodedNodeSet &Dst,
+                                    ExplodedNodeSet &Src, SVal location,
+                                    SVal val, bool isPrevisit) {
 
   if (Checkers.empty()) {
     Dst.insert(Src);
@@ -328,7 +333,7 @@ void GRExprEngine::CheckerVisitBind(const Stmt *AssignE, const Stmt *StoreE,
 
     for (ExplodedNodeSet::iterator NI = PrevSet->begin(), NE = PrevSet->end();
          NI != NE; ++NI)
-      checker->GR_VisitBind(*CurrSet, *Builder, *this, AssignE, StoreE,
+      checker->GR_VisitBind(*CurrSet, *Builder, *this, StoreE,
                             *NI, tag, location, val, isPrevisit);
 
     // Update which NodeSet is the current one.
@@ -457,7 +462,7 @@ const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
         break;
 
       SVal V = state->getSVal(loc::MemRegionVal(R));
-      SVal Constraint_untested = EvalBinOp(state, BinaryOperator::GT, V,
+      SVal Constraint_untested = EvalBinOp(state, BO_GT, V,
                                            ValMgr.makeZeroVal(T),
                                            getContext().IntTy);
 
@@ -499,29 +504,135 @@ const GRState* GRExprEngine::getInitialState(const LocationContext *InitLoc) {
 ///  logic for handling assumptions on symbolic values.
 const GRState *GRExprEngine::ProcessAssume(const GRState *state, SVal cond,
                                            bool assumption) {
-  for (CheckersOrdered::iterator I = Checkers.begin(), E = Checkers.end();
-        I != E; ++I) {
+  // Determine if we already have a cached 'CheckersOrdered' vector
+  // specifically tailored for processing assumptions.  This
+  // can reduce the number of checkers actually called.
+  CheckersOrdered *CO = &Checkers;
+  llvm::OwningPtr<CheckersOrdered> NewCO;
 
-    if (!state)
-      return NULL;
+  CallbackTag K = GetCallbackTag(ProcessAssumeCallback);
+  CheckersOrdered *& CO_Ref = COCache[K];
+
+  if (!CO_Ref) {
+    // If we have no previously cached CheckersOrdered vector for this
+    // statement kind, then create one.
+    NewCO.reset(new CheckersOrdered);
+  }
+  else {
+    // Use the already cached set.
+    CO = CO_Ref;
+  }
+
+  if (!CO->empty()) {
+    // Let the checkers have a crack at the assume before the transfer functions
+    // get their turn.
+    for (CheckersOrdered::iterator I = CO->begin(), E = CO->end(); I!=E; ++I) {
 
-    state = I->second->EvalAssume(state, cond, assumption);
+      // If any checker declares the state infeasible (or if it starts that
+      // way), bail out.
+      if (!state)
+        return NULL;
+
+      Checker *C = I->second;
+      bool respondsToCallback = true;
+
+      state = C->EvalAssume(state, cond, assumption, &respondsToCallback);
+
+      // Check if we're building the cache of checkers that care about Assumes.
+      if (NewCO.get() && respondsToCallback)
+        NewCO->push_back(*I);
+    }
+
+    // If we got through all the checkers, and we built a list of those that
+    // care about Assumes, save it.
+    if (NewCO.get())
+      CO_Ref = NewCO.take();
   }
 
+  // If the state is infeasible at this point, bail out.
   if (!state)
     return NULL;
 
   return TF->EvalAssume(state, cond, assumption);
 }
 
+bool GRExprEngine::WantsRegionChangeUpdate(const GRState* state) {
+  CallbackTag K = GetCallbackTag(EvalRegionChangesCallback);
+  CheckersOrdered *CO = COCache[K];
+
+  if (!CO)
+    CO = &Checkers;
+
+  for (CheckersOrdered::iterator I = CO->begin(), E = CO->end(); I != E; ++I) {
+    Checker *C = I->second;
+    if (C->WantsRegionChangeUpdate(state))
+      return true;
+  }
+
+  return false;
+}
+
+const GRState *
+GRExprEngine::ProcessRegionChanges(const GRState *state,
+                                   const MemRegion * const *Begin,
+                                   const MemRegion * const *End) {
+  // FIXME: Most of this method is copy-pasted from ProcessAssume.
+
+  // Determine if we already have a cached 'CheckersOrdered' vector
+  // specifically tailored for processing region changes.  This
+  // can reduce the number of checkers actually called.
+  CheckersOrdered *CO = &Checkers;
+  llvm::OwningPtr<CheckersOrdered> NewCO;
+
+  CallbackTag K = GetCallbackTag(EvalRegionChangesCallback);
+  CheckersOrdered *& CO_Ref = COCache[K];
+
+  if (!CO_Ref) {
+    // If we have no previously cached CheckersOrdered vector for this
+    // callback, then create one.
+    NewCO.reset(new CheckersOrdered);
+  }
+  else {
+    // Use the already cached set.
+    CO = CO_Ref;
+  }
+
+  // If there are no checkers, just return the state as is.
+  if (CO->empty())
+    return state;
+
+  for (CheckersOrdered::iterator I = CO->begin(), E = CO->end(); I != E; ++I) {
+    // If any checker declares the state infeasible (or if it starts that way),
+    // bail out.
+    if (!state)
+      return NULL;
+
+    Checker *C = I->second;
+    bool respondsToCallback = true;
+
+    state = C->EvalRegionChanges(state, Begin, End, &respondsToCallback);
+
+    // See if we're building a cache of checkers that care about region changes.
+    if (NewCO.get() && respondsToCallback)
+      NewCO->push_back(*I);
+  }
+
+  // If we got through all the checkers, and we built a list of those that
+  // care about region changes, save it.
+  if (NewCO.get())
+    CO_Ref = NewCO.take();
+
+  return state;
+}
+
 void GRExprEngine::ProcessEndWorklist(bool hasWorkRemaining) {
   for (CheckersOrdered::iterator I = Checkers.begin(), E = Checkers.end();
        I != E; ++I) {
-    I->second->VisitEndAnalysis(G, BR, hasWorkRemaining);
+    I->second->VisitEndAnalysis(G, BR, *this);
   }
 }
 
-void GRExprEngine::ProcessStmt(CFGElement CE, GRStmtNodeBuilder& builder) {
+void GRExprEngine::ProcessStmt(const CFGElement CE,GRStmtNodeBuilder& builder) {
   CurrentStmt = CE.getStmt();
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
                                 CurrentStmt->getLocStart(),
@@ -535,15 +646,23 @@ void GRExprEngine::ProcessStmt(CFGElement CE, GRStmtNodeBuilder& builder) {
     Builder->setAuditor(BatchAuditor.get());
 
   // Create the cleaned state.
-  const ExplodedNode *BasePred = Builder->getBasePredecessor();
+  const LocationContext *LC = EntryNode->getLocationContext();
+  SymbolReaper SymReaper(LC, CurrentStmt, SymMgr);
+
+  if (AMgr.shouldPurgeDead()) {
+    const GRState *St = EntryNode->getState();
 
-  SymbolReaper SymReaper(BasePred->getLocationContext(), CurrentStmt, SymMgr);
+    for (CheckersOrdered::iterator I = Checkers.begin(), E = Checkers.end();
+         I != E; ++I) {
+      Checker *checker = I->second;
+      checker->MarkLiveSymbols(St, SymReaper);
+    }
 
-  CleanedState = AMgr.shouldPurgeDead()
-    ? StateMgr.RemoveDeadBindings(EntryNode->getState(), 
-                         BasePred->getLocationContext()->getCurrentStackFrame(),
-                                  SymReaper)
-    : EntryNode->getState();
+    const StackFrameContext *SFC = LC->getCurrentStackFrame();
+    CleanedState = StateMgr.RemoveDeadBindings(St, SFC, SymReaper);
+  } else {
+    CleanedState = EntryNode->getState();
+  }
 
   // Process any special transfer function for dead symbols.
   ExplodedNodeSet Tmp;
@@ -625,7 +744,8 @@ void GRExprEngine::ProcessStmt(CFGElement CE, GRStmtNodeBuilder& builder) {
   Builder = NULL;
 }
 
-void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
+void GRExprEngine::Visit(const Stmt* S, ExplodedNode* Pred, 
+                         ExplodedNodeSet& Dst) {
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
                                 S->getLocStart(),
                                 "Error evaluating statement");
@@ -641,7 +761,6 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
 
   switch (S->getStmtClass()) {
     // C++ stuff we don't support yet.
-    case Stmt::CXXBindReferenceExprClass:
     case Stmt::CXXBindTemporaryExprClass:
     case Stmt::CXXCatchStmtClass:
     case Stmt::CXXConstructExprClass:
@@ -740,13 +859,13 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
       break;
 
     case Stmt::BinaryOperatorClass: {
-      BinaryOperator* B = cast<BinaryOperator>(S);
+      const BinaryOperator* B = cast<BinaryOperator>(S);
 
       if (B->isLogicalOp()) {
         VisitLogicalExpr(B, Pred, Dst);
         break;
       }
-      else if (B->getOpcode() == BinaryOperator::Comma) {
+      else if (B->getOpcode() == BO_Comma) {
         const GRState* state = GetState(Pred);
         MakeNode(Dst, B, Pred, state->BindExpr(B, state->getSVal(B->getRHS())));
         break;
@@ -766,25 +885,25 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
 
     case Stmt::CallExprClass:
     case Stmt::CXXOperatorCallExprClass: {
-      CallExpr* C = cast<CallExpr>(S);
+      const CallExpr* C = cast<CallExpr>(S);
       VisitCall(C, Pred, C->arg_begin(), C->arg_end(), Dst, false);
       break;
     }
 
     case Stmt::CXXMemberCallExprClass: {
-      CXXMemberCallExpr *MCE = cast<CXXMemberCallExpr>(S);
+      const CXXMemberCallExpr *MCE = cast<CXXMemberCallExpr>(S);
       VisitCXXMemberCallExpr(MCE, Pred, Dst);
       break;
     }
 
     case Stmt::CXXNewExprClass: {
-      CXXNewExpr *NE = cast<CXXNewExpr>(S);
+      const CXXNewExpr *NE = cast<CXXNewExpr>(S);
       VisitCXXNewExpr(NE, Pred, Dst);
       break;
     }
 
     case Stmt::CXXDeleteExprClass: {
-      CXXDeleteExpr *CDE = cast<CXXDeleteExpr>(S);
+      const CXXDeleteExpr *CDE = cast<CXXDeleteExpr>(S);
       VisitCXXDeleteExpr(CDE, Pred, Dst);
       break;
     }
@@ -792,7 +911,7 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
       //        the CFG do not model them as explicit control-flow.
 
     case Stmt::ChooseExprClass: { // __builtin_choose_expr
-      ChooseExpr* C = cast<ChooseExpr>(S);
+      const ChooseExpr* C = cast<ChooseExpr>(S);
       VisitGuardedExpr(C, C->getLHS(), C->getRHS(), Pred, Dst);
       break;
     }
@@ -806,7 +925,7 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
       break;
 
     case Stmt::ConditionalOperatorClass: { // '?' operator
-      ConditionalOperator* C = cast<ConditionalOperator>(S);
+      const ConditionalOperator* C = cast<ConditionalOperator>(S);
       VisitGuardedExpr(C, C->getLHS(), C->getRHS(), Pred, Dst);
       break;
     }
@@ -836,7 +955,7 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
     case Stmt::CXXReinterpretCastExprClass:
     case Stmt::CXXConstCastExprClass:
     case Stmt::CXXFunctionalCastExprClass: {
-      CastExpr* C = cast<CastExpr>(S);
+      const CastExpr* C = cast<CastExpr>(S);
       VisitCast(C, C->getSubExpr(), Pred, Dst, false);
       break;
     }
@@ -893,7 +1012,7 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
       break;
 
     case Stmt::StmtExprClass: {
-      StmtExpr* SE = cast<StmtExpr>(S);
+      const StmtExpr* SE = cast<StmtExpr>(S);
 
       if (SE->getSubStmt()->body_empty()) {
         // Empty statement expression.
@@ -924,8 +1043,8 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
       break;
 
     case Stmt::UnaryOperatorClass: {
-      UnaryOperator *U = cast<UnaryOperator>(S);
-      if (AMgr.shouldEagerlyAssume()&&(U->getOpcode() == UnaryOperator::LNot)) {
+      const UnaryOperator *U = cast<UnaryOperator>(S);
+      if (AMgr.shouldEagerlyAssume()&&(U->getOpcode() == UO_LNot)) {
         ExplodedNodeSet Tmp;
         VisitUnaryOperator(U, Pred, Tmp, false);
         EvalEagerlyAssume(Dst, Tmp, U);
@@ -943,7 +1062,7 @@ void GRExprEngine::Visit(Stmt* S, ExplodedNode* Pred, ExplodedNodeSet& Dst) {
   }
 }
 
-void GRExprEngine::VisitLValue(Expr* Ex, ExplodedNode* Pred,
+void GRExprEngine::VisitLValue(const Expr* Ex, ExplodedNode* Pred,
                                ExplodedNodeSet& Dst) {
 
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
@@ -984,7 +1103,7 @@ void GRExprEngine::VisitLValue(Expr* Ex, ExplodedNode* Pred,
 
     case Stmt::CallExprClass:
     case Stmt::CXXOperatorCallExprClass: {
-      CallExpr *C = cast<CallExpr>(Ex);
+      const CallExpr *C = cast<CallExpr>(Ex);
       assert(CalleeReturnsReferenceOrRecord(C));
       VisitCall(C, Pred, C->arg_begin(), C->arg_end(), Dst, true);
       break;
@@ -1000,7 +1119,7 @@ void GRExprEngine::VisitLValue(Expr* Ex, ExplodedNode* Pred,
 
     case Stmt::ImplicitCastExprClass:
     case Stmt::CStyleCastExprClass: {
-      CastExpr *C = cast<CastExpr>(Ex);
+      const CastExpr *C = cast<CastExpr>(Ex);
       QualType T = Ex->getType();
       VisitCast(C, C->getSubExpr(), Pred, Dst, true);
       break;
@@ -1015,7 +1134,7 @@ void GRExprEngine::VisitLValue(Expr* Ex, ExplodedNode* Pred,
       return;
 
     case Stmt::ObjCMessageExprClass: {
-      ObjCMessageExpr *ME = cast<ObjCMessageExpr>(Ex);
+      const ObjCMessageExpr *ME = cast<ObjCMessageExpr>(Ex);
       assert(ReceiverReturnsReferenceOrRecord(ME));
       VisitObjCMessageExpr(ME, Pred, Dst, true);
       return;
@@ -1056,6 +1175,9 @@ void GRExprEngine::VisitLValue(Expr* Ex, ExplodedNode* Pred,
     // In C++, binding an rvalue to a reference requires to create an object.
     case Stmt::CXXBoolLiteralExprClass:
     case Stmt::IntegerLiteralClass:
+    case Stmt::CharacterLiteralClass:
+    case Stmt::FloatingLiteralClass:
+    case Stmt::ImaginaryLiteralClass:
       CreateCXXTemporaryObject(Ex, Pred, Dst);
       return;
 
@@ -1081,7 +1203,8 @@ void GRExprEngine::VisitLValue(Expr* Ex, ExplodedNode* Pred,
 // Block entrance.  (Update counters).
 //===----------------------------------------------------------------------===//
 
-bool GRExprEngine::ProcessBlockEntrance(CFGBlock* B, const ExplodedNode *Pred,
+bool GRExprEngine::ProcessBlockEntrance(const CFGBlock* B, 
+                                        const ExplodedNode *Pred,
                                         GRBlockCounter BC) {
   return BC.getNumVisited(Pred->getLocationContext()->getCurrentStackFrame(), 
                           B->getBlockID()) < AMgr.getMaxLoop();
@@ -1091,7 +1214,7 @@ bool GRExprEngine::ProcessBlockEntrance(CFGBlock* B, const ExplodedNode *Pred,
 // Generic node creation.
 //===----------------------------------------------------------------------===//
 
-ExplodedNode* GRExprEngine::MakeNode(ExplodedNodeSet& Dst, Stmt* S,
+ExplodedNode* GRExprEngine::MakeNode(ExplodedNodeSet& Dst, const Stmt* S,
                                      ExplodedNode* Pred, const GRState* St,
                                      ProgramPoint::Kind K, const void *tag) {
   assert (Builder && "GRStmtNodeBuilder not present.");
@@ -1105,8 +1228,8 @@ ExplodedNode* GRExprEngine::MakeNode(ExplodedNodeSet& Dst, Stmt* S,
 //===----------------------------------------------------------------------===//
 
 const GRState* GRExprEngine::MarkBranch(const GRState* state,
-                                           Stmt* Terminator,
-                                           bool branchTaken) {
+                                        const Stmt* Terminator,
+                                        bool branchTaken) {
 
   switch (Terminator->getStmtClass()) {
     default:
@@ -1114,10 +1237,10 @@ const GRState* GRExprEngine::MarkBranch(const GRState* state,
 
     case Stmt::BinaryOperatorClass: { // '&&' and '||'
 
-      BinaryOperator* B = cast<BinaryOperator>(Terminator);
+      const BinaryOperator* B = cast<BinaryOperator>(Terminator);
       BinaryOperator::Opcode Op = B->getOpcode();
 
-      assert (Op == BinaryOperator::LAnd || Op == BinaryOperator::LOr);
+      assert (Op == BO_LAnd || Op == BO_LOr);
 
       // For &&, if we take the true branch, then the value of the whole
       // expression is that of the RHS expression.
@@ -1125,21 +1248,21 @@ const GRState* GRExprEngine::MarkBranch(const GRState* state,
       // For ||, if we take the false branch, then the value of the whole
       // expression is that of the RHS expression.
 
-      Expr* Ex = (Op == BinaryOperator::LAnd && branchTaken) ||
-                 (Op == BinaryOperator::LOr && !branchTaken)
-               ? B->getRHS() : B->getLHS();
+      const Expr* Ex = (Op == BO_LAnd && branchTaken) ||
+                       (Op == BO_LOr && !branchTaken)
+                       ? B->getRHS() : B->getLHS();
 
       return state->BindExpr(B, UndefinedVal(Ex));
     }
 
     case Stmt::ConditionalOperatorClass: { // ?:
 
-      ConditionalOperator* C = cast<ConditionalOperator>(Terminator);
+      const ConditionalOperator* C = cast<ConditionalOperator>(Terminator);
 
       // For ?, if branchTaken == true then the value is either the LHS or
       // the condition itself. (GNU extension).
 
-      Expr* Ex;
+      const Expr* Ex;
 
       if (branchTaken)
         Ex = C->getLHS() ? C->getLHS() : C->getCond();
@@ -1151,9 +1274,9 @@ const GRState* GRExprEngine::MarkBranch(const GRState* state,
 
     case Stmt::ChooseExprClass: { // ?:
 
-      ChooseExpr* C = cast<ChooseExpr>(Terminator);
+      const ChooseExpr* C = cast<ChooseExpr>(Terminator);
 
-      Expr* Ex = branchTaken ? C->getLHS() : C->getRHS();
+      const Expr* Ex = branchTaken ? C->getLHS() : C->getRHS();
       return state->BindExpr(C, UndefinedVal(Ex));
     }
   }
@@ -1165,16 +1288,16 @@ const GRState* GRExprEngine::MarkBranch(const GRState* state,
 /// This function returns the SVal bound to Condition->IgnoreCasts if all the
 //  cast(s) did was sign-extend the original value.
 static SVal RecoverCastedSymbol(GRStateManager& StateMgr, const GRState* state,
-                                Stmt* Condition, ASTContext& Ctx) {
+                                const Stmt* Condition, ASTContext& Ctx) {
 
-  Expr *Ex = dyn_cast<Expr>(Condition);
+  const Expr *Ex = dyn_cast<Expr>(Condition);
   if (!Ex)
     return UnknownVal();
 
   uint64_t bits = 0;
   bool bitsInit = false;
 
-  while (CastExpr *CE = dyn_cast<CastExpr>(Ex)) {
+  while (const CastExpr *CE = dyn_cast<CastExpr>(Ex)) {
     QualType T = CE->getType();
 
     if (!T->isIntegerType())
@@ -1198,7 +1321,7 @@ static SVal RecoverCastedSymbol(GRStateManager& StateMgr, const GRState* state,
   return state->getSVal(Ex);
 }
 
-void GRExprEngine::ProcessBranch(Stmt* Condition, Stmt* Term,
+void GRExprEngine::ProcessBranch(const Stmt* Condition, const Stmt* Term,
                                  GRBranchNodeBuilder& builder) {
 
   // Check for NULL conditions; e.g. "for(;;)"
@@ -1285,7 +1408,7 @@ void GRExprEngine::ProcessIndirectGoto(GRIndirectGotoNodeBuilder& builder) {
   typedef GRIndirectGotoNodeBuilder::iterator iterator;
 
   if (isa<loc::GotoLabel>(V)) {
-    LabelStmt* L = cast<loc::GotoLabel>(V).getLabel();
+    const LabelStmt* L = cast<loc::GotoLabel>(V).getLabel();
 
     for (iterator I=builder.begin(), E=builder.end(); I != E; ++I) {
       if (I.getLabel() == L) {
@@ -1314,7 +1437,8 @@ void GRExprEngine::ProcessIndirectGoto(GRIndirectGotoNodeBuilder& builder) {
 }
 
 
-void GRExprEngine::VisitGuardedExpr(Expr* Ex, Expr* L, Expr* R,
+void GRExprEngine::VisitGuardedExpr(const Expr* Ex, const Expr* L, 
+                                    const Expr* R,
                                     ExplodedNode* Pred, ExplodedNodeSet& Dst) {
 
   assert(Ex == CurrentStmt &&
@@ -1325,7 +1449,7 @@ void GRExprEngine::VisitGuardedExpr(Expr* Ex, Expr* L, Expr* R,
 
   assert (X.isUndef());
 
-  Expr *SE = (Expr*) cast<UndefinedVal>(X).getData();
+  const Expr *SE = (Expr*) cast<UndefinedVal>(X).getData();
   assert(SE);
   X = state->getSVal(SE);
 
@@ -1350,7 +1474,7 @@ void GRExprEngine::ProcessEndPath(GREndPathNodeBuilder& builder) {
 void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
   typedef GRSwitchNodeBuilder::iterator iterator;
   const GRState* state = builder.getState();
-  Expr* CondE = builder.getCondition();
+  const Expr* CondE = builder.getCondition();
   SVal  CondV_untested = state->getSVal(CondE);
 
   if (CondV_untested.isUndef()) {
@@ -1363,10 +1487,12 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
   DefinedOrUnknownSVal CondV = cast<DefinedOrUnknownSVal>(CondV_untested);
 
   const GRState *DefaultSt = state;
-  bool defaultIsFeasible = false;
+  
+  iterator I = builder.begin(), EI = builder.end();
+  bool defaultIsFeasible = I == EI;
 
-  for (iterator I = builder.begin(), EI = builder.end(); I != EI; ++I) {
-    CaseStmt* Case = cast<CaseStmt>(I.getCase());
+  for ( ; I != EI; ++I) {
+    const CaseStmt* Case = I.getCase();
 
     // Evaluate the LHS of the case value.
     Expr::EvalResult V1;
@@ -1382,7 +1508,7 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
     // Get the RHS of the case, if it exists.
     Expr::EvalResult V2;
 
-    if (Expr* E = Case->getRHS()) {
+    if (const Expr* E = Case->getRHS()) {
       b = E->Evaluate(V2, getContext());
       assert(b && V2.Val.isInt() && !V2.HasSideEffects
              && "Case condition must evaluate to an integer constant.");
@@ -1440,15 +1566,14 @@ void GRExprEngine::ProcessSwitch(GRSwitchNodeBuilder& builder) {
 }
 
 void GRExprEngine::ProcessCallEnter(GRCallEnterNodeBuilder &B) {
-  const FunctionDecl *FD = B.getCallee();
-  const StackFrameContext *LocCtx = AMgr.getStackFrame(FD, 
-                                                       B.getLocationContext(),
-                                                       B.getCallExpr(),
-                                                       B.getBlock(),
-                                                       B.getIndex());
+  const StackFrameContext *LocCtx 
+    = AMgr.getStackFrame(B.getCalleeContext(),
+                         B.getLocationContext(),
+                         B.getCallExpr(),
+                         B.getBlock(),
+                         B.getIndex());
 
-  const GRState *state = B.getState();
-  state = getStoreManager().EnterStackFrame(state, LocCtx);
+  const GRState *state = B.getState()->EnterStackFrame(LocCtx);
 
   B.GenerateNode(state, LocCtx);
 }
@@ -1490,11 +1615,11 @@ void GRExprEngine::ProcessCallExit(GRCallExitNodeBuilder &B) {
 // Transfer functions: logical operations ('&&', '||').
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitLogicalExpr(BinaryOperator* B, ExplodedNode* Pred,
+void GRExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode* Pred,
                                     ExplodedNodeSet& Dst) {
 
-  assert(B->getOpcode() == BinaryOperator::LAnd ||
-         B->getOpcode() == BinaryOperator::LOr);
+  assert(B->getOpcode() == BO_LAnd ||
+         B->getOpcode() == BO_LOr);
 
   assert(B==CurrentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(B));
 
@@ -1534,7 +1659,7 @@ void GRExprEngine::VisitLogicalExpr(BinaryOperator* B, ExplodedNode* Pred,
     // We took the LHS expression.  Depending on whether we are '&&' or
     // '||' we know what the value of the expression is via properties of
     // the short-circuiting.
-    X = ValMgr.makeIntVal(B->getOpcode() == BinaryOperator::LAnd ? 0U : 1U,
+    X = ValMgr.makeIntVal(B->getOpcode() == BO_LAnd ? 0U : 1U,
                           B->getType());
     MakeNode(Dst, B, Pred, state->BindExpr(B, X));
   }
@@ -1544,7 +1669,7 @@ void GRExprEngine::VisitLogicalExpr(BinaryOperator* B, ExplodedNode* Pred,
 // Transfer functions: Loads and stores.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitBlockExpr(BlockExpr *BE, ExplodedNode *Pred,
+void GRExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
                                   ExplodedNodeSet &Dst) {
 
   ExplodedNodeSet Tmp;
@@ -1557,21 +1682,21 @@ void GRExprEngine::VisitBlockExpr(BlockExpr *BE, ExplodedNode *Pred,
            ProgramPoint::PostLValueKind);
 
   // Post-visit the BlockExpr.
-  CheckerVisit(BE, Dst, Tmp, false);
+  CheckerVisit(BE, Dst, Tmp, PostVisitStmtCallback);
 }
 
-void GRExprEngine::VisitDeclRefExpr(DeclRefExpr *Ex, ExplodedNode *Pred,
+void GRExprEngine::VisitDeclRefExpr(const DeclRefExpr *Ex, ExplodedNode *Pred,
                                     ExplodedNodeSet &Dst, bool asLValue) {
   VisitCommonDeclRefExpr(Ex, Ex->getDecl(), Pred, Dst, asLValue);
 }
 
-void GRExprEngine::VisitBlockDeclRefExpr(BlockDeclRefExpr *Ex,
+void GRExprEngine::VisitBlockDeclRefExpr(const BlockDeclRefExpr *Ex,
                                          ExplodedNode *Pred,
                                     ExplodedNodeSet &Dst, bool asLValue) {
   VisitCommonDeclRefExpr(Ex, Ex->getDecl(), Pred, Dst, asLValue);
 }
 
-void GRExprEngine::VisitCommonDeclRefExpr(Expr *Ex, const NamedDecl *D,
+void GRExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
                                           ExplodedNode *Pred,
                                           ExplodedNodeSet &Dst, bool asLValue) {
 
@@ -1618,12 +1743,12 @@ void GRExprEngine::VisitCommonDeclRefExpr(Expr *Ex, const NamedDecl *D,
 }
 
 /// VisitArraySubscriptExpr - Transfer function for array accesses
-void GRExprEngine::VisitArraySubscriptExpr(ArraySubscriptExpr* A,
+void GRExprEngine::VisitArraySubscriptExpr(const ArraySubscriptExpr* A,
                                            ExplodedNode* Pred,
                                            ExplodedNodeSet& Dst, bool asLValue){
 
-  Expr* Base = A->getBase()->IgnoreParens();
-  Expr* Idx  = A->getIdx()->IgnoreParens();
+  const Expr* Base = A->getBase()->IgnoreParens();
+  const Expr* Idx  = A->getIdx()->IgnoreParens();
   ExplodedNodeSet Tmp;
 
   if (Base->getType()->isVectorType()) {
@@ -1641,7 +1766,7 @@ void GRExprEngine::VisitArraySubscriptExpr(ArraySubscriptExpr* A,
     Visit(Idx, *I1, Tmp2);     // Evaluate the index.
 
     ExplodedNodeSet Tmp3;
-    CheckerVisit(A, Tmp3, Tmp2, true);
+    CheckerVisit(A, Tmp3, Tmp2, PreVisitStmtCallback);
 
     for (ExplodedNodeSet::iterator I2=Tmp3.begin(),E2=Tmp3.end();I2!=E2; ++I2) {
       const GRState* state = GetState(*I2);
@@ -1658,7 +1783,7 @@ void GRExprEngine::VisitArraySubscriptExpr(ArraySubscriptExpr* A,
 }
 
 /// VisitMemberExpr - Transfer function for member expressions.
-void GRExprEngine::VisitMemberExpr(MemberExpr* M, ExplodedNode* Pred,
+void GRExprEngine::VisitMemberExpr(const MemberExpr* M, ExplodedNode* Pred,
                                    ExplodedNodeSet& Dst, bool asLValue) {
 
   Expr* Base = M->getBase()->IgnoreParens();
@@ -1689,16 +1814,15 @@ void GRExprEngine::VisitMemberExpr(MemberExpr* M, ExplodedNode* Pred,
 
 /// EvalBind - Handle the semantics of binding a value to a specific location.
 ///  This method is used by EvalStore and (soon) VisitDeclStmt, and others.
-void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, Stmt *AssignE,
-                            Stmt* StoreE, ExplodedNode* Pred,
-                            const GRState* state, SVal location, SVal Val,
-                            bool atDeclInit) {
+void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, const Stmt* StoreE,
+                            ExplodedNode* Pred, const GRState* state,
+                            SVal location, SVal Val, bool atDeclInit) {
 
 
   // Do a previsit of the bind.
   ExplodedNodeSet CheckedSet, Src;
   Src.Add(Pred);
-  CheckerVisitBind(AssignE, StoreE, CheckedSet, Src, location, Val, true);
+  CheckerVisitBind(StoreE, CheckedSet, Src, location, Val, true);
 
   for (ExplodedNodeSet::iterator I = CheckedSet.begin(), E = CheckedSet.end();
        I!=E; ++I) {
@@ -1731,6 +1855,10 @@ void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, Stmt *AssignE,
     // The next thing to do is check if the GRTransferFuncs object wants to
     // update the state based on the new binding.  If the GRTransferFunc object
     // doesn't do anything, just auto-propagate the current state.
+    
+    // NOTE: We use 'AssignE' for the location of the PostStore if 'AssignE'
+    // is non-NULL.  Checkers typically care about 
+    
     GRStmtNodeBuilderRef BuilderRef(Dst, *Builder, *this, *I, newState, StoreE,
                                     newState != state);
 
@@ -1740,12 +1868,14 @@ void GRExprEngine::EvalBind(ExplodedNodeSet& Dst, Stmt *AssignE,
 
 /// EvalStore - Handle the semantics of a store via an assignment.
 ///  @param Dst The node set to store generated state nodes
-///  @param Ex The expression representing the location of the store
+///  @param AssignE The assignment expression if the store happens in an 
+///         assignment.
+///  @param LocatioinE The location expression that is stored to.
 ///  @param state The current simulation state
 ///  @param location The location to store the value
 ///  @param Val The value to be stored
-void GRExprEngine::EvalStore(ExplodedNodeSet& Dst, Expr *AssignE,
-                             Expr* StoreE,
+void GRExprEngine::EvalStore(ExplodedNodeSet& Dst, const Expr *AssignE,
+                             const Expr* LocationE,
                              ExplodedNode* Pred,
                              const GRState* state, SVal location, SVal Val,
                              const void *tag) {
@@ -1754,7 +1884,7 @@ void GRExprEngine::EvalStore(ExplodedNodeSet& Dst, Expr *AssignE,
 
   // Evaluate the location (checks for bad dereferences).
   ExplodedNodeSet Tmp;
-  EvalLocation(Tmp, StoreE, Pred, state, location, tag, false);
+  EvalLocation(Tmp, LocationE, Pred, state, location, tag, false);
 
   if (Tmp.empty())
     return;
@@ -1765,12 +1895,16 @@ void GRExprEngine::EvalStore(ExplodedNodeSet& Dst, Expr *AssignE,
                                                    ProgramPoint::PostStoreKind);
   SaveAndRestore<const void*> OldTag(Builder->Tag, tag);
 
-  // Proceed with the store.
+  // Proceed with the store.  We use AssignE as the anchor for the PostStore
+  // ProgramPoint if it is non-NULL, and LocationE otherwise.
+  const Expr *StoreE = AssignE ? AssignE : LocationE;
+
   for (ExplodedNodeSet::iterator NI=Tmp.begin(), NE=Tmp.end(); NI!=NE; ++NI)
-    EvalBind(Dst, AssignE, StoreE, *NI, GetState(*NI), location, Val);
+    EvalBind(Dst, StoreE, *NI, GetState(*NI), location, Val);
 }
 
-void GRExprEngine::EvalLoad(ExplodedNodeSet& Dst, Expr *Ex, ExplodedNode* Pred,
+void GRExprEngine::EvalLoad(ExplodedNodeSet& Dst, const Expr *Ex, 
+                            ExplodedNode* Pred,
                             const GRState* state, SVal location,
                             const void *tag, QualType LoadTy) {
 
@@ -1780,7 +1914,7 @@ void GRExprEngine::EvalLoad(ExplodedNodeSet& Dst, Expr *Ex, ExplodedNode* Pred,
   if (const TypedRegion *TR =
         dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
 
-    QualType ValTy = TR->getValueType(getContext());
+    QualType ValTy = TR->getValueType();
     if (const ReferenceType *RT = ValTy->getAs<ReferenceType>()) {
       static int loadReferenceTag = 0;
       ExplodedNodeSet Tmp;
@@ -1800,7 +1934,7 @@ void GRExprEngine::EvalLoad(ExplodedNodeSet& Dst, Expr *Ex, ExplodedNode* Pred,
   EvalLoadCommon(Dst, Ex, Pred, state, location, tag, LoadTy);
 }
 
-void GRExprEngine::EvalLoadCommon(ExplodedNodeSet& Dst, Expr *Ex,
+void GRExprEngine::EvalLoadCommon(ExplodedNodeSet& Dst, const Expr *Ex,
                                   ExplodedNode* Pred,
                                   const GRState* state, SVal location,
                                   const void *tag, QualType LoadTy) {
@@ -1834,7 +1968,7 @@ void GRExprEngine::EvalLoadCommon(ExplodedNodeSet& Dst, Expr *Ex,
   }
 }
 
-void GRExprEngine::EvalLocation(ExplodedNodeSet &Dst, Stmt *S,
+void GRExprEngine::EvalLocation(ExplodedNodeSet &Dst, const Stmt *S,
                                 ExplodedNode* Pred,
                                 const GRState* state, SVal location,
                                 const void *tag, bool isLoad) {
@@ -1885,21 +2019,34 @@ bool GRExprEngine::InlineCall(ExplodedNodeSet &Dst, const CallExpr *CE,
   if (!FD)
     return false;
 
-  if (!FD->hasBody(FD))
-    return false;
+  // Check if the function definition is in the same translation unit.
+  if (FD->hasBody(FD)) {
+    // Now we have the definition of the callee, create a CallEnter node.
+    CallEnter Loc(CE, AMgr.getAnalysisContext(FD), Pred->getLocationContext());
+
+    ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
+    Dst.Add(N);
+    return true;
+  }
 
-  // Now we have the definition of the callee, create a CallEnter node.
-  CallEnter Loc(CE, FD, Pred->getLocationContext());
+  // Check if we can find the function definition in other translation units.
+  if (AMgr.hasIndexer()) {
+    const AnalysisContext *C = AMgr.getAnalysisContextInAnotherTU(FD);
+    if (C == 0)
+      return false;
 
-  ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
-  if (N)
+    CallEnter Loc(CE, C, Pred->getLocationContext());
+    ExplodedNode *N = Builder->generateNode(Loc, state, Pred);
     Dst.Add(N);
-  return true;
+    return true;
+  }
+
+  return false;
 }
 
-void GRExprEngine::VisitCall(CallExpr* CE, ExplodedNode* Pred,
-                             CallExpr::arg_iterator AI,
-                             CallExpr::arg_iterator AE,
+void GRExprEngine::VisitCall(const CallExpr* CE, ExplodedNode* Pred,
+                             CallExpr::const_arg_iterator AI,
+                             CallExpr::const_arg_iterator AE,
                              ExplodedNodeSet& Dst, bool asLValue) {
 
   // Determine the type of function we're calling (if available).
@@ -1946,7 +2093,7 @@ void GRExprEngine::VisitCall(CallExpr* CE, ExplodedNode* Pred,
 
   // Now process the call itself.
   ExplodedNodeSet DstTmp;
-  Expr* Callee = CE->getCallee()->IgnoreParens();
+  const Expr* Callee = CE->getCallee()->IgnoreParens();
 
   for (ExplodedNodeSet::iterator NI=ArgsEvaluated.begin(),
                                  NE=ArgsEvaluated.end(); NI != NE; ++NI) {
@@ -1954,7 +2101,7 @@ void GRExprEngine::VisitCall(CallExpr* CE, ExplodedNode* Pred,
     ExplodedNodeSet DstTmp2;
     Visit(Callee, *NI, DstTmp2);
     // Perform the previsit of the CallExpr, storing the results in DstTmp.
-    CheckerVisit(CE, DstTmp, DstTmp2, true);
+    CheckerVisit(CE, DstTmp, DstTmp2, PreVisitStmtCallback);
   }
 
   // Finally, evaluate the function call.  We try each of the checkers
@@ -2009,7 +2156,7 @@ void GRExprEngine::VisitCall(CallExpr* CE, ExplodedNode* Pred,
   // If the callee returns a reference and we want an rvalue, skip this check
   // and do the load.
   if (!(!asLValue && CalleeReturnsReference(CE))) {
-    CheckerVisit(CE, Dst, DstTmp3, false);
+    CheckerVisit(CE, Dst, DstTmp3, PostVisitStmtCallback);
     return;
   }
 
@@ -2019,7 +2166,7 @@ void GRExprEngine::VisitCall(CallExpr* CE, ExplodedNode* Pred,
   // FIXME: This conversion doesn't actually happen unless the result
   //  of CallExpr is consumed by another expression.
   ExplodedNodeSet DstTmp4;
-  CheckerVisit(CE, DstTmp4, DstTmp3, false);
+  CheckerVisit(CE, DstTmp4, DstTmp3, PostVisitStmtCallback);
   QualType LoadTy = CE->getType();
 
   static int *ConvertToRvalueTag = 0;
@@ -2039,7 +2186,7 @@ static std::pair<const void*,const void*> EagerlyAssumeTag
   = std::pair<const void*,const void*>(&EagerlyAssumeTag,static_cast<void*>(0));
 
 void GRExprEngine::EvalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
-                                     Expr *Ex) {
+                                     const Expr *Ex) {
   for (ExplodedNodeSet::iterator I=Src.begin(), E=Src.end(); I!=E; ++I) {
     ExplodedNode *Pred = *I;
 
@@ -2082,10 +2229,11 @@ void GRExprEngine::EvalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
 // Transfer function: Objective-C ivar references.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitObjCIvarRefExpr(ObjCIvarRefExpr* Ex, ExplodedNode* Pred,
+void GRExprEngine::VisitObjCIvarRefExpr(const ObjCIvarRefExpr* Ex, 
+                                        ExplodedNode* Pred,
                                         ExplodedNodeSet& Dst, bool asLValue) {
 
-  Expr* Base = cast<Expr>(Ex->getBase());
+  const Expr* Base = cast<Expr>(Ex->getBase());
   ExplodedNodeSet Tmp;
   Visit(Base, Pred, Tmp);
 
@@ -2105,7 +2253,7 @@ void GRExprEngine::VisitObjCIvarRefExpr(ObjCIvarRefExpr* Ex, ExplodedNode* Pred,
 // Transfer function: Objective-C fast enumeration 'for' statements.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S,
+void GRExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt* S,
                                      ExplodedNode* Pred, ExplodedNodeSet& Dst) {
 
   // ObjCForCollectionStmts are processed in two places.  This method
@@ -2133,11 +2281,11 @@ void GRExprEngine::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S,
   //    container is empty.  Thus this transfer function will by default
   //    result in state splitting.
 
-  Stmt* elem = S->getElement();
+  const Stmt* elem = S->getElement();
   SVal ElementV;
 
-  if (DeclStmt* DS = dyn_cast<DeclStmt>(elem)) {
-    VarDecl* ElemD = cast<VarDecl>(DS->getSingleDecl());
+  if (const DeclStmt* DS = dyn_cast<DeclStmt>(elem)) {
+    const VarDecl* ElemD = cast<VarDecl>(DS->getSingleDecl());
     assert (ElemD->getInit() == 0);
     ElementV = GetState(Pred)->getLValue(ElemD, Pred->getLocationContext());
     VisitObjCForCollectionStmtAux(S, Pred, Dst, ElementV);
@@ -2153,12 +2301,12 @@ void GRExprEngine::VisitObjCForCollectionStmt(ObjCForCollectionStmt* S,
   }
 }
 
-void GRExprEngine::VisitObjCForCollectionStmtAux(ObjCForCollectionStmt* S,
+void GRExprEngine::VisitObjCForCollectionStmtAux(const ObjCForCollectionStmt* S,
                                        ExplodedNode* Pred, ExplodedNodeSet& Dst,
                                                  SVal ElementV) {
 
   // Check if the location we are writing back to is a null pointer.
-  Stmt* elem = S->getElement();
+  const Stmt* elem = S->getElement();
   ExplodedNodeSet Tmp;
   EvalLocation(Tmp, elem, Pred, GetState(Pred), ElementV, NULL, false);
 
@@ -2182,7 +2330,7 @@ void GRExprEngine::VisitObjCForCollectionStmtAux(ObjCForCollectionStmt* S,
         // FIXME: The proper thing to do is to really iterate over the
         //  container.  We will do this with dispatch logic to the store.
         //  For now, just 'conjure' up a symbolic value.
-        QualType T = R->getValueType(getContext());
+        QualType T = R->getValueType();
         assert(Loc::IsLocType(T));
         unsigned Count = Builder->getCurrentBlockCount();
         SymbolRef Sym = SymMgr.getConjuredSymbol(elem, T, Count);
@@ -2207,23 +2355,24 @@ void GRExprEngine::VisitObjCForCollectionStmtAux(ObjCForCollectionStmt* S,
 namespace {
 class ObjCMsgWLItem {
 public:
-  ObjCMessageExpr::arg_iterator I;
+  ObjCMessageExpr::const_arg_iterator I;
   ExplodedNode *N;
 
-  ObjCMsgWLItem(const ObjCMessageExpr::arg_iterator &i, ExplodedNode *n)
+  ObjCMsgWLItem(const ObjCMessageExpr::const_arg_iterator &i, ExplodedNode *n)
     : I(i), N(n) {}
 };
 } // end anonymous namespace
 
-void GRExprEngine::VisitObjCMessageExpr(ObjCMessageExpr* ME, ExplodedNode* Pred,
+void GRExprEngine::VisitObjCMessageExpr(const ObjCMessageExpr* ME, 
+                                        ExplodedNode* Pred,
                                         ExplodedNodeSet& Dst, bool asLValue){
 
   // Create a worklist to process both the arguments.
   llvm::SmallVector<ObjCMsgWLItem, 20> WL;
 
   // But first evaluate the receiver (if any).
-  ObjCMessageExpr::arg_iterator AI = ME->arg_begin(), AE = ME->arg_end();
-  if (Expr *Receiver = ME->getInstanceReceiver()) {
+  ObjCMessageExpr::const_arg_iterator AI = ME->arg_begin(), AE = ME->arg_end();
+  if (const Expr *Receiver = ME->getInstanceReceiver()) {
     ExplodedNodeSet Tmp;
     Visit(Receiver, Pred, Tmp);
 
@@ -2261,7 +2410,7 @@ void GRExprEngine::VisitObjCMessageExpr(ObjCMessageExpr* ME, ExplodedNode* Pred,
 
   // Now that the arguments are processed, handle the previsits checks.
   ExplodedNodeSet DstPrevisit;
-  CheckerVisit(ME, DstPrevisit, ArgsEvaluated, true);
+  CheckerVisit(ME, DstPrevisit, ArgsEvaluated, PreVisitStmtCallback);
 
   // Proceed with evaluate the message expression.
   ExplodedNodeSet DstEval;
@@ -2364,7 +2513,7 @@ void GRExprEngine::VisitObjCMessageExpr(ObjCMessageExpr* ME, ExplodedNode* Pred,
   // Finally, perform the post-condition check of the ObjCMessageExpr and store
   // the created nodes in 'Dst'.
   if (!(!asLValue && ReceiverReturnsReference(ME))) {
-    CheckerVisit(ME, Dst, DstEval, false);
+    CheckerVisit(ME, Dst, DstEval, PostVisitStmtCallback);
     return;
   }
 
@@ -2374,7 +2523,7 @@ void GRExprEngine::VisitObjCMessageExpr(ObjCMessageExpr* ME, ExplodedNode* Pred,
   // FIXME: This conversion doesn't actually happen unless the result
   //  of ObjCMessageExpr is consumed by another expression.
   ExplodedNodeSet DstRValueConvert;
-  CheckerVisit(ME, DstRValueConvert, DstEval, false);
+  CheckerVisit(ME, DstRValueConvert, DstEval, PostVisitStmtCallback);
   QualType LoadTy = ME->getType();
 
   static int *ConvertToRvalueTag = 0;
@@ -2390,8 +2539,9 @@ void GRExprEngine::VisitObjCMessageExpr(ObjCMessageExpr* ME, ExplodedNode* Pred,
 // Transfer functions: Miscellaneous statements.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
-                             ExplodedNodeSet &Dst, bool asLValue) {
+void GRExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex, 
+                             ExplodedNode *Pred, ExplodedNodeSet &Dst, 
+                             bool asLValue) {
   ExplodedNodeSet S1;
   QualType T = CastE->getType();
   QualType ExTy = Ex->getType();
@@ -2406,7 +2556,7 @@ void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
     Visit(Ex, Pred, S1);
 
   ExplodedNodeSet S2;
-  CheckerVisit(CastE, S2, S1, true);
+  CheckerVisit(CastE, S2, S1, PreVisitStmtCallback);
 
   // If we are evaluating the cast in an lvalue context, we implicitly want
   // the cast to evaluate to a location.
@@ -2417,14 +2567,14 @@ void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
   }
 
   switch (CastE->getCastKind()) {
-  case CastExpr::CK_ToVoid:
+  case CK_ToVoid:
     assert(!asLValue);
     for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I)
       Dst.Add(*I);
     return;
 
-  case CastExpr::CK_NoOp:
-  case CastExpr::CK_FunctionToPointerDecay:
+  case CK_NoOp:
+  case CK_FunctionToPointerDecay:
     for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I) {
       // Copy the SVal of Ex to CastE.
       ExplodedNode *N = *I;
@@ -2435,20 +2585,21 @@ void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
     }
     return;
 
-  case CastExpr::CK_Unknown:
-  case CastExpr::CK_ArrayToPointerDecay:
-  case CastExpr::CK_BitCast:
-  case CastExpr::CK_LValueBitCast:
-  case CastExpr::CK_IntegralCast:
-  case CastExpr::CK_IntegralToPointer:
-  case CastExpr::CK_PointerToIntegral:
-  case CastExpr::CK_IntegralToFloating:
-  case CastExpr::CK_FloatingToIntegral:
-  case CastExpr::CK_FloatingCast:
-  case CastExpr::CK_AnyPointerToObjCPointerCast:
-  case CastExpr::CK_AnyPointerToBlockPointerCast:
-  case CastExpr::CK_DerivedToBase:
-  case CastExpr::CK_UncheckedDerivedToBase: {
+  case CK_Unknown:
+  case CK_ArrayToPointerDecay:
+  case CK_BitCast:
+  case CK_LValueBitCast:
+  case CK_IntegralCast:
+  case CK_IntegralToPointer:
+  case CK_PointerToIntegral:
+  case CK_IntegralToFloating:
+  case CK_FloatingToIntegral:
+  case CK_FloatingCast:
+  case CK_AnyPointerToObjCPointerCast:
+  case CK_AnyPointerToBlockPointerCast:
+  case CK_DerivedToBase:
+  case CK_UncheckedDerivedToBase:
+  case CK_ObjCObjectLValueCast: {
     // Delegate to SValuator to process.
     for (ExplodedNodeSet::iterator I = S2.begin(), E = S2.end(); I != E; ++I) {
       ExplodedNode* N = *I;
@@ -2462,16 +2613,16 @@ void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
   }
   
   // Various C++ casts that are not handled yet.
-  case CastExpr::CK_Dynamic:  
-  case CastExpr::CK_ToUnion:
-  case CastExpr::CK_BaseToDerived:
-  case CastExpr::CK_NullToMemberPointer:
-  case CastExpr::CK_BaseToDerivedMemberPointer:
-  case CastExpr::CK_DerivedToBaseMemberPointer:
-  case CastExpr::CK_UserDefinedConversion:
-  case CastExpr::CK_ConstructorConversion:
-  case CastExpr::CK_VectorSplat:
-  case CastExpr::CK_MemberPointerToBoolean: {
+  case CK_Dynamic:  
+  case CK_ToUnion:
+  case CK_BaseToDerived:
+  case CK_NullToMemberPointer:
+  case CK_BaseToDerivedMemberPointer:
+  case CK_DerivedToBaseMemberPointer:
+  case CK_UserDefinedConversion:
+  case CK_ConstructorConversion:
+  case CK_VectorSplat:
+  case CK_MemberPointerToBoolean: {
     SaveAndRestore<bool> OldSink(Builder->BuildSinks);
     Builder->BuildSinks = true;
     MakeNode(Dst, CastE, Pred, GetState(Pred));
@@ -2480,11 +2631,12 @@ void GRExprEngine::VisitCast(CastExpr *CastE, Expr *Ex, ExplodedNode *Pred,
   }
 }
 
-void GRExprEngine::VisitCompoundLiteralExpr(CompoundLiteralExpr* CL,
+void GRExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr* CL,
                                             ExplodedNode* Pred,
                                             ExplodedNodeSet& Dst,
                                             bool asLValue) {
-  InitListExpr* ILE = cast<InitListExpr>(CL->getInitializer()->IgnoreParens());
+  const InitListExpr* ILE 
+    = cast<InitListExpr>(CL->getInitializer()->IgnoreParens());
   ExplodedNodeSet Tmp;
   Visit(ILE, Pred, Tmp);
 
@@ -2502,17 +2654,17 @@ void GRExprEngine::VisitCompoundLiteralExpr(CompoundLiteralExpr* CL,
   }
 }
 
-void GRExprEngine::VisitDeclStmt(DeclStmt *DS, ExplodedNode *Pred,
+void GRExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
                                  ExplodedNodeSet& Dst) {
 
   // The CFG has one DeclStmt per Decl.
-  Decl* D = *DS->decl_begin();
+  const Decl* D = *DS->decl_begin();
 
   if (!D || !isa<VarDecl>(D))
     return;
 
   const VarDecl* VD = dyn_cast<VarDecl>(D);
-  Expr* InitEx = const_cast<Expr*>(VD->getInit());
+  const Expr* InitEx = VD->getInit();
 
   // FIXME: static variables may have an initializer, but the second
   //  time a function is called those values may not be current.
@@ -2534,7 +2686,7 @@ void GRExprEngine::VisitDeclStmt(DeclStmt *DS, ExplodedNode *Pred,
     Tmp.Add(Pred);
 
   ExplodedNodeSet Tmp2;
-  CheckerVisit(DS, Tmp2, Tmp, true);
+  CheckerVisit(DS, Tmp2, Tmp, PreVisitStmtCallback);
 
   for (ExplodedNodeSet::iterator I=Tmp2.begin(), E=Tmp2.end(); I!=E; ++I) {
     ExplodedNode *N = *I;
@@ -2555,7 +2707,7 @@ void GRExprEngine::VisitDeclStmt(DeclStmt *DS, ExplodedNode *Pred,
                                                Builder->getCurrentBlockCount());
       }
 
-      EvalBind(Dst, DS, DS, *I, state,
+      EvalBind(Dst, DS, *I, state,
                loc::MemRegionVal(state->getRegion(VD, LC)), InitVal, true);
     }
     else {
@@ -2565,10 +2717,10 @@ void GRExprEngine::VisitDeclStmt(DeclStmt *DS, ExplodedNode *Pred,
   }
 }
 
-void GRExprEngine::VisitCondInit(VarDecl *VD, Stmt *S,
+void GRExprEngine::VisitCondInit(const VarDecl *VD, const Stmt *S,
                                  ExplodedNode *Pred, ExplodedNodeSet& Dst) {
 
-  Expr* InitEx = VD->getInit();
+  const Expr* InitEx = VD->getInit();
   ExplodedNodeSet Tmp;
   Visit(InitEx, Pred, Tmp);
 
@@ -2587,7 +2739,7 @@ void GRExprEngine::VisitCondInit(VarDecl *VD, Stmt *S,
                                             Builder->getCurrentBlockCount());
     }
 
-    EvalBind(Dst, S, S, N, state,
+    EvalBind(Dst, S, N, state,
              loc::MemRegionVal(state->getRegion(VD, LC)), InitVal, true);
   }
 }
@@ -2599,16 +2751,16 @@ class InitListWLItem {
 public:
   llvm::ImmutableList<SVal> Vals;
   ExplodedNode* N;
-  InitListExpr::reverse_iterator Itr;
+  InitListExpr::const_reverse_iterator Itr;
 
   InitListWLItem(ExplodedNode* n, llvm::ImmutableList<SVal> vals,
-                 InitListExpr::reverse_iterator itr)
+                 InitListExpr::const_reverse_iterator itr)
   : Vals(vals), N(n), Itr(itr) {}
 };
 }
 
 
-void GRExprEngine::VisitInitListExpr(InitListExpr* E, ExplodedNode* Pred,
+void GRExprEngine::VisitInitListExpr(const InitListExpr* E, ExplodedNode* Pred,
                                      ExplodedNodeSet& Dst) {
 
   const GRState* state = GetState(Pred);
@@ -2630,7 +2782,7 @@ void GRExprEngine::VisitInitListExpr(InitListExpr* E, ExplodedNode* Pred,
     llvm::SmallVector<InitListWLItem, 10> WorkList;
     WorkList.reserve(NumInitElements);
     WorkList.push_back(InitListWLItem(Pred, StartVals, E->rbegin()));
-    InitListExpr::reverse_iterator ItrEnd = E->rend();
+    InitListExpr::const_reverse_iterator ItrEnd = E->rend();
     assert(!(E->rbegin() == E->rend()));
 
     // Process the worklist until it is empty.
@@ -2641,7 +2793,7 @@ void GRExprEngine::VisitInitListExpr(InitListExpr* E, ExplodedNode* Pred,
       ExplodedNodeSet Tmp;
       Visit(*X.Itr, X.N, Tmp);
 
-      InitListExpr::reverse_iterator NewItr = X.Itr + 1;
+      InitListExpr::const_reverse_iterator NewItr = X.Itr + 1;
 
       for (ExplodedNodeSet::iterator NI=Tmp.begin(),NE=Tmp.end();NI!=NE;++NI) {
         // Get the last initializer value.
@@ -2673,7 +2825,7 @@ void GRExprEngine::VisitInitListExpr(InitListExpr* E, ExplodedNode* Pred,
   if (Loc::IsLocType(T) || T->isIntegerType()) {
     assert (E->getNumInits() == 1);
     ExplodedNodeSet Tmp;
-    Expr* Init = E->getInit(0);
+    const Expr* Init = E->getInit(0);
     Visit(Init, Pred, Tmp);
     for (ExplodedNodeSet::iterator I=Tmp.begin(), EI=Tmp.end(); I != EI; ++I) {
       state = GetState(*I);
@@ -2686,7 +2838,7 @@ void GRExprEngine::VisitInitListExpr(InitListExpr* E, ExplodedNode* Pred,
 }
 
 /// VisitSizeOfAlignOfExpr - Transfer function for sizeof(type).
-void GRExprEngine::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr* Ex,
+void GRExprEngine::VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr* Ex,
                                           ExplodedNode* Pred,
                                           ExplodedNodeSet& Dst) {
   QualType T = Ex->getTypeOfArgument();
@@ -2709,7 +2861,7 @@ void GRExprEngine::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr* Ex,
 
       // Get the size by getting the extent of the sub-expression.
       // First, visit the sub-expression to find its region.
-      Expr *Arg = Ex->getArgumentExpr();
+      const Expr *Arg = Ex->getArgumentExpr();
       ExplodedNodeSet Tmp;
       VisitLValue(Arg, Pred, Tmp);
 
@@ -2751,8 +2903,8 @@ void GRExprEngine::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr* Ex,
               ValMgr.makeIntVal(amt.getQuantity(), Ex->getType())));
 }
 
-void GRExprEngine::VisitOffsetOfExpr(OffsetOfExpr* OOE, ExplodedNode* Pred,
-                                     ExplodedNodeSet& Dst) {
+void GRExprEngine::VisitOffsetOfExpr(const OffsetOfExpr* OOE, 
+                                     ExplodedNode* Pred, ExplodedNodeSet& Dst) {
   Expr::EvalResult Res;
   if (OOE->Evaluate(Res, getContext()) && Res.Val.isInt()) {
     const APSInt &IV = Res.Val.getInt();
@@ -2767,7 +2919,8 @@ void GRExprEngine::VisitOffsetOfExpr(OffsetOfExpr* OOE, ExplodedNode* Pred,
   Dst.Add(Pred);
 }
 
-void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
+void GRExprEngine::VisitUnaryOperator(const UnaryOperator* U, 
+                                      ExplodedNode* Pred,
                                       ExplodedNodeSet& Dst, bool asLValue) {
 
   switch (U->getOpcode()) {
@@ -2775,9 +2928,9 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
     default:
       break;
 
-    case UnaryOperator::Deref: {
+    case UO_Deref: {
 
-      Expr* Ex = U->getSubExpr()->IgnoreParens();
+      const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       Visit(Ex, Pred, Tmp);
 
@@ -2796,9 +2949,9 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
       return;
     }
 
-    case UnaryOperator::Real: {
+    case UO_Real: {
 
-      Expr* Ex = U->getSubExpr()->IgnoreParens();
+      const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       Visit(Ex, Pred, Tmp);
 
@@ -2811,7 +2964,7 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
           continue;
         }
 
-        // For all other types, UnaryOperator::Real is an identity operation.
+        // For all other types, UO_Real is an identity operation.
         assert (U->getType() == Ex->getType());
         const GRState* state = GetState(*I);
         MakeNode(Dst, U, *I, state->BindExpr(U, state->getSVal(Ex)));
@@ -2820,9 +2973,9 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
       return;
     }
 
-    case UnaryOperator::Imag: {
+    case UO_Imag: {
 
-      Expr* Ex = U->getSubExpr()->IgnoreParens();
+      const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       Visit(Ex, Pred, Tmp);
 
@@ -2834,8 +2987,7 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
           continue;
         }
 
-        // For all other types, UnaryOperator::Float returns 0.
-        assert (Ex->getType()->isIntegerType());
+        // For all other types, UO_Imag returns 0.
         const GRState* state = GetState(*I);
         SVal X = ValMgr.makeZeroVal(Ex->getType());
         MakeNode(Dst, U, *I, state->BindExpr(U, X));
@@ -2843,38 +2995,22 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
 
       return;
     }
-
-    case UnaryOperator::OffsetOf: {
-      Expr::EvalResult Res;
-      if (U->Evaluate(Res, getContext()) && Res.Val.isInt()) {
-          const APSInt &IV = Res.Val.getInt();
-          assert(IV.getBitWidth() == getContext().getTypeSize(U->getType()));
-          assert(U->getType()->isIntegerType());
-          assert(IV.isSigned() == U->getType()->isSignedIntegerType());
-          SVal X = ValMgr.makeIntVal(IV);
-          MakeNode(Dst, U, Pred, GetState(Pred)->BindExpr(U, X));
-          return;
-        }
-      // FIXME: Handle the case where __builtin_offsetof is not a constant.
-      Dst.Add(Pred);
-      return;
-    }
       
-    case UnaryOperator::Plus: assert(!asLValue);  // FALL-THROUGH.
-    case UnaryOperator::Extension: {
+    case UO_Plus: assert(!asLValue);  // FALL-THROUGH.
+    case UO_Extension: {
 
       // Unary "+" is a no-op, similar to a parentheses.  We still have places
       // where it may be a block-level expression, so we need to
       // generate an extra node that just propagates the value of the
       // subexpression.
 
-      Expr* Ex = U->getSubExpr()->IgnoreParens();
+      const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
 
       if (asLValue)
-	VisitLValue(Ex, Pred, Tmp);
+        VisitLValue(Ex, Pred, Tmp);
       else
-	Visit(Ex, Pred, Tmp);
+        Visit(Ex, Pred, Tmp);
 
       for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end(); I!=E; ++I) {
         const GRState* state = GetState(*I);
@@ -2884,10 +3020,10 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
       return;
     }
 
-    case UnaryOperator::AddrOf: {
+    case UO_AddrOf: {
 
       assert(!asLValue);
-      Expr* Ex = U->getSubExpr()->IgnoreParens();
+      const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       VisitLValue(Ex, Pred, Tmp);
 
@@ -2901,12 +3037,12 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
       return;
     }
 
-    case UnaryOperator::LNot:
-    case UnaryOperator::Minus:
-    case UnaryOperator::Not: {
+    case UO_LNot:
+    case UO_Minus:
+    case UO_Not: {
 
       assert (!asLValue);
-      Expr* Ex = U->getSubExpr()->IgnoreParens();
+      const Expr* Ex = U->getSubExpr()->IgnoreParens();
       ExplodedNodeSet Tmp;
       Visit(Ex, Pred, Tmp);
 
@@ -2937,17 +3073,17 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
             assert(false && "Invalid Opcode.");
             break;
 
-          case UnaryOperator::Not:
+          case UO_Not:
             // FIXME: Do we need to handle promotions?
             state = state->BindExpr(U, EvalComplement(cast<NonLoc>(V)));
             break;
 
-          case UnaryOperator::Minus:
+          case UO_Minus:
             // FIXME: Do we need to handle promotions?
             state = state->BindExpr(U, EvalMinus(cast<NonLoc>(V)));
             break;
 
-          case UnaryOperator::LNot:
+          case UO_LNot:
 
             // C99 6.5.3.3: "The expression !E is equivalent to (0==E)."
             //
@@ -2957,12 +3093,12 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
 
             if (isa<Loc>(V)) {
               Loc X = ValMgr.makeNull();
-              Result = EvalBinOp(state, BinaryOperator::EQ, cast<Loc>(V), X,
+              Result = EvalBinOp(state, BO_EQ, cast<Loc>(V), X,
                                  U->getType());
             }
             else {
               nonloc::ConcreteInt X(getBasicVals().getValue(0, Ex->getType()));
-              Result = EvalBinOp(state, BinaryOperator::EQ, cast<NonLoc>(V), X,
+              Result = EvalBinOp(state, BO_EQ, cast<NonLoc>(V), X,
                                  U->getType());
             }
 
@@ -2982,7 +3118,7 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
 
   assert (U->isIncrementDecrementOp());
   ExplodedNodeSet Tmp;
-  Expr* Ex = U->getSubExpr()->IgnoreParens();
+  const Expr* Ex = U->getSubExpr()->IgnoreParens();
   VisitLValue(Ex, Pred, Tmp);
 
   for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I!=E; ++I) {
@@ -3007,8 +3143,8 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
       DefinedSVal V2 = cast<DefinedSVal>(V2_untested);
 
       // Handle all other values.
-      BinaryOperator::Opcode Op = U->isIncrementOp() ? BinaryOperator::Add
-                                                     : BinaryOperator::Sub;
+      BinaryOperator::Opcode Op = U->isIncrementOp() ? BO_Add
+                                                     : BO_Sub;
 
       // If the UnaryOperator has non-location type, use its type to create the
       // constant value. If the UnaryOperator has location type, create the
@@ -3057,14 +3193,14 @@ void GRExprEngine::VisitUnaryOperator(UnaryOperator* U, ExplodedNode* Pred,
   }
 }
 
-void GRExprEngine::VisitAsmStmt(AsmStmt* A, ExplodedNode* Pred,
+void GRExprEngine::VisitAsmStmt(const AsmStmt* A, ExplodedNode* Pred,
                                 ExplodedNodeSet& Dst) {
   VisitAsmStmtHelperOutputs(A, A->begin_outputs(), A->end_outputs(), Pred, Dst);
 }
 
-void GRExprEngine::VisitAsmStmtHelperOutputs(AsmStmt* A,
-                                             AsmStmt::outputs_iterator I,
-                                             AsmStmt::outputs_iterator E,
+void GRExprEngine::VisitAsmStmtHelperOutputs(const AsmStmt* A,
+                                             AsmStmt::const_outputs_iterator I,
+                                             AsmStmt::const_outputs_iterator E,
                                      ExplodedNode* Pred, ExplodedNodeSet& Dst) {
   if (I == E) {
     VisitAsmStmtHelperInputs(A, A->begin_inputs(), A->end_inputs(), Pred, Dst);
@@ -3080,9 +3216,9 @@ void GRExprEngine::VisitAsmStmtHelperOutputs(AsmStmt* A,
     VisitAsmStmtHelperOutputs(A, I, E, *NI, Dst);
 }
 
-void GRExprEngine::VisitAsmStmtHelperInputs(AsmStmt* A,
-                                            AsmStmt::inputs_iterator I,
-                                            AsmStmt::inputs_iterator E,
+void GRExprEngine::VisitAsmStmtHelperInputs(const AsmStmt* A,
+                                            AsmStmt::const_inputs_iterator I,
+                                            AsmStmt::const_inputs_iterator E,
                                             ExplodedNode* Pred,
                                             ExplodedNodeSet& Dst) {
   if (I == E) {
@@ -3096,7 +3232,7 @@ void GRExprEngine::VisitAsmStmtHelperInputs(AsmStmt* A,
 
     const GRState* state = GetState(Pred);
 
-    for (AsmStmt::outputs_iterator OI = A->begin_outputs(),
+    for (AsmStmt::const_outputs_iterator OI = A->begin_outputs(),
                                    OE = A->end_outputs(); OI != OE; ++OI) {
 
       SVal X = state->getSVal(*OI);
@@ -3119,10 +3255,10 @@ void GRExprEngine::VisitAsmStmtHelperInputs(AsmStmt* A,
     VisitAsmStmtHelperInputs(A, I, E, *NI, Dst);
 }
 
-void GRExprEngine::VisitReturnStmt(ReturnStmt *RS, ExplodedNode *Pred,
+void GRExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
                                    ExplodedNodeSet &Dst) {
   ExplodedNodeSet Src;
-  if (Expr *RetE = RS->getRetValue()) {
+  if (const Expr *RetE = RS->getRetValue()) {
     // Record the returned expression in the state. It will be used in
     // ProcessCallExit to bind the return value to the call expr.
     {
@@ -3141,7 +3277,7 @@ void GRExprEngine::VisitReturnStmt(ReturnStmt *RS, ExplodedNode *Pred,
   }
 
   ExplodedNodeSet CheckedSet;
-  CheckerVisit(RS, CheckedSet, Src, true);
+  CheckerVisit(RS, CheckedSet, Src, PreVisitStmtCallback);
 
   for (ExplodedNodeSet::iterator I = CheckedSet.begin(), E = CheckedSet.end();
        I != E; ++I) {
@@ -3167,7 +3303,7 @@ void GRExprEngine::VisitReturnStmt(ReturnStmt *RS, ExplodedNode *Pred,
 // Transfer functions: Binary operators.
 //===----------------------------------------------------------------------===//
 
-void GRExprEngine::VisitBinaryOperator(BinaryOperator* B,
+void GRExprEngine::VisitBinaryOperator(const BinaryOperator* B,
                                        ExplodedNode* Pred,
                                        ExplodedNodeSet& Dst, bool asLValue) {
 
@@ -3194,7 +3330,7 @@ void GRExprEngine::VisitBinaryOperator(BinaryOperator* B,
     Visit(RHS, *I1, Tmp2);
 
     ExplodedNodeSet CheckedSet;
-    CheckerVisit(B, CheckedSet, Tmp2, true);
+    CheckerVisit(B, CheckedSet, Tmp2, PreVisitStmtCallback);
 
     // With both the LHS and RHS evaluated, process the operation itself.
 
@@ -3207,13 +3343,13 @@ void GRExprEngine::VisitBinaryOperator(BinaryOperator* B,
 
       BinaryOperator::Opcode Op = B->getOpcode();
 
-      if (Op == BinaryOperator::Assign) {
+      if (Op == BO_Assign) {
         // EXPERIMENTAL: "Conjured" symbols.
         // FIXME: Handle structs.
         QualType T = RHS->getType();
 
-        if ((RightV.isUnknown()||!getConstraintManager().canReasonAbout(RightV))
-            && (Loc::IsLocType(T) || (T->isScalarType()&&T->isIntegerType()))) {
+        if (RightV.isUnknown() ||!getConstraintManager().canReasonAbout(RightV))
+        {
           unsigned Count = Builder->getCurrentBlockCount();
           RightV = ValMgr.getConjuredSymbolVal(NULL, B->getRHS(), Count);
         }
@@ -3253,16 +3389,16 @@ void GRExprEngine::VisitBinaryOperator(BinaryOperator* B,
       switch (Op) {
         default:
           assert(0 && "Invalid opcode for compound assignment.");
-        case BinaryOperator::MulAssign: Op = BinaryOperator::Mul; break;
-        case BinaryOperator::DivAssign: Op = BinaryOperator::Div; break;
-        case BinaryOperator::RemAssign: Op = BinaryOperator::Rem; break;
-        case BinaryOperator::AddAssign: Op = BinaryOperator::Add; break;
-        case BinaryOperator::SubAssign: Op = BinaryOperator::Sub; break;
-        case BinaryOperator::ShlAssign: Op = BinaryOperator::Shl; break;
-        case BinaryOperator::ShrAssign: Op = BinaryOperator::Shr; break;
-        case BinaryOperator::AndAssign: Op = BinaryOperator::And; break;
-        case BinaryOperator::XorAssign: Op = BinaryOperator::Xor; break;
-        case BinaryOperator::OrAssign:  Op = BinaryOperator::Or;  break;
+        case BO_MulAssign: Op = BO_Mul; break;
+        case BO_DivAssign: Op = BO_Div; break;
+        case BO_RemAssign: Op = BO_Rem; break;
+        case BO_AddAssign: Op = BO_Add; break;
+        case BO_SubAssign: Op = BO_Sub; break;
+        case BO_ShlAssign: Op = BO_Shl; break;
+        case BO_ShrAssign: Op = BO_Shr; break;
+        case BO_AndAssign: Op = BO_And; break;
+        case BO_XorAssign: Op = BO_Xor; break;
+        case BO_OrAssign:  Op = BO_Or;  break;
       }
 
       // Perform a load (the LHS).  This performs the checks for
@@ -3300,10 +3436,8 @@ void GRExprEngine::VisitBinaryOperator(BinaryOperator* B,
 
         SVal LHSVal;
 
-        if ((Result.isUnknown() ||
-             !getConstraintManager().canReasonAbout(Result))
-            && (Loc::IsLocType(CTy)
-                || (CTy->isScalarType() && CTy->isIntegerType()))) {
+        if (Result.isUnknown() ||
+            !getConstraintManager().canReasonAbout(Result)) {
 
           unsigned Count = Builder->getCurrentBlockCount();
 
@@ -3327,7 +3461,7 @@ void GRExprEngine::VisitBinaryOperator(BinaryOperator* B,
     }
   }
 
-  CheckerVisit(B, Dst, Tmp3, false);
+  CheckerVisit(B, Dst, Tmp3, PostVisitStmtCallback);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3457,7 +3591,7 @@ struct DOTGraphTraits<ExplodedNode*> :
         Out << "Edge: (B" << E.getSrc()->getBlockID() << ", B"
             << E.getDst()->getBlockID()  << ')';
 
-        if (Stmt* T = E.getSrc()->getTerminator()) {
+        if (const Stmt* T = E.getSrc()->getTerminator()) {
 
           SourceLocation SLoc = T->getLocStart();
 
@@ -3473,15 +3607,15 @@ struct DOTGraphTraits<ExplodedNode*> :
           }
 
           if (isa<SwitchStmt>(T)) {
-            Stmt* Label = E.getDst()->getLabel();
+            const Stmt* Label = E.getDst()->getLabel();
 
             if (Label) {
-              if (CaseStmt* C = dyn_cast<CaseStmt>(Label)) {
+              if (const CaseStmt* C = dyn_cast<CaseStmt>(Label)) {
                 Out << "\\lcase ";
                 LangOptions LO; // FIXME.
                 C->getLHS()->printPretty(Out, 0, PrintingPolicy(LO));
 
-                if (Stmt* RHS = C->getRHS()) {
+                if (const Stmt* RHS = C->getRHS()) {
                   Out << " .. ";
                   RHS->printPretty(Out, 0, PrintingPolicy(LO));
                 }
diff --git a/lib/Checker/GRExprEngineExperimentalChecks.cpp b/lib/Checker/GRExprEngineExperimentalChecks.cpp
index d138e81c4691..84262b004341 100644
--- a/lib/Checker/GRExprEngineExperimentalChecks.cpp
+++ b/lib/Checker/GRExprEngineExperimentalChecks.cpp
@@ -18,13 +18,14 @@
 
 using namespace clang;
 
-void clang::RegisterExperimentalChecks(GRExprEngine &Eng) {  
+void clang::RegisterExperimentalChecks(GRExprEngine &Eng) {
   // These are checks that never belong as internal checks
   // within GRExprEngine.
-  RegisterPthreadLockChecker(Eng);  
+  RegisterCStringChecker(Eng);
   RegisterMallocChecker(Eng);
+  RegisterPthreadLockChecker(Eng);
   RegisterStreamChecker(Eng);
-  RegisterCStringChecker(Eng);
+  RegisterUnreachableCodeChecker(Eng);
 }
 
 void clang::RegisterExperimentalInternalChecks(GRExprEngine &Eng) {
@@ -34,11 +35,10 @@ void clang::RegisterExperimentalInternalChecks(GRExprEngine &Eng) {
   // Note that this must be registered after ReturnStackAddresEngsChecker.
   RegisterReturnPointerRangeChecker(Eng);
   
+  RegisterArrayBoundChecker(Eng);
+  RegisterCastSizeChecker(Eng);
+  RegisterCastToStructChecker(Eng);
   RegisterFixedAddressChecker(Eng);
-  RegisterPointerSubChecker(Eng);
   RegisterPointerArithChecker(Eng);
-  RegisterCastToStructChecker(Eng);
-  RegisterCastSizeChecker(Eng);
-  RegisterArrayBoundChecker(Eng);
-
+  RegisterPointerSubChecker(Eng);
 }
diff --git a/lib/Checker/GRExprEngineExperimentalChecks.h b/lib/Checker/GRExprEngineExperimentalChecks.h
index 7d1eb77f9fd6..7b5b0ed7ba00 100644
--- a/lib/Checker/GRExprEngineExperimentalChecks.h
+++ b/lib/Checker/GRExprEngineExperimentalChecks.h
@@ -20,10 +20,11 @@ namespace clang {
 class GRExprEngine;
 
 void RegisterCStringChecker(GRExprEngine &Eng);
-void RegisterPthreadLockChecker(GRExprEngine &Eng);
+void RegisterIdempotentOperationChecker(GRExprEngine &Eng);
 void RegisterMallocChecker(GRExprEngine &Eng);
+void RegisterPthreadLockChecker(GRExprEngine &Eng);
 void RegisterStreamChecker(GRExprEngine &Eng);
-void RegisterIdempotentOperationChecker(GRExprEngine &Eng);
+void RegisterUnreachableCodeChecker(GRExprEngine &Eng);
 
 } // end clang namespace
 #endif
diff --git a/lib/Checker/GRState.cpp b/lib/Checker/GRState.cpp
index 9e584b56148c..d38ae21fce94 100644
--- a/lib/Checker/GRState.cpp
+++ b/lib/Checker/GRState.cpp
@@ -14,6 +14,7 @@
 #include "clang/Analysis/CFG.h"
 #include "clang/Checker/PathSensitive/GRStateTrait.h"
 #include "clang/Checker/PathSensitive/GRState.h"
+#include "clang/Checker/PathSensitive/GRSubEngine.h"
 #include "clang/Checker/PathSensitive/GRTransferFuncs.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -51,22 +52,105 @@ GRStateManager::RemoveDeadBindings(const GRState* state,
                                            state, RegionRoots);
 
   // Clean up the store.
-  const GRState *s = StoreMgr->RemoveDeadBindings(NewState, LCtx, 
-                                                  SymReaper, RegionRoots);
+  NewState.St = StoreMgr->RemoveDeadBindings(NewState.St, LCtx, 
+                                             SymReaper, RegionRoots);
+  state = getPersistentState(NewState);
+  return ConstraintMgr->RemoveDeadBindings(state, SymReaper);
+}
+
+const GRState *GRStateManager::MarshalState(const GRState *state,
+                                            const StackFrameContext *InitLoc) {
+  // make up an empty state for now.
+  GRState State(this,
+                EnvMgr.getInitialEnvironment(),
+                StoreMgr->getInitialStore(InitLoc),
+                GDMFactory.GetEmptyMap());
+
+  return getPersistentState(State);
+}
+
+const GRState *GRState::bindCompoundLiteral(const CompoundLiteralExpr* CL,
+                                            const LocationContext *LC,
+                                            SVal V) const {
+  Store new_store = 
+    getStateManager().StoreMgr->BindCompoundLiteral(St, CL, LC, V);
+  return makeWithStore(new_store);
+}
+
+const GRState *GRState::bindDecl(const VarRegion* VR, SVal IVal) const {
+  Store new_store = getStateManager().StoreMgr->BindDecl(St, VR, IVal);
+  return makeWithStore(new_store);
+}
+
+const GRState *GRState::bindDeclWithNoInit(const VarRegion* VR) const {
+  Store new_store = getStateManager().StoreMgr->BindDeclWithNoInit(St, VR);
+  return makeWithStore(new_store);
+}
+
+const GRState *GRState::bindLoc(Loc LV, SVal V) const {
+  GRStateManager &Mgr = getStateManager();
+  Store new_store = Mgr.StoreMgr->Bind(St, LV, V);
+  const GRState *new_state = makeWithStore(new_store);
 
-  return ConstraintMgr->RemoveDeadBindings(s, SymReaper);
+  const MemRegion *MR = LV.getAsRegion();
+  if (MR)
+    return Mgr.getOwningEngine().ProcessRegionChange(new_state, MR);
+
+  return new_state;
+}
+
+const GRState *GRState::bindDefault(SVal loc, SVal V) const {
+  GRStateManager &Mgr = getStateManager();
+  const MemRegion *R = cast<loc::MemRegionVal>(loc).getRegion();
+  Store new_store = Mgr.StoreMgr->BindDefault(St, R, V);
+  const GRState *new_state = makeWithStore(new_store);
+  return Mgr.getOwningEngine().ProcessRegionChange(new_state, R);
+}
+
+const GRState *GRState::InvalidateRegions(const MemRegion * const *Begin,
+                                          const MemRegion * const *End,
+                                          const Expr *E, unsigned Count,
+                                          StoreManager::InvalidatedSymbols *IS,
+                                          bool invalidateGlobals) const {
+  GRStateManager &Mgr = getStateManager();
+  GRSubEngine &Eng = Mgr.getOwningEngine();
+
+  if (Eng.WantsRegionChangeUpdate(this)) {
+    StoreManager::InvalidatedRegions Regions;
+
+    Store new_store = Mgr.StoreMgr->InvalidateRegions(St, Begin, End,
+                                                      E, Count, IS,
+                                                      invalidateGlobals,
+                                                      &Regions);
+    const GRState *new_state = makeWithStore(new_store);
+
+    return Eng.ProcessRegionChanges(new_state,
+                                    &Regions.front(),
+                                    &Regions.back()+1);
+  }
+
+  Store new_store = Mgr.StoreMgr->InvalidateRegions(St, Begin, End,
+                                                    E, Count, IS,
+                                                    invalidateGlobals,
+                                                    NULL);
+  return makeWithStore(new_store);
 }
 
 const GRState *GRState::unbindLoc(Loc LV) const {
+  assert(!isa<loc::MemRegionVal>(LV) && "Use InvalidateRegion instead.");
+
   Store OldStore = getStore();
   Store NewStore = getStateManager().StoreMgr->Remove(OldStore, LV);
 
   if (NewStore == OldStore)
     return this;
 
-  GRState NewSt = *this;
-  NewSt.St = NewStore;
-  return getStateManager().getPersistentState(NewSt);
+  return makeWithStore(NewStore);
+}
+
+const GRState *GRState::EnterStackFrame(const StackFrameContext *frame) const {
+  Store new_store = getStateManager().StoreMgr->EnterStackFrame(this, frame);
+  return makeWithStore(new_store);
 }
 
 SVal GRState::getSValAsScalarOrLoc(const MemRegion *R) const {
@@ -77,7 +161,7 @@ SVal GRState::getSValAsScalarOrLoc(const MemRegion *R) const {
     return UnknownVal();
 
   if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
-    QualType T = TR->getValueType(getStateManager().getContext());
+    QualType T = TR->getValueType();
     if (Loc::IsLocType(T) || T->isIntegerType())
       return getSVal(R);
   }
@@ -85,9 +169,45 @@ SVal GRState::getSValAsScalarOrLoc(const MemRegion *R) const {
   return UnknownVal();
 }
 
+SVal GRState::getSimplifiedSVal(Loc location, QualType T) const {
+  SVal V = getSVal(cast<Loc>(location), T);
+  
+  // If 'V' is a symbolic value that is *perfectly* constrained to
+  // be a constant value, use that value instead to lessen the burden
+  // on later analysis stages (so we have less symbolic values to reason
+  // about).
+  if (!T.isNull()) {
+    if (SymbolRef sym = V.getAsSymbol()) {
+      if (const llvm::APSInt *Int = getSymVal(sym)) {
+        // FIXME: Because we don't correctly model (yet) sign-extension
+        // and truncation of symbolic values, we need to convert
+        // the integer value to the correct signedness and bitwidth.
+        //
+        // This shows up in the following:
+        //
+        //   char foo();
+        //   unsigned x = foo();
+        //   if (x == 54)
+        //     ...
+        //
+        //  The symbolic value stored to 'x' is actually the conjured
+        //  symbol for the call to foo(); the type of that symbol is 'char',
+        //  not unsigned.
+        const llvm::APSInt &NewV = getBasicVals().Convert(T, *Int);
+        
+        if (isa<Loc>(V))
+          return loc::ConcreteInt(NewV);
+        else
+          return nonloc::ConcreteInt(NewV);
+      }
+    }
+  }
+  
+  return V;
+}
 
-const GRState *GRState::BindExpr(const Stmt* Ex, SVal V, bool Invalidate) const{
-  Environment NewEnv = getStateManager().EnvMgr.BindExpr(Env, Ex, V,
+const GRState *GRState::BindExpr(const Stmt* S, SVal V, bool Invalidate) const{
+  Environment NewEnv = getStateManager().EnvMgr.bindExpr(Env, S, V,
                                                          Invalidate);
   if (NewEnv == Env)
     return this;
@@ -97,6 +217,63 @@ const GRState *GRState::BindExpr(const Stmt* Ex, SVal V, bool Invalidate) const{
   return getStateManager().getPersistentState(NewSt);
 }
 
+const GRState *GRState::bindExprAndLocation(const Stmt *S, SVal location,
+                                            SVal V) const {
+  Environment NewEnv =
+    getStateManager().EnvMgr.bindExprAndLocation(Env, S, location, V);
+
+  if (NewEnv == Env)
+    return this;
+  
+  GRState NewSt = *this;
+  NewSt.Env = NewEnv;
+  return getStateManager().getPersistentState(NewSt);
+}
+
+const GRState *GRState::AssumeInBound(DefinedOrUnknownSVal Idx,
+                                      DefinedOrUnknownSVal UpperBound,
+                                      bool Assumption) const {
+  if (Idx.isUnknown() || UpperBound.isUnknown())
+    return this;
+
+  // Build an expression for 0 <= Idx < UpperBound.
+  // This is the same as Idx + MIN < UpperBound + MIN, if overflow is allowed.
+  // FIXME: This should probably be part of SValuator.
+  GRStateManager &SM = getStateManager();
+  ValueManager &VM = SM.getValueManager();
+  SValuator &SV = VM.getSValuator();
+  ASTContext &Ctx = VM.getContext();
+
+  // Get the offset: the minimum value of the array index type.
+  BasicValueFactory &BVF = VM.getBasicValueFactory();
+  // FIXME: This should be using ValueManager::ArrayIndexTy...somehow.
+  QualType IndexTy = Ctx.IntTy;
+  nonloc::ConcreteInt Min = BVF.getMinValue(IndexTy);
+
+  // Adjust the index.
+  SVal NewIdx = SV.EvalBinOpNN(this, BO_Add,
+                               cast<NonLoc>(Idx), Min, IndexTy);
+  if (NewIdx.isUnknownOrUndef())
+    return this;
+
+  // Adjust the upper bound.
+  SVal NewBound = SV.EvalBinOpNN(this, BO_Add,
+                                 cast<NonLoc>(UpperBound), Min, IndexTy);
+  if (NewBound.isUnknownOrUndef())
+    return this;
+
+  // Build the actual comparison.
+  SVal InBound = SV.EvalBinOpNN(this, BO_LT,
+                                cast<NonLoc>(NewIdx), cast<NonLoc>(NewBound),
+                                Ctx.IntTy);
+  if (InBound.isUnknownOrUndef())
+    return this;
+
+  // Finally, let the constraint manager take care of it.
+  ConstraintManager &CM = SM.getConstraintManager();
+  return CM.Assume(this, cast<DefinedSVal>(InBound), Assumption);
+}
+
 const GRState* GRStateManager::getInitialState(const LocationContext *InitLoc) {
   GRState State(this,
                 EnvMgr.getInitialEnvironment(),
@@ -131,6 +308,11 @@ const GRState* GRState::makeWithStore(Store store) const {
 //  State pretty-printing.
 //===----------------------------------------------------------------------===//
 
+static bool IsEnvLoc(const Stmt *S) {
+  // FIXME: This is a layering violation.  Should be in environment.
+  return (bool) (((uintptr_t) S) & 0x1);
+}
+
 void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
                     const char* sep) const {
   // Print the store.
@@ -140,8 +322,9 @@ void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
   // Print Subexpression bindings.
   bool isFirst = true;
 
+  // FIXME: All environment printing should be moved inside Environment.
   for (Environment::iterator I = Env.begin(), E = Env.end(); I != E; ++I) {
-    if (C.isBlkExpr(I.getKey()))
+    if (C.isBlkExpr(I.getKey()) || IsEnvLoc(I.getKey()))
       continue;
 
     if (isFirst) {
@@ -174,6 +357,27 @@ void GRState::print(llvm::raw_ostream& Out, CFG &C, const char* nl,
     I.getKey()->printPretty(Out, 0, PrintingPolicy(LO));
     Out << " : " << I.getData();
   }
+  
+  // Print locations.
+  isFirst = true;
+  
+  for (Environment::iterator I = Env.begin(), E = Env.end(); I != E; ++I) {
+    if (!IsEnvLoc(I.getKey()))
+      continue;
+    
+    if (isFirst) {
+      Out << nl << nl << "Load/store locations:" << nl;
+      isFirst = false;
+    }
+    else { Out << nl; }
+
+    const Stmt *S = (Stmt*) (((uintptr_t) I.getKey()) & ((uintptr_t) ~0x1));
+    
+    Out << " (" << (void*) S << ") ";
+    LangOptions LO; // FIXME.
+    S->printPretty(Out, 0, PrintingPolicy(LO));
+    Out << " : " << I.getData();
+  }
 
   Mgr.getConstraintManager().print(this, Out, nl, sep);
 
diff --git a/lib/Checker/IdempotentOperationChecker.cpp b/lib/Checker/IdempotentOperationChecker.cpp
index 6ed18417a2c2..6411c790ef7c 100644
--- a/lib/Checker/IdempotentOperationChecker.cpp
+++ b/lib/Checker/IdempotentOperationChecker.cpp
@@ -36,25 +36,26 @@
 //  !=      |   0    |        |        |        |        |         |
 //===----------------------------------------------------------------------===//
 //
-// Ways to reduce false positives (that need to be implemented):
-// - Don't flag downsizing casts
-// - Improved handling of static/global variables
-// - Per-block marking of incomplete analysis
-// - Handling ~0 values
-// - False positives involving silencing unused variable warnings
-//
-// Other things TODO:
+// Things TODO:
 // - Improved error messages
 // - Handle mixed assumptions (which assumptions can belong together?)
 // - Finer grained false positive control (levels)
+// - Handling ~0 values
 
 #include "GRExprEngineExperimentalChecks.h"
+#include "clang/Analysis/CFGStmtMap.h"
+#include "clang/Analysis/Analyses/PseudoConstantAnalysis.h"
+#include "clang/Checker/BugReporter/BugReporter.h"
 #include "clang/Checker/BugReporter/BugType.h"
+#include "clang/Checker/PathSensitive/CheckerHelpers.h"
 #include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "clang/Checker/PathSensitive/GRCoreEngine.h"
 #include "clang/Checker/PathSensitive/SVals.h"
 #include "clang/AST/Stmt.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/ErrorHandling.h"
+#include <deque>
 
 using namespace clang;
 
@@ -64,38 +65,41 @@ class IdempotentOperationChecker
   public:
     static void *getTag();
     void PreVisitBinaryOperator(CheckerContext &C, const BinaryOperator *B);
-    void VisitEndAnalysis(ExplodedGraph &G, BugReporter &B,
-        bool hasWorkRemaining);
+    void PostVisitBinaryOperator(CheckerContext &C, const BinaryOperator *B);
+    void VisitEndAnalysis(ExplodedGraph &G, BugReporter &B, GRExprEngine &Eng);
 
   private:
     // Our assumption about a particular operation.
-    enum Assumption { Possible, Impossible, Equal, LHSis1, RHSis1, LHSis0,
+    enum Assumption { Possible = 0, Impossible, Equal, LHSis1, RHSis1, LHSis0,
         RHSis0 };
 
     void UpdateAssumption(Assumption &A, const Assumption &New);
 
-    /// contains* - Useful recursive methods to see if a statement contains an
-    ///   element somewhere. Used in static analysis to reduce false positives.
-    static bool containsMacro(const Stmt *S);
-    static bool containsEnum(const Stmt *S);
-    static bool containsBuiltinOffsetOf(const Stmt *S);
-    static bool containsZeroConstant(const Stmt *S);
-    static bool containsOneConstant(const Stmt *S);
-    template <class T> static bool containsStmt(const Stmt *S) {
-      if (isa<T>(S))
-          return true;
-
-      for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
-          ++I)
-        if (const Stmt *child = *I)
-          if (containsStmt<T>(child))
-            return true;
-
-        return false;
-    }
-
-    // Hash table
-    typedef llvm::DenseMap<const BinaryOperator *, Assumption> AssumptionMap;
+    // False positive reduction methods
+    static bool isSelfAssign(const Expr *LHS, const Expr *RHS);
+    static bool isUnused(const Expr *E, AnalysisContext *AC);
+    //static bool isTruncationExtensionAssignment(const Expr *LHS,
+    //                                            const Expr *RHS);
+    static bool PathWasCompletelyAnalyzed(const CFG *C,
+                                          const CFGBlock *CB,
+                                          const GRCoreEngine &CE);
+    static bool CanVary(const Expr *Ex,
+                        AnalysisContext *AC);
+    static bool isConstantOrPseudoConstant(const DeclRefExpr *DR,
+                                           AnalysisContext *AC);
+    static bool containsNonLocalVarDecl(const Stmt *S);
+
+    // Hash table and related data structures
+    struct BinaryOperatorData {
+      BinaryOperatorData() : assumption(Possible), analysisContext(0) {}
+
+      Assumption assumption;
+      AnalysisContext *analysisContext;
+      ExplodedNodeSet explodedNodes; // Set of ExplodedNodes that refer to a
+                                     // BinaryOperator
+    };
+    typedef llvm::DenseMap<const BinaryOperator *, BinaryOperatorData>
+        AssumptionMap;
     AssumptionMap hash;
 };
 }
@@ -112,30 +116,38 @@ void clang::RegisterIdempotentOperationChecker(GRExprEngine &Eng) {
 void IdempotentOperationChecker::PreVisitBinaryOperator(
                                                       CheckerContext &C,
                                                       const BinaryOperator *B) {
-  // Find or create an entry in the hash for this BinaryOperator instance
-  AssumptionMap::iterator i = hash.find(B);
-  Assumption &A = i == hash.end() ? hash[B] : i->second;
-
-  // If we had to create an entry, initialise the value to Possible
-  if (i == hash.end())
-    A = Possible;
+  // Find or create an entry in the hash for this BinaryOperator instance.
+  // If we haven't done a lookup before, it will get default initialized to
+  // 'Possible'. At this stage we do not store the ExplodedNode, as it has not
+  // been created yet.
+  BinaryOperatorData &Data = hash[B];
+  Assumption &A = Data.assumption;
+  AnalysisContext *AC = C.getCurrentAnalysisContext();
+  Data.analysisContext = AC;
 
   // If we already have visited this node on a path that does not contain an
   // idempotent operation, return immediately.
   if (A == Impossible)
     return;
 
-  // Skip binary operators containing common false positives
-  if (containsMacro(B) || containsEnum(B) || containsStmt<SizeOfAlignOfExpr>(B)
-      || containsZeroConstant(B) || containsOneConstant(B)
-      || containsBuiltinOffsetOf(B)) {
-    A = Impossible;
-    return;
-  }
-
+  // Retrieve both sides of the operator and determine if they can vary (which
+  // may mean this is a false positive.
   const Expr *LHS = B->getLHS();
   const Expr *RHS = B->getRHS();
 
+  // At this stage we can calculate whether each side contains a false positive
+  // that applies to all operators. We only need to calculate this the first
+  // time.
+  bool LHSContainsFalsePositive = false, RHSContainsFalsePositive = false;
+  if (A == Possible) {
+    // An expression contains a false positive if it can't vary, or if it
+    // contains a known false positive VarDecl.
+    LHSContainsFalsePositive = !CanVary(LHS, AC)
+        || containsNonLocalVarDecl(LHS);
+    RHSContainsFalsePositive = !CanVary(RHS, AC)
+        || containsNonLocalVarDecl(RHS);
+  }
+
   const GRState *state = C.getState();
 
   SVal LHSVal = state->getSVal(LHS);
@@ -154,16 +166,16 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
     break;
 
   // Fall through intentional
-  case BinaryOperator::AddAssign:
-  case BinaryOperator::SubAssign:
-  case BinaryOperator::MulAssign:
-  case BinaryOperator::DivAssign:
-  case BinaryOperator::AndAssign:
-  case BinaryOperator::OrAssign:
-  case BinaryOperator::XorAssign:
-  case BinaryOperator::ShlAssign:
-  case BinaryOperator::ShrAssign:
-  case BinaryOperator::Assign:
+  case BO_AddAssign:
+  case BO_SubAssign:
+  case BO_MulAssign:
+  case BO_DivAssign:
+  case BO_AndAssign:
+  case BO_OrAssign:
+  case BO_XorAssign:
+  case BO_ShlAssign:
+  case BO_ShrAssign:
+  case BO_Assign:
   // Assign statements have one extra level of indirection
     if (!isa<Loc>(LHSVal)) {
       A = Impossible;
@@ -181,20 +193,37 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
     break; // We don't care about any other operators.
 
   // Fall through intentional
-  case BinaryOperator::SubAssign:
-  case BinaryOperator::DivAssign:
-  case BinaryOperator::AndAssign:
-  case BinaryOperator::OrAssign:
-  case BinaryOperator::XorAssign:
-  case BinaryOperator::Assign:
-  case BinaryOperator::Sub:
-  case BinaryOperator::Div:
-  case BinaryOperator::And:
-  case BinaryOperator::Or:
-  case BinaryOperator::Xor:
-  case BinaryOperator::LOr:
-  case BinaryOperator::LAnd:
-    if (LHSVal != RHSVal)
+  case BO_Assign:
+    // x Assign x can be used to silence unused variable warnings intentionally.
+    // If this is a self assignment and the variable is referenced elsewhere,
+    // then it is a false positive.
+    if (isSelfAssign(LHS, RHS)) {
+      if (!isUnused(LHS, AC)) {
+        UpdateAssumption(A, Equal);
+        return;
+      }
+      else {
+        A = Impossible;
+        return;
+      }
+    }
+
+  case BO_SubAssign:
+  case BO_DivAssign:
+  case BO_AndAssign:
+  case BO_OrAssign:
+  case BO_XorAssign:
+  case BO_Sub:
+  case BO_Div:
+  case BO_And:
+  case BO_Or:
+  case BO_Xor:
+  case BO_LOr:
+  case BO_LAnd:
+  case BO_EQ:
+  case BO_NE:
+    if (LHSVal != RHSVal || LHSContainsFalsePositive
+        || RHSContainsFalsePositive)
       break;
     UpdateAssumption(A, Equal);
     return;
@@ -206,13 +235,13 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
      break; // We don't care about any other operators.
 
    // Fall through intentional
-   case BinaryOperator::MulAssign:
-   case BinaryOperator::DivAssign:
-   case BinaryOperator::Mul:
-   case BinaryOperator::Div:
-   case BinaryOperator::LOr:
-   case BinaryOperator::LAnd:
-     if (!RHSVal.isConstant(1))
+   case BO_MulAssign:
+   case BO_DivAssign:
+   case BO_Mul:
+   case BO_Div:
+   case BO_LOr:
+   case BO_LAnd:
+     if (!RHSVal.isConstant(1) || RHSContainsFalsePositive)
        break;
      UpdateAssumption(A, RHSis1);
      return;
@@ -224,11 +253,11 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
     break; // We don't care about any other operators.
 
   // Fall through intentional
-  case BinaryOperator::MulAssign:
-  case BinaryOperator::Mul:
-  case BinaryOperator::LOr:
-  case BinaryOperator::LAnd:
-    if (!LHSVal.isConstant(1))
+  case BO_MulAssign:
+  case BO_Mul:
+  case BO_LOr:
+  case BO_LAnd:
+    if (!LHSVal.isConstant(1) || LHSContainsFalsePositive)
       break;
     UpdateAssumption(A, LHSis1);
     return;
@@ -240,23 +269,23 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
     break; // We don't care about any other operators.
 
   // Fall through intentional
-  case BinaryOperator::AddAssign:
-  case BinaryOperator::SubAssign:
-  case BinaryOperator::MulAssign:
-  case BinaryOperator::AndAssign:
-  case BinaryOperator::OrAssign:
-  case BinaryOperator::XorAssign:
-  case BinaryOperator::Add:
-  case BinaryOperator::Sub:
-  case BinaryOperator::Mul:
-  case BinaryOperator::And:
-  case BinaryOperator::Or:
-  case BinaryOperator::Xor:
-  case BinaryOperator::Shl:
-  case BinaryOperator::Shr:
-  case BinaryOperator::LOr:
-  case BinaryOperator::LAnd:
-    if (!RHSVal.isConstant(0))
+  case BO_AddAssign:
+  case BO_SubAssign:
+  case BO_MulAssign:
+  case BO_AndAssign:
+  case BO_OrAssign:
+  case BO_XorAssign:
+  case BO_Add:
+  case BO_Sub:
+  case BO_Mul:
+  case BO_And:
+  case BO_Or:
+  case BO_Xor:
+  case BO_Shl:
+  case BO_Shr:
+  case BO_LOr:
+  case BO_LAnd:
+    if (!RHSVal.isConstant(0) || RHSContainsFalsePositive)
       break;
     UpdateAssumption(A, RHSis0);
     return;
@@ -268,27 +297,27 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
     break; // We don't care about any other operators.
 
   // Fall through intentional
-  //case BinaryOperator::AddAssign: // Common false positive
-  case BinaryOperator::SubAssign: // Check only if unsigned
-  case BinaryOperator::MulAssign:
-  case BinaryOperator::DivAssign:
-  case BinaryOperator::AndAssign:
-  //case BinaryOperator::OrAssign: // Common false positive
-  //case BinaryOperator::XorAssign: // Common false positive
-  case BinaryOperator::ShlAssign:
-  case BinaryOperator::ShrAssign:
-  case BinaryOperator::Add:
-  case BinaryOperator::Sub:
-  case BinaryOperator::Mul:
-  case BinaryOperator::Div:
-  case BinaryOperator::And:
-  case BinaryOperator::Or:
-  case BinaryOperator::Xor:
-  case BinaryOperator::Shl:
-  case BinaryOperator::Shr:
-  case BinaryOperator::LOr:
-  case BinaryOperator::LAnd:
-    if (!LHSVal.isConstant(0))
+  //case BO_AddAssign: // Common false positive
+  case BO_SubAssign: // Check only if unsigned
+  case BO_MulAssign:
+  case BO_DivAssign:
+  case BO_AndAssign:
+  //case BO_OrAssign: // Common false positive
+  //case BO_XorAssign: // Common false positive
+  case BO_ShlAssign:
+  case BO_ShrAssign:
+  case BO_Add:
+  case BO_Sub:
+  case BO_Mul:
+  case BO_Div:
+  case BO_And:
+  case BO_Or:
+  case BO_Xor:
+  case BO_Shl:
+  case BO_Shr:
+  case BO_LOr:
+  case BO_LAnd:
+    if (!LHSVal.isConstant(0) || LHSContainsFalsePositive)
       break;
     UpdateAssumption(A, LHSis0);
     return;
@@ -298,47 +327,103 @@ void IdempotentOperationChecker::PreVisitBinaryOperator(
   A = Impossible;
 }
 
-void IdempotentOperationChecker::VisitEndAnalysis(ExplodedGraph &G,
-                                                  BugReporter &B,
-                                                  bool hasWorkRemaining) {
-  // If there is any work remaining we cannot be 100% sure about our warnings
-  if (hasWorkRemaining)
-    return;
+// At the post visit stage, the predecessor ExplodedNode will be the
+// BinaryOperator that was just created. We use this hook to collect the
+// ExplodedNode.
+void IdempotentOperationChecker::PostVisitBinaryOperator(
+                                                      CheckerContext &C,
+                                                      const BinaryOperator *B) {
+  // Add the ExplodedNode we just visited
+  BinaryOperatorData &Data = hash[B];
+  Data.explodedNodes.Add(C.getPredecessor());
+}
 
+void IdempotentOperationChecker::VisitEndAnalysis(ExplodedGraph &G,
+                                                  BugReporter &BR,
+                                                  GRExprEngine &Eng) {
+  BugType *BT = new BugType("Idempotent operation", "Dead code");
   // Iterate over the hash to see if we have any paths with definite
   // idempotent operations.
-  for (AssumptionMap::const_iterator i =
-      hash.begin(); i != hash.end(); ++i) {
-    if (i->second != Impossible) {
-      // Select the error message.
-      const char *msg = 0;
-      switch (i->second) {
-      case Equal:
-        msg = "idempotent operation; both operands are always equal in value";
-        break;
-      case LHSis1:
-        msg = "idempotent operation; the left operand is always 1";
-        break;
-      case RHSis1:
-        msg = "idempotent operation; the right operand is always 1";
-        break;
-      case LHSis0:
-        msg = "idempotent operation; the left operand is always 0";
-        break;
-      case RHSis0:
-        msg = "idempotent operation; the right operand is always 0";
-        break;
-      case Possible:
-        llvm_unreachable("Operation was never marked with an assumption");
-      case Impossible:
-        llvm_unreachable(0);
+  for (AssumptionMap::const_iterator i = hash.begin(); i != hash.end(); ++i) {
+    // Unpack the hash contents
+    const BinaryOperatorData &Data = i->second;
+    const Assumption &A = Data.assumption;
+    AnalysisContext *AC = Data.analysisContext;
+    const ExplodedNodeSet &ES = Data.explodedNodes;
+
+    const BinaryOperator *B = i->first;
+
+    if (A == Impossible)
+      continue;
+
+    // If the analyzer did not finish, check to see if we can still emit this
+    // warning
+    if (Eng.hasWorkRemaining()) {
+      const CFGStmtMap *CBM = CFGStmtMap::Build(AC->getCFG(),
+                                                &AC->getParentMap());
+
+      // If we can trace back
+      if (!PathWasCompletelyAnalyzed(AC->getCFG(),
+                                     CBM->getBlock(B),
+                                     Eng.getCoreEngine()))
+        continue;
+
+      delete CBM;
+    }
+
+    // Select the error message and SourceRanges to report.
+    llvm::SmallString<128> buf;
+    llvm::raw_svector_ostream os(buf);
+    bool LHSRelevant = false, RHSRelevant = false;
+    switch (A) {
+    case Equal:
+      LHSRelevant = true;
+      RHSRelevant = true;
+      if (B->getOpcode() == BO_Assign)
+        os << "Assigned value is always the same as the existing value";
+      else
+        os << "Both operands to '" << B->getOpcodeStr()
+           << "' always have the same value";
+      break;
+    case LHSis1:
+      LHSRelevant = true;
+      os << "The left operand to '" << B->getOpcodeStr() << "' is always 1";
+      break;
+    case RHSis1:
+      RHSRelevant = true;
+      os << "The right operand to '" << B->getOpcodeStr() << "' is always 1";
+      break;
+    case LHSis0:
+      LHSRelevant = true;
+      os << "The left operand to '" << B->getOpcodeStr() << "' is always 0";
+      break;
+    case RHSis0:
+      RHSRelevant = true;
+      os << "The right operand to '" << B->getOpcodeStr() << "' is always 0";
+      break;
+    case Possible:
+      llvm_unreachable("Operation was never marked with an assumption");
+    case Impossible:
+      llvm_unreachable(0);
+    }
+
+    // Add a report for each ExplodedNode
+    for (ExplodedNodeSet::iterator I = ES.begin(), E = ES.end(); I != E; ++I) {
+      EnhancedBugReport *report = new EnhancedBugReport(*BT, os.str(), *I);
+
+      // Add source ranges and visitor hooks
+      if (LHSRelevant) {
+        const Expr *LHS = i->first->getLHS();
+        report->addRange(LHS->getSourceRange());
+        report->addVisitorCreator(bugreporter::registerVarDeclsLastStore, LHS);
+      }
+      if (RHSRelevant) {
+        const Expr *RHS = i->first->getRHS();
+        report->addRange(i->first->getRHS()->getSourceRange());
+        report->addVisitorCreator(bugreporter::registerVarDeclsLastStore, RHS);
       }
 
-      // Create the SourceRange Arrays
-      SourceRange S[2] = { i->first->getLHS()->getSourceRange(),
-                           i->first->getRHS()->getSourceRange() };
-      B.EmitBasicReport("Idempotent operation", msg, i->first->getOperatorLoc(),
-          S, 2);
+      BR.EmitReport(report);
     }
   }
 }
@@ -346,6 +431,10 @@ void IdempotentOperationChecker::VisitEndAnalysis(ExplodedGraph &G,
 // Updates the current assumption given the new assumption
 inline void IdempotentOperationChecker::UpdateAssumption(Assumption &A,
                                                         const Assumption &New) {
+// If the assumption is the same, there is nothing to do
+  if (A == New)
+    return;
+
   switch (A) {
   // If we don't currently have an assumption, set it
   case Possible:
@@ -366,89 +455,249 @@ inline void IdempotentOperationChecker::UpdateAssumption(Assumption &A,
   }
 }
 
-// Recursively find any substatements containing macros
-bool IdempotentOperationChecker::containsMacro(const Stmt *S) {
-  if (S->getLocStart().isMacroID())
-    return true;
+// Check for a statement where a variable is self assigned to possibly avoid an
+// unused variable warning.
+bool IdempotentOperationChecker::isSelfAssign(const Expr *LHS, const Expr *RHS) {
+  LHS = LHS->IgnoreParenCasts();
+  RHS = RHS->IgnoreParenCasts();
 
-  if (S->getLocEnd().isMacroID())
-    return true;
+  const DeclRefExpr *LHS_DR = dyn_cast<DeclRefExpr>(LHS);
+  if (!LHS_DR)
+    return false;
 
-  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
-      ++I)
-    if (const Stmt *child = *I)
-      if (containsMacro(child))
-        return true;
+  const VarDecl *VD = dyn_cast<VarDecl>(LHS_DR->getDecl());
+  if (!VD)
+    return false;
 
-  return false;
+  const DeclRefExpr *RHS_DR = dyn_cast<DeclRefExpr>(RHS);
+  if (!RHS_DR)
+    return false;
+
+  if (VD != RHS_DR->getDecl())
+    return false;
+
+  return true;
 }
 
-// Recursively find any substatements containing enum constants
-bool IdempotentOperationChecker::containsEnum(const Stmt *S) {
-  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S);
+// Returns true if the Expr points to a VarDecl that is not read anywhere
+// outside of self-assignments.
+bool IdempotentOperationChecker::isUnused(const Expr *E,
+                                          AnalysisContext *AC) {
+  if (!E)
+    return false;
 
-  if (DR && isa<EnumConstantDecl>(DR->getDecl()))
-    return true;
+  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E->IgnoreParenCasts());
+  if (!DR)
+    return false;
 
-  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
-      ++I)
-    if (const Stmt *child = *I)
-      if (containsEnum(child))
-        return true;
+  const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
+  if (!VD)
+    return false;
 
-  return false;
+  if (AC->getPseudoConstantAnalysis()->wasReferenced(VD))
+    return false;
+
+  return true;
 }
 
-// Recursively find any substatements containing __builtin_offset_of
-bool IdempotentOperationChecker::containsBuiltinOffsetOf(const Stmt *S) {
-  const UnaryOperator *UO = dyn_cast<UnaryOperator>(S);
+#if 0
+// Check for self casts truncating/extending a variable
+bool IdempotentOperationChecker::isTruncationExtensionAssignment(
+                                                              const Expr *LHS,
+                                                              const Expr *RHS) {
 
-  if (UO && UO->getOpcode() == UnaryOperator::OffsetOf)
-    return true;
+  const DeclRefExpr *LHS_DR = dyn_cast<DeclRefExpr>(LHS->IgnoreParenCasts());
+  if (!LHS_DR)
+    return false;
 
-  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
-      ++I)
-    if (const Stmt *child = *I)
-      if (containsBuiltinOffsetOf(child))
-        return true;
+  const VarDecl *VD = dyn_cast<VarDecl>(LHS_DR->getDecl());
+  if (!VD)
+    return false;
 
-  return false;
+  const DeclRefExpr *RHS_DR = dyn_cast<DeclRefExpr>(RHS->IgnoreParenCasts());
+  if (!RHS_DR)
+    return false;
+
+  if (VD != RHS_DR->getDecl())
+     return false;
+
+  return dyn_cast<DeclRefExpr>(RHS->IgnoreParens()) == NULL;
 }
+#endif
+
+// Returns false if a path to this block was not completely analyzed, or true
+// otherwise.
+bool IdempotentOperationChecker::PathWasCompletelyAnalyzed(
+                                                       const CFG *C,
+                                                       const CFGBlock *CB,
+                                                       const GRCoreEngine &CE) {
+  std::deque<const CFGBlock *> WorkList;
+  llvm::SmallSet<unsigned, 8> Aborted;
+  llvm::SmallSet<unsigned, 128> Visited;
+
+  // Create a set of all aborted blocks
+  typedef GRCoreEngine::BlocksAborted::const_iterator AbortedIterator;
+  for (AbortedIterator I = CE.blocks_aborted_begin(),
+      E = CE.blocks_aborted_end(); I != E; ++I) {
+    const BlockEdge &BE =  I->first;
+
+    // The destination block on the BlockEdge is the first block that was not
+    // analyzed.
+    Aborted.insert(BE.getDst()->getBlockID());
+  }
 
-bool IdempotentOperationChecker::containsZeroConstant(const Stmt *S) {
-  const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(S);
-  if (IL && IL->getValue() == 0)
+  // Save the entry block ID for early exiting
+  unsigned EntryBlockID = C->getEntry().getBlockID();
+
+  // Create initial node
+  WorkList.push_back(CB);
+
+  while (!WorkList.empty()) {
+    const CFGBlock *Head = WorkList.front();
+    WorkList.pop_front();
+    Visited.insert(Head->getBlockID());
+
+    // If we found the entry block, then there exists a path from the target
+    // node to the entry point of this function -> the path was completely
+    // analyzed.
+    if (Head->getBlockID() == EntryBlockID)
+      return true;
+
+    // If any of the aborted blocks are on the path to the beginning, then all
+    // paths to this block were not analyzed.
+    if (Aborted.count(Head->getBlockID()))
+      return false;
+
+    // Add the predecessors to the worklist unless we have already visited them
+    for (CFGBlock::const_pred_iterator I = Head->pred_begin();
+        I != Head->pred_end(); ++I)
+      if (!Visited.count((*I)->getBlockID()))
+        WorkList.push_back(*I);
+  }
+
+  // If we get to this point, there is no connection to the entry block or an
+  // aborted block. This path is unreachable and we can report the error.
+  return true;
+}
+
+// Recursive function that determines whether an expression contains any element
+// that varies. This could be due to a compile-time constant like sizeof. An
+// expression may also involve a variable that behaves like a constant. The
+// function returns true if the expression varies, and false otherwise.
+bool IdempotentOperationChecker::CanVary(const Expr *Ex,
+                                         AnalysisContext *AC) {
+  // Parentheses and casts are irrelevant here
+  Ex = Ex->IgnoreParenCasts();
+
+  if (Ex->getLocStart().isMacroID())
+    return false;
+
+  switch (Ex->getStmtClass()) {
+  // Trivially true cases
+  case Stmt::ArraySubscriptExprClass:
+  case Stmt::MemberExprClass:
+  case Stmt::StmtExprClass:
+  case Stmt::CallExprClass:
+  case Stmt::VAArgExprClass:
+  case Stmt::ShuffleVectorExprClass:
+    return true;
+  default:
     return true;
 
-  const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(S);
-  if (FL && FL->getValue().isZero())
+  // Trivially false cases
+  case Stmt::IntegerLiteralClass:
+  case Stmt::CharacterLiteralClass:
+  case Stmt::FloatingLiteralClass:
+  case Stmt::PredefinedExprClass:
+  case Stmt::ImaginaryLiteralClass:
+  case Stmt::StringLiteralClass:
+  case Stmt::OffsetOfExprClass:
+  case Stmt::CompoundLiteralExprClass:
+  case Stmt::AddrLabelExprClass:
+  case Stmt::TypesCompatibleExprClass:
+  case Stmt::GNUNullExprClass:
+  case Stmt::InitListExprClass:
+  case Stmt::DesignatedInitExprClass:
+  case Stmt::BlockExprClass:
+  case Stmt::BlockDeclRefExprClass:
+    return false;
+
+  // Cases requiring custom logic
+  case Stmt::SizeOfAlignOfExprClass: {
+    const SizeOfAlignOfExpr *SE = cast<const SizeOfAlignOfExpr>(Ex);
+    if (!SE->isSizeOf())
+      return false;
+    return SE->getTypeOfArgument()->isVariableArrayType();
+  }
+  case Stmt::DeclRefExprClass:
+    // Check for constants/pseudoconstants
+    return !isConstantOrPseudoConstant(cast<DeclRefExpr>(Ex), AC);
+
+  // The next cases require recursion for subexpressions
+  case Stmt::BinaryOperatorClass: {
+    const BinaryOperator *B = cast<const BinaryOperator>(Ex);
+    return CanVary(B->getRHS(), AC)
+        || CanVary(B->getLHS(), AC);
+   }
+  case Stmt::UnaryOperatorClass: {
+    const UnaryOperator *U = cast<const UnaryOperator>(Ex);
+    // Handle trivial case first
+    switch (U->getOpcode()) {
+    case UO_Extension:
+      return false;
+    default:
+      return CanVary(U->getSubExpr(), AC);
+    }
+  }
+  case Stmt::ChooseExprClass:
+    return CanVary(cast<const ChooseExpr>(Ex)->getChosenSubExpr(
+        AC->getASTContext()), AC);
+  case Stmt::ConditionalOperatorClass:
+    return CanVary(cast<const ConditionalOperator>(Ex)->getCond(), AC);
+  }
+}
+
+// Returns true if a DeclRefExpr is or behaves like a constant.
+bool IdempotentOperationChecker::isConstantOrPseudoConstant(
+                                                          const DeclRefExpr *DR,
+                                                          AnalysisContext *AC) {
+  // Check if the type of the Decl is const-qualified
+  if (DR->getType().isConstQualified())
     return true;
 
-  for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
-      ++I)
-    if (const Stmt *child = *I)
-      if (containsZeroConstant(child))
-        return true;
+  // Check for an enum
+  if (isa<EnumConstantDecl>(DR->getDecl()))
+    return true;
+
+  const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl());
+  if (!VD)
+    return true;
+
+  // Check if the Decl behaves like a constant. This check also takes care of
+  // static variables, which can only change between function calls if they are
+  // modified in the AST.
+  PseudoConstantAnalysis *PCA = AC->getPseudoConstantAnalysis();
+  if (PCA->isPseudoConstant(VD))
+    return true;
 
   return false;
 }
 
-bool IdempotentOperationChecker::containsOneConstant(const Stmt *S) {
-  const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(S);
-  if (IL && IL->getValue() == 1)
-    return true;
+// Recursively find any substatements containing VarDecl's with storage other
+// than local
+bool IdempotentOperationChecker::containsNonLocalVarDecl(const Stmt *S) {
+  const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(S);
 
-  const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(S);
-  const llvm::APFloat one(1.0);
-  if (FL && FL->getValue().compare(one) == llvm::APFloat::cmpEqual)
-    return true;
+  if (DR)
+    if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl()))
+      if (!VD->hasLocalStorage())
+        return true;
 
   for (Stmt::const_child_iterator I = S->child_begin(); I != S->child_end();
       ++I)
     if (const Stmt *child = *I)
-      if (containsOneConstant(child))
+      if (containsNonLocalVarDecl(child))
         return true;
 
   return false;
 }
-
diff --git a/lib/Checker/LLVMConventionsChecker.cpp b/lib/Checker/LLVMConventionsChecker.cpp
index c1212572f8f4..2f87da142c1a 100644
--- a/lib/Checker/LLVMConventionsChecker.cpp
+++ b/lib/Checker/LLVMConventionsChecker.cpp
@@ -128,7 +128,6 @@ public:
   void VisitDeclStmt(DeclStmt *DS);
 private:
   void VisitVarDecl(VarDecl *VD);
-  void CheckStringRefBoundtoTemporaryString(VarDecl *VD);
 };
 } // end anonymous namespace
 
diff --git a/lib/Checker/Makefile b/lib/Checker/Makefile
index 1bc652916403..4ec6f65a0bb1 100644
--- a/lib/Checker/Makefile
+++ b/lib/Checker/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangChecker
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Checker/MallocChecker.cpp b/lib/Checker/MallocChecker.cpp
index dcc21ca38619..c9b6d7588be1 100644
--- a/lib/Checker/MallocChecker.cpp
+++ b/lib/Checker/MallocChecker.cpp
@@ -24,15 +24,18 @@ using namespace clang;
 namespace {
 
 class RefState {
-  enum Kind { AllocateUnchecked, AllocateFailed, Released, Escaped } K;
+  enum Kind { AllocateUnchecked, AllocateFailed, Released, Escaped,
+              Relinquished } K;
   const Stmt *S;
 
 public:
   RefState(Kind k, const Stmt *s) : K(k), S(s) {}
 
   bool isAllocated() const { return K == AllocateUnchecked; }
+  //bool isFailed() const { return K == AllocateFailed; }
   bool isReleased() const { return K == Released; }
-  bool isEscaped() const { return K == Escaped; }
+  //bool isEscaped() const { return K == Escaped; }
+  //bool isRelinquished() const { return K == Relinquished; }
 
   bool operator==(const RefState &X) const {
     return K == X.K && S == X.S;
@@ -46,6 +49,9 @@ public:
   }
   static RefState getReleased(const Stmt *s) { return RefState(Released, s); }
   static RefState getEscaped(const Stmt *s) { return RefState(Escaped, s); }
+  static RefState getRelinquished(const Stmt *s) {
+    return RefState(Relinquished, s);
+  }
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
     ID.AddInteger(K);
@@ -59,23 +65,30 @@ class MallocChecker : public CheckerVisitor<MallocChecker> {
   BuiltinBug *BT_DoubleFree;
   BuiltinBug *BT_Leak;
   BuiltinBug *BT_UseFree;
+  BuiltinBug *BT_UseRelinquished;
   BuiltinBug *BT_BadFree;
   IdentifierInfo *II_malloc, *II_free, *II_realloc, *II_calloc;
 
 public:
   MallocChecker() 
-    : BT_DoubleFree(0), BT_Leak(0), BT_UseFree(0), BT_BadFree(0),
+    : BT_DoubleFree(0), BT_Leak(0), BT_UseFree(0), BT_UseRelinquished(0),
+      BT_BadFree(0),
       II_malloc(0), II_free(0), II_realloc(0), II_calloc(0) {}
   static void *getTag();
   bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
   void EvalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper);
   void EvalEndPath(GREndPathNodeBuilder &B, void *tag, GRExprEngine &Eng);
   void PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S);
-  const GRState *EvalAssume(const GRState *state, SVal Cond, bool Assumption);
+  const GRState *EvalAssume(const GRState *state, SVal Cond, bool Assumption,
+                            bool *respondsToCallback);
   void VisitLocation(CheckerContext &C, const Stmt *S, SVal l);
+  virtual void PreVisitBind(CheckerContext &C, const Stmt *StoreE,
+                            SVal location, SVal val);
 
 private:
   void MallocMem(CheckerContext &C, const CallExpr *CE);
+  void MallocMemReturnsAttr(CheckerContext &C, const CallExpr *CE,
+                            const OwnershipAttr* Att);
   const GRState *MallocMemAux(CheckerContext &C, const CallExpr *CE,
                               const Expr *SizeEx, SVal Init,
                               const GRState *state) {
@@ -86,8 +99,10 @@ private:
                               const GRState *state);
 
   void FreeMem(CheckerContext &C, const CallExpr *CE);
+  void FreeMemAttr(CheckerContext &C, const CallExpr *CE,
+                   const OwnershipAttr* Att);
   const GRState *FreeMemAux(CheckerContext &C, const CallExpr *CE,
-                            const GRState *state);
+                            const GRState *state, unsigned Num, bool Hold);
 
   void ReallocMem(CheckerContext &C, const CallExpr *CE);
   void CallocMem(CheckerContext &C, const CallExpr *CE);
@@ -103,7 +118,7 @@ typedef llvm::ImmutableMap<SymbolRef, RefState> RegionStateTy;
 namespace clang {
   template <>
   struct GRStateTrait<RegionState> 
-    : public GRStatePartialTrait<llvm::ImmutableMap<SymbolRef, RefState> > {
+    : public GRStatePartialTrait<RegionStateTy> {
     static void *GDMIndex() { return MallocChecker::getTag(); }
   };
 }
@@ -156,7 +171,32 @@ bool MallocChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
     return true;
   }
 
-  return false;
+  // Check all the attributes, if there are any.
+  // There can be multiple of these attributes.
+  bool rv = false;
+  if (FD->hasAttrs()) {
+    for (specific_attr_iterator<OwnershipAttr>
+                  i = FD->specific_attr_begin<OwnershipAttr>(),
+                  e = FD->specific_attr_end<OwnershipAttr>();
+         i != e; ++i) {
+      switch ((*i)->getOwnKind()) {
+      case OwnershipAttr::Returns: {
+        MallocMemReturnsAttr(C, CE, *i);
+        rv = true;
+        break;
+      }
+      case OwnershipAttr::Takes:
+      case OwnershipAttr::Holds: {
+        FreeMemAttr(C, CE, *i);
+        rv = true;
+        break;
+      }
+      default:
+        break;
+      }
+    }
+  }
+  return rv;
 }
 
 void MallocChecker::MallocMem(CheckerContext &C, const CallExpr *CE) {
@@ -165,6 +205,23 @@ void MallocChecker::MallocMem(CheckerContext &C, const CallExpr *CE) {
   C.addTransition(state);
 }
 
+void MallocChecker::MallocMemReturnsAttr(CheckerContext &C, const CallExpr *CE,
+                                         const OwnershipAttr* Att) {
+  if (Att->getModule() != "malloc")
+    return;
+
+  OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
+  if (I != E) {
+    const GRState *state =
+        MallocMemAux(C, CE, CE->getArg(*I), UndefinedVal(), C.getState());
+    C.addTransition(state);
+    return;
+  }
+  const GRState *state = MallocMemAux(C, CE, UnknownVal(), UndefinedVal(),
+                                        C.getState());
+  C.addTransition(state);
+}
+
 const GRState *MallocChecker::MallocMemAux(CheckerContext &C,  
                                            const CallExpr *CE,
                                            SVal Size, SVal Init,
@@ -196,25 +253,53 @@ const GRState *MallocChecker::MallocMemAux(CheckerContext &C,
 }
 
 void MallocChecker::FreeMem(CheckerContext &C, const CallExpr *CE) {
-  const GRState *state = FreeMemAux(C, CE, C.getState());
+  const GRState *state = FreeMemAux(C, CE, C.getState(), 0, false);
 
   if (state)
     C.addTransition(state);
 }
 
+void MallocChecker::FreeMemAttr(CheckerContext &C, const CallExpr *CE,
+                                const OwnershipAttr* Att) {
+  if (Att->getModule() != "malloc")
+    return;
+
+  for (OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
+       I != E; ++I) {
+    const GRState *state = FreeMemAux(C, CE, C.getState(), *I,
+                                      Att->getOwnKind() == OwnershipAttr::Holds);
+    if (state)
+      C.addTransition(state);
+  }
+}
+
 const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
-                                         const GRState *state) {
-  const Expr *ArgExpr = CE->getArg(0);
+                                         const GRState *state, unsigned Num,
+                                         bool Hold) {
+  const Expr *ArgExpr = CE->getArg(Num);
   SVal ArgVal = state->getSVal(ArgExpr);
 
-  // If ptr is NULL, no operation is preformed.
-  if (ArgVal.isZeroConstant())
+  DefinedOrUnknownSVal location = cast<DefinedOrUnknownSVal>(ArgVal);
+
+  // Check for null dereferences.
+  if (!isa<Loc>(location))
     return state;
-  
+
+  // FIXME: Technically using 'Assume' here can result in a path
+  //  bifurcation.  In such cases we need to return two states, not just one.
+  const GRState *notNullState, *nullState;
+  llvm::tie(notNullState, nullState) = state->Assume(location);
+
+  // The explicit NULL case, no operation is performed.
+  if (nullState && !notNullState)
+    return nullState;
+
+  assert(notNullState);
+
   // Unknown values could easily be okay
   // Undefined values are handled elsewhere
   if (ArgVal.isUnknownOrUndef())
-    return state;
+    return notNullState;
 
   const MemRegion *R = ArgVal.getAsRegion();
   
@@ -253,24 +338,23 @@ const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
   // Various cases could lead to non-symbol values here.
   // For now, ignore them.
   if (!SR)
-    return state;
+    return notNullState;
 
   SymbolRef Sym = SR->getSymbol();
-  
   const RefState *RS = state->get<RegionState>(Sym);
 
   // If the symbol has not been tracked, return. This is possible when free() is
   // called on a pointer that does not get its pointee directly from malloc(). 
   // Full support of this requires inter-procedural analysis.
   if (!RS)
-    return state;
+    return notNullState;
 
   // Check double free.
   if (RS->isReleased()) {
-    ExplodedNode *N = C.GenerateSink();
-    if (N) {
+    if (ExplodedNode *N = C.GenerateSink()) {
       if (!BT_DoubleFree)
-        BT_DoubleFree = new BuiltinBug("Double free",
+        BT_DoubleFree
+          = new BuiltinBug("Double free",
                          "Try to free a memory block that has been released");
       // FIXME: should find where it's freed last time.
       BugReport *R = new BugReport(*BT_DoubleFree, 
@@ -281,7 +365,9 @@ const GRState *MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
   }
 
   // Normal free.
-  return state->set<RegionState>(Sym, RefState::getReleased(CE));
+  if (Hold)
+    return notNullState->set<RegionState>(Sym, RefState::getRelinquished(CE));
+  return notNullState->set<RegionState>(Sym, RefState::getReleased(CE));
 }
 
 bool MallocChecker::SummarizeValue(llvm::raw_ostream& os, SVal V) {
@@ -376,8 +462,7 @@ bool MallocChecker::SummarizeRegion(llvm::raw_ostream& os,
 
 void MallocChecker::ReportBadFree(CheckerContext &C, SVal ArgVal,
                                   SourceRange range) {
-  ExplodedNode *N = C.GenerateSink();
-  if (N) {
+  if (ExplodedNode *N = C.GenerateSink()) {
     if (!BT_BadFree)
       BT_BadFree = new BuiltinBug("Bad free");
     
@@ -446,13 +531,13 @@ void MallocChecker::ReallocMem(CheckerContext &C, const CallExpr *CE) {
                                       ValMgr.makeIntValWithPtrWidth(0, false));
 
     if (const GRState *stateSizeZero = stateNotEqual->Assume(SizeZero, true)) {
-      const GRState *stateFree = FreeMemAux(C, CE, stateSizeZero);
+      const GRState *stateFree = FreeMemAux(C, CE, stateSizeZero, 0, false);
       if (stateFree)
         C.addTransition(stateFree->BindExpr(CE, UndefinedVal(), true));
     }
 
     if (const GRState *stateSizeNotZero=stateNotEqual->Assume(SizeZero,false)) {
-      const GRState *stateFree = FreeMemAux(C, CE, stateSizeNotZero);
+      const GRState *stateFree = FreeMemAux(C, CE, stateSizeNotZero, 0, false);
       if (stateFree) {
         // FIXME: We should copy the content of the original buffer.
         const GRState *stateRealloc = MallocMemAux(C, CE, CE->getArg(1), 
@@ -471,7 +556,7 @@ void MallocChecker::CallocMem(CheckerContext &C, const CallExpr *CE) {
 
   SVal Count = state->getSVal(CE->getArg(0));
   SVal EleSize = state->getSVal(CE->getArg(1));
-  SVal TotalSize = SVator.EvalBinOp(state, BinaryOperator::Mul, Count, EleSize,
+  SVal TotalSize = SVator.EvalBinOp(state, BO_Mul, Count, EleSize,
                                     ValMgr.getContext().getSizeType());
   
   SVal Zero = ValMgr.makeZeroVal(ValMgr.getContext().CharTy);
@@ -481,36 +566,42 @@ void MallocChecker::CallocMem(CheckerContext &C, const CallExpr *CE) {
 }
 
 void MallocChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
-  for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
-         E = SymReaper.dead_end(); I != E; ++I) {
-    SymbolRef Sym = *I;
-    const GRState *state = C.getState();
-    const RefState *RS = state->get<RegionState>(Sym);
-    if (!RS)
-      return;
-
-    if (RS->isAllocated()) {
-      ExplodedNode *N = C.GenerateSink();
-      if (N) {
-        if (!BT_Leak)
-          BT_Leak = new BuiltinBug("Memory leak",
+  if (!SymReaper.hasDeadSymbols())
+    return;
+
+  const GRState *state = C.getState();
+  RegionStateTy RS = state->get<RegionState>();
+  RegionStateTy::Factory &F = state->get_context<RegionState>();
+
+  for (RegionStateTy::iterator I = RS.begin(), E = RS.end(); I != E; ++I) {
+    if (SymReaper.isDead(I->first)) {
+      if (I->second.isAllocated()) {
+        if (ExplodedNode *N = C.GenerateNode()) {
+          if (!BT_Leak)
+            BT_Leak = new BuiltinBug("Memory leak",
                      "Allocated memory never released. Potential memory leak.");
-        // FIXME: where it is allocated.
-        BugReport *R = new BugReport(*BT_Leak, BT_Leak->getDescription(), N);
-        C.EmitReport(R);
+          // FIXME: where it is allocated.
+          BugReport *R = new BugReport(*BT_Leak, BT_Leak->getDescription(), N);
+          C.EmitReport(R);
+        }
       }
+
+      // Remove the dead symbol from the map.
+      RS = F.Remove(RS, I->first);
     }
   }
+
+  state = state->set<RegionState>(RS);
+  C.GenerateNode(state);
 }
 
 void MallocChecker::EvalEndPath(GREndPathNodeBuilder &B, void *tag,
                                 GRExprEngine &Eng) {
   SaveAndRestore<bool> OldHasGen(B.HasGeneratedNode);
   const GRState *state = B.getState();
-  typedef llvm::ImmutableMap<SymbolRef, RefState> SymMap;
-  SymMap M = state->get<RegionState>();
+  RegionStateTy M = state->get<RegionState>();
 
-  for (SymMap::iterator I = M.begin(), E = M.end(); I != E; ++I) {
+  for (RegionStateTy::iterator I = M.begin(), E = M.end(); I != E; ++I) {
     RefState RS = I->second;
     if (RS.isAllocated()) {
       ExplodedNode *N = B.generateNode(state, tag, B.getPredecessor());
@@ -549,7 +640,8 @@ void MallocChecker::PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S) {
 }
 
 const GRState *MallocChecker::EvalAssume(const GRState *state, SVal Cond, 
-                                         bool Assumption) {
+                                         bool Assumption,
+                                         bool * /* respondsToCallback */) {
   // If a symblic region is assumed to NULL, set its state to AllocateFailed.
   // FIXME: should also check symbols assumed to non-null.
 
@@ -568,9 +660,8 @@ void MallocChecker::VisitLocation(CheckerContext &C, const Stmt *S, SVal l) {
   SymbolRef Sym = l.getLocSymbolInBase();
   if (Sym) {
     const RefState *RS = C.getState()->get<RegionState>(Sym);
-    if (RS)
-      if (RS->isReleased()) {
-        ExplodedNode *N = C.GenerateSink();
+    if (RS && RS->isReleased()) {
+      if (ExplodedNode *N = C.GenerateNode()) {
         if (!BT_UseFree)
           BT_UseFree = new BuiltinBug("Use dynamically allocated memory after"
                                       " it is freed.");
@@ -579,5 +670,67 @@ void MallocChecker::VisitLocation(CheckerContext &C, const Stmt *S, SVal l) {
                                      N);
         C.EmitReport(R);
       }
+    }
+  }
+}
+
+void MallocChecker::PreVisitBind(CheckerContext &C,
+                                 const Stmt *StoreE,
+                                 SVal location,
+                                 SVal val) {
+  // The PreVisitBind implements the same algorithm as already used by the 
+  // Objective C ownership checker: if the pointer escaped from this scope by 
+  // assignment, let it go.  However, assigning to fields of a stack-storage 
+  // structure does not transfer ownership.
+
+  const GRState *state = C.getState();
+  DefinedOrUnknownSVal l = cast<DefinedOrUnknownSVal>(location);
+
+  // Check for null dereferences.
+  if (!isa<Loc>(l))
+    return;
+
+  // Before checking if the state is null, check if 'val' has a RefState.
+  // Only then should we check for null and bifurcate the state.
+  SymbolRef Sym = val.getLocSymbolInBase();
+  if (Sym) {
+    if (const RefState *RS = state->get<RegionState>(Sym)) {
+      // If ptr is NULL, no operation is performed.
+      const GRState *notNullState, *nullState;
+      llvm::tie(notNullState, nullState) = state->Assume(l);
+
+      // Generate a transition for 'nullState' to record the assumption
+      // that the state was null.
+      if (nullState)
+        C.addTransition(nullState);
+
+      if (!notNullState)
+        return;
+
+      if (RS->isAllocated()) {
+        // Something we presently own is being assigned somewhere.
+        const MemRegion *AR = location.getAsRegion();
+        if (!AR)
+          return;
+        AR = AR->StripCasts()->getBaseRegion();
+        do {
+          // If it is on the stack, we still own it.
+          if (AR->hasStackNonParametersStorage())
+            break;
+
+          // If the state can't represent this binding, we still own it.
+          if (notNullState == (notNullState->bindLoc(cast<Loc>(location),
+                                                     UnknownVal())))
+            break;
+
+          // We no longer own this pointer.
+          notNullState =
+            notNullState->set<RegionState>(Sym,
+                                           RefState::getRelinquished(StoreE));
+        }
+        while (false);
+      }
+      C.addTransition(notNullState);
+    }
   }
 }
diff --git a/lib/Checker/MemRegion.cpp b/lib/Checker/MemRegion.cpp
index 9cfeb7ae2b5c..3f706e145a82 100644
--- a/lib/Checker/MemRegion.cpp
+++ b/lib/Checker/MemRegion.cpp
@@ -18,6 +18,7 @@
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/Support/BumpVector.h"
 #include "clang/AST/CharUnits.h"
+#include "clang/AST/RecordLayout.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
@@ -177,7 +178,7 @@ const StackFrameContext *VarRegion::getStackFrame() const {
 
 DefinedOrUnknownSVal DeclRegion::getExtent(ValueManager& ValMgr) const {
   ASTContext& Ctx = ValMgr.getContext();
-  QualType T = getDesugaredValueType(Ctx);
+  QualType T = getDesugaredValueType();
 
   if (isa<VariableArrayType>(T))
     return nonloc::SymbolVal(ValMgr.getSymbolManager().getExtentSymbol(this));
@@ -195,8 +196,7 @@ DefinedOrUnknownSVal FieldRegion::getExtent(ValueManager& ValMgr) const {
   // A zero-length array at the end of a struct often stands for dynamically-
   // allocated extra memory.
   if (Extent.isZeroConstant()) {
-    ASTContext& Ctx = ValMgr.getContext();
-    QualType T = getDesugaredValueType(Ctx);
+    QualType T = getDesugaredValueType();
 
     if (isa<ConstantArrayType>(T))
       return UnknownVal();
@@ -785,7 +785,7 @@ static bool IsCompleteType(ASTContext &Ctx, QualType Ty) {
   return true;
 }
 
-RegionRawOffset ElementRegion::getAsRawOffset() const {
+RegionRawOffset ElementRegion::getAsArrayOffset() const {
   CharUnits offset = CharUnits::Zero();
   const ElementRegion *ER = this;
   const MemRegion *superR = NULL;
@@ -827,6 +827,67 @@ RegionRawOffset ElementRegion::getAsRawOffset() const {
   return RegionRawOffset(superR, offset.getQuantity());
 }
 
+RegionOffset MemRegion::getAsOffset() const {
+  const MemRegion *R = this;
+  int64_t Offset = 0;
+
+  while (1) {
+    switch (R->getKind()) {
+    default:
+      return RegionOffset(0);
+    case SymbolicRegionKind:
+    case AllocaRegionKind:
+    case CompoundLiteralRegionKind:
+    case CXXThisRegionKind:
+    case StringRegionKind:
+    case VarRegionKind:
+    case CXXObjectRegionKind:
+      goto Finish;
+    case ElementRegionKind: {
+      const ElementRegion *ER = cast<ElementRegion>(R);
+      QualType EleTy = ER->getValueType();
+
+      if (!IsCompleteType(getContext(), EleTy))
+        return RegionOffset(0);
+
+      SVal Index = ER->getIndex();
+      if (const nonloc::ConcreteInt *CI=dyn_cast<nonloc::ConcreteInt>(&Index)) {
+        int64_t i = CI->getValue().getSExtValue();
+        CharUnits Size = getContext().getTypeSizeInChars(EleTy);
+        Offset += i * Size.getQuantity() * 8;
+      } else {
+        // We cannot compute offset for non-concrete index.
+        return RegionOffset(0);
+      }
+      R = ER->getSuperRegion();
+      break;
+    }
+    case FieldRegionKind: {
+      const FieldRegion *FR = cast<FieldRegion>(R);
+      const RecordDecl *RD = FR->getDecl()->getParent();
+      if (!RD->isDefinition())
+        // We cannot compute offset for incomplete type.
+        return RegionOffset(0);
+      // Get the field number.
+      unsigned idx = 0;
+      for (RecordDecl::field_iterator FI = RD->field_begin(), 
+             FE = RD->field_end(); FI != FE; ++FI, ++idx)
+        if (FR->getDecl() == *FI)
+          break;
+
+      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
+      // This is offset in bits.
+      Offset += Layout.getFieldOffset(idx);
+      R = FR->getSuperRegion();
+      break;
+    }
+    }
+  }
+
+ Finish:
+  return RegionOffset(R, Offset);
+}
+
 //===----------------------------------------------------------------------===//
 // BlockDataRegion
 //===----------------------------------------------------------------------===//
diff --git a/lib/Checker/OSAtomicChecker.cpp b/lib/Checker/OSAtomicChecker.cpp
index 1ea1bd98d6dc..02de0a84a4e4 100644
--- a/lib/Checker/OSAtomicChecker.cpp
+++ b/lib/Checker/OSAtomicChecker.cpp
@@ -110,9 +110,9 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
   QualType LoadTy;
   if (const TypedRegion *TR =
       dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
-    LoadTy = TR->getValueType(Ctx);
+    LoadTy = TR->getValueType();
   }
-  Engine.EvalLoad(Tmp, const_cast<Expr *>(theValueExpr), C.getPredecessor(), 
+  Engine.EvalLoad(Tmp, theValueExpr, C.getPredecessor(), 
                   state, location, OSAtomicLoadTag, LoadTy);
 
   if (Tmp.empty()) {
@@ -158,10 +158,10 @@ bool OSAtomicChecker::EvalOSAtomicCompareAndSwap(CheckerContext &C,
       // Handle implicit value casts.
       if (const TypedRegion *R =
           dyn_cast_or_null<TypedRegion>(location.getAsRegion())) {
-        val = SVator.EvalCast(val,R->getValueType(Ctx),newValueExpr->getType());
+        val = SVator.EvalCast(val,R->getValueType(), newValueExpr->getType());
       }
 
-      Engine.EvalStore(TmpStore, NULL, const_cast<Expr *>(theValueExpr), N, 
+      Engine.EvalStore(TmpStore, NULL, theValueExpr, N, 
                        stateEqual, location, val, OSAtomicStoreTag);
 
       if (TmpStore.empty()) {
diff --git a/lib/Checker/PointerArithChecker.cpp b/lib/Checker/PointerArithChecker.cpp
index ed60c42613fe..cbac423b467c 100644
--- a/lib/Checker/PointerArithChecker.cpp
+++ b/lib/Checker/PointerArithChecker.cpp
@@ -36,8 +36,7 @@ void *PointerArithChecker::getTag() {
 
 void PointerArithChecker::PreVisitBinaryOperator(CheckerContext &C,
                                                  const BinaryOperator *B) {
-  if (B->getOpcode() != BinaryOperator::Sub &&
-      B->getOpcode() != BinaryOperator::Add)
+  if (B->getOpcode() != BO_Sub && B->getOpcode() != BO_Add)
     return;
 
   const GRState *state = C.getState();
diff --git a/lib/Checker/PointerSubChecker.cpp b/lib/Checker/PointerSubChecker.cpp
index bc0fd24d19b4..d64b6ae3fe3f 100644
--- a/lib/Checker/PointerSubChecker.cpp
+++ b/lib/Checker/PointerSubChecker.cpp
@@ -39,7 +39,7 @@ void PointerSubChecker::PreVisitBinaryOperator(CheckerContext &C,
                                                const BinaryOperator *B) {
   // When doing pointer subtraction, if the two pointers do not point to the
   // same memory chunk, emit a warning.
-  if (B->getOpcode() != BinaryOperator::Sub)
+  if (B->getOpcode() != BO_Sub)
     return;
 
   const GRState *state = C.getState();
diff --git a/lib/Checker/RangeConstraintManager.cpp b/lib/Checker/RangeConstraintManager.cpp
index 2a35d326a988..697694e72a36 100644
--- a/lib/Checker/RangeConstraintManager.cpp
+++ b/lib/Checker/RangeConstraintManager.cpp
@@ -83,7 +83,6 @@ public:
   typedef PrimRangeSet::iterator iterator;
 
   RangeSet(PrimRangeSet RS) : ranges(RS) {}
-  RangeSet(Factory& F) : ranges(F.GetEmptySet()) {}
 
   iterator begin() const { return ranges.begin(); }
   iterator end() const { return ranges.end(); }
diff --git a/lib/Checker/RegionStore.cpp b/lib/Checker/RegionStore.cpp
index 74a7fee04889..1a3eded7cb06 100644
--- a/lib/Checker/RegionStore.cpp
+++ b/lib/Checker/RegionStore.cpp
@@ -44,10 +44,9 @@ private:
   uint64_t Offset;
 
   explicit BindingKey(const MemRegion *r, uint64_t offset, Kind k)
-    : P(r, (unsigned) k), Offset(offset) { assert(r); }
+    : P(r, (unsigned) k), Offset(offset) {}
 public:
 
-  bool isDefault() const { return P.getInt() == Default; }
   bool isDirect() const { return P.getInt() == Direct; }
 
   const MemRegion *getRegion() const { return P.getPointer(); }
@@ -72,9 +71,26 @@ public:
     return P.getOpaqueValue() == X.P.getOpaqueValue() &&
            Offset == X.Offset;
   }
+
+  bool isValid() const {
+    return getRegion() != NULL;
+  }
 };
 } // end anonymous namespace
 
+BindingKey BindingKey::Make(const MemRegion *R, Kind k) {
+  if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
+    const RegionRawOffset &O = ER->getAsArrayOffset();
+
+    // FIXME: There are some ElementRegions for which we cannot compute
+    // raw offsets yet, including regions with symbolic offsets. These will be
+    // ignored by the store.
+    return BindingKey(O.getRegion(), O.getByteOffset(), k);
+  }
+
+  return BindingKey(R, 0, k);
+}
+
 namespace llvm {
   static inline
   llvm::raw_ostream& operator<<(llvm::raw_ostream& os, BindingKey K) {
@@ -101,35 +117,20 @@ struct maximal_features_tag {};
 
 class RegionStoreFeatures {
   bool SupportsFields;
-  bool SupportsRemaining;
-
 public:
   RegionStoreFeatures(minimal_features_tag) :
-    SupportsFields(false), SupportsRemaining(false) {}
+    SupportsFields(false) {}
 
   RegionStoreFeatures(maximal_features_tag) :
-    SupportsFields(true), SupportsRemaining(false) {}
+    SupportsFields(true) {}
 
   void enableFields(bool t) { SupportsFields = t; }
 
   bool supportsFields() const { return SupportsFields; }
-  bool supportsRemaining() const { return SupportsRemaining; }
 };
 }
 
 //===----------------------------------------------------------------------===//
-// Utility functions.
-//===----------------------------------------------------------------------===//
-
-static bool IsAnyPointerOrIntptr(QualType ty, ASTContext &Ctx) {
-  if (ty->isAnyPointerType())
-    return true;
-
-  return ty->isIntegerType() && ty->isScalarType() &&
-         Ctx.getTypeSize(ty) == Ctx.getTypeSize(Ctx.VoidPtrTy);
-}
-
-//===----------------------------------------------------------------------===//
 // Main RegionStore logic.
 //===----------------------------------------------------------------------===//
 
@@ -180,6 +181,14 @@ public:
   }
 };
 
+void
+RegionStoreSubRegionMap::process(llvm::SmallVectorImpl<const SubRegion*> &WL,
+                                 const SubRegion *R) {
+  const MemRegion *superR = R->getSuperRegion();
+  if (add(superR, R))
+    if (const SubRegion *sr = dyn_cast<SubRegion>(superR))
+      WL.push_back(sr);
+}
 
 class RegionStoreManager : public StoreManager {
   const RegionStoreFeatures Features;
@@ -197,7 +206,6 @@ public:
 
   RegionStoreSubRegionMap *getRegionStoreSubRegionMap(Store store);
 
-  Optional<SVal> getBinding(RegionBindings B, const MemRegion *R);
   Optional<SVal> getDirectBinding(RegionBindings B, const MemRegion *R);
   /// getDefaultBinding - Returns an SVal* representing an optional default
   ///  binding associated with a region and its subregions.
@@ -226,18 +234,13 @@ public:
   // Binding values to regions.
   //===-------------------------------------------------------------------===//
 
-  Store InvalidateRegion(Store store, const MemRegion *R, const Expr *E,
-                         unsigned Count, InvalidatedSymbols *IS) {
-    return RegionStoreManager::InvalidateRegions(store, &R, &R+1, E, Count, IS,
-                                                 false);
-  }
-
   Store InvalidateRegions(Store store,
                           const MemRegion * const *Begin,
                           const MemRegion * const *End,
                           const Expr *E, unsigned Count,
                           InvalidatedSymbols *IS,
-                          bool invalidateGlobals);
+                          bool invalidateGlobals,
+                          InvalidatedRegions *Regions);
 
 public:   // Made public for helper classes.
 
@@ -260,8 +263,6 @@ public:   // Made public for helper classes.
     return Remove(Remove(B, R, BindingKey::Direct), R, BindingKey::Default);
   }
 
-  Store Remove(Store store, BindingKey K);
-
 public: // Part of public interface to class.
 
   Store Bind(Store store, Loc LV, SVal V);
@@ -289,7 +290,7 @@ public: // Part of public interface to class.
   Store BindArray(Store store, const TypedRegion* R, SVal V);
 
   /// KillStruct - Set the entire struct to unknown.
-  Store KillStruct(Store store, const TypedRegion* R);
+  Store KillStruct(Store store, const TypedRegion* R, SVal DefaultVal);
 
   Store Remove(Store store, Loc LV);
 
@@ -352,13 +353,11 @@ public: // Part of public interface to class.
 
   /// RemoveDeadBindings - Scans the RegionStore of 'state' for dead values.
   ///  It returns a new Store with these values removed.
-  const GRState *RemoveDeadBindings(GRState &state, 
-                                    const StackFrameContext *LCtx,
-                                    SymbolReaper& SymReaper,
+  Store RemoveDeadBindings(Store store, const StackFrameContext *LCtx,
+                           SymbolReaper& SymReaper,
                           llvm::SmallVectorImpl<const MemRegion*>& RegionRoots);
 
-  const GRState *EnterStackFrame(const GRState *state,
-                                 const StackFrameContext *frame);
+  Store EnterStackFrame(const GRState *state, const StackFrameContext *frame);
 
   //===------------------------------------------------------------------===//
   // Region "extents".
@@ -392,9 +391,6 @@ public: // Part of public interface to class.
       }
     }
   }
-
-  // FIXME: Remove.
-  ASTContext& getContext() { return StateMgr.getContext(); }
 };
 
 } // end anonymous namespace
@@ -414,14 +410,6 @@ StoreManager *clang::CreateFieldsOnlyRegionStoreManager(GRStateManager &StMgr) {
   return new RegionStoreManager(StMgr, F);
 }
 
-void
-RegionStoreSubRegionMap::process(llvm::SmallVectorImpl<const SubRegion*> &WL,
-                                 const SubRegion *R) {
-  const MemRegion *superR = R->getSuperRegion();
-  if (add(superR, R))
-    if (const SubRegion *sr = dyn_cast<SubRegion>(superR))
-      WL.push_back(sr);
-}
 
 RegionStoreSubRegionMap*
 RegionStoreManager::getRegionStoreSubRegionMap(Store store) {
@@ -579,14 +567,16 @@ class InvalidateRegionsWorker : public ClusterAnalysis<InvalidateRegionsWorker>
   const Expr *Ex;
   unsigned Count;
   StoreManager::InvalidatedSymbols *IS;
+  StoreManager::InvalidatedRegions *Regions;
 public:
   InvalidateRegionsWorker(RegionStoreManager &rm,
                           GRStateManager &stateMgr,
                           RegionBindings b,
                           const Expr *ex, unsigned count,
-                          StoreManager::InvalidatedSymbols *is)
+                          StoreManager::InvalidatedSymbols *is,
+                          StoreManager::InvalidatedRegions *r)
     : ClusterAnalysis<InvalidateRegionsWorker>(rm, stateMgr, b),
-      Ex(ex), Count(count), IS(is) {}
+      Ex(ex), Count(count), IS(is), Regions(r) {}
 
   void VisitCluster(const MemRegion *baseR, BindingKey *I, BindingKey *E);
   void VisitBaseRegion(const MemRegion *baseR);
@@ -657,6 +647,10 @@ void InvalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
     return;
   }
 
+  // Otherwise, we have a normal data region. Record that we touched the region.
+  if (Regions)
+    Regions->push_back(baseR);
+
   if (isa<AllocaRegion>(baseR) || isa<SymbolicRegion>(baseR)) {
     // Invalidate the region by setting its default value to
     // conjured symbol. The type of the symbol is irrelavant.
@@ -670,19 +664,12 @@ void InvalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
     return;
 
   const TypedRegion *TR = cast<TypedRegion>(baseR);
-  QualType T = TR->getValueType(Ctx);
+  QualType T = TR->getValueType();
 
     // Invalidate the binding.
-  if (const RecordType *RT = T->getAsStructureType()) {
-    const RecordDecl *RD = RT->getDecl()->getDefinition();
-      // No record definition.  There is nothing we can do.
-    if (!RD) {
-      B = RM.Remove(B, baseR);
-      return;
-    }
-
-      // Invalidate the region by setting its default value to
-      // conjured symbol. The type of the symbol is irrelavant.
+  if (T->isStructureType()) {
+    // Invalidate the region by setting its default value to
+    // conjured symbol. The type of the symbol is irrelavant.
     DefinedOrUnknownSVal V = ValMgr.getConjuredSymbolVal(baseR, Ex, Ctx.IntTy,
                                                          Count);
     B = RM.Add(B, baseR, BindingKey::Default, V);
@@ -707,10 +694,11 @@ Store RegionStoreManager::InvalidateRegions(Store store,
                                             const MemRegion * const *E,
                                             const Expr *Ex, unsigned Count,
                                             InvalidatedSymbols *IS,
-                                            bool invalidateGlobals) {
+                                            bool invalidateGlobals,
+                                            InvalidatedRegions *Regions) {
   InvalidateRegionsWorker W(*this, StateMgr,
                             RegionStoreManager::GetRegionBindings(store),
-                            Ex, Count, IS);
+                            Ex, Count, IS, Regions);
 
   // Scan the bindings and generate the clusters.
   W.GenerateClusters(invalidateGlobals);
@@ -733,6 +721,11 @@ Store RegionStoreManager::InvalidateRegions(Store store,
                                   /* symbol type, doesn't matter */ Ctx.IntTy,
                                   Count);
     B = Add(B, BindingKey::Make(GS, BindingKey::Default), V);
+
+    // Even if there are no bindings in the global scope, we still need to
+    // record that we touched it.
+    if (Regions)
+      Regions->push_back(GS);
   }
 
   return B.getRoot();
@@ -752,7 +745,7 @@ DefinedOrUnknownSVal RegionStoreManager::getSizeInElements(const GRState *state,
     return UnknownVal();
 
   CharUnits RegionSize = CharUnits::fromQuantity(SizeInt->getSExtValue());
-  CharUnits EleSize = getContext().getTypeSizeInChars(EleTy);
+  CharUnits EleSize = Ctx.getTypeSizeInChars(EleTy);
 
   // If a variable is reinterpreted as a type that doesn't fit into a larger
   // type evenly, round it down.
@@ -781,13 +774,12 @@ SVal RegionStoreManager::ArrayToPointer(Loc Array) {
     return UnknownVal();
 
   // Strip off typedefs from the ArrayRegion's ValueType.
-  QualType T = ArrayR->getValueType(getContext()).getDesugaredType();
+  QualType T = ArrayR->getValueType().getDesugaredType();
   ArrayType *AT = cast<ArrayType>(T);
   T = AT->getElementType();
 
   SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
-  return loc::MemRegionVal(MRMgr.getElementRegion(T, ZeroIdx, ArrayR,
-                                                  getContext()));
+  return loc::MemRegionVal(MRMgr.getElementRegion(T, ZeroIdx, ArrayR, Ctx));
 }
 
 //===----------------------------------------------------------------------===//
@@ -806,8 +798,8 @@ SVal RegionStoreManager::EvalBinOp(BinaryOperator::Opcode Op, Loc L, NonLoc R,
     default:
       // Handle it normally.
       break;
-    case BinaryOperator::Add:
-    case BinaryOperator::Sub:
+    case BO_Add:
+    case BO_Sub:
       // FIXME: does this need to be casted to match resultTy?
       return L;
     }
@@ -820,7 +812,7 @@ SVal RegionStoreManager::EvalBinOp(BinaryOperator::Opcode Op, Loc L, NonLoc R,
     case MemRegion::SymbolicRegionKind: {
       const SymbolicRegion *SR = cast<SymbolicRegion>(MR);
       SymbolRef Sym = SR->getSymbol();
-      QualType T = Sym->getType(getContext());
+      QualType T = Sym->getType(Ctx);
       QualType EleTy;
 
       if (const PointerType *PT = T->getAs<PointerType>())
@@ -829,14 +821,14 @@ SVal RegionStoreManager::EvalBinOp(BinaryOperator::Opcode Op, Loc L, NonLoc R,
         EleTy = T->getAs<ObjCObjectPointerType>()->getPointeeType();
 
       SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
-      ER = MRMgr.getElementRegion(EleTy, ZeroIdx, SR, getContext());
+      ER = MRMgr.getElementRegion(EleTy, ZeroIdx, SR, Ctx);
       break;
     }
     case MemRegion::AllocaRegionKind: {
       const AllocaRegion *AR = cast<AllocaRegion>(MR);
-      QualType EleTy = getContext().CharTy; // Create an ElementRegion of bytes.
+      QualType EleTy = Ctx.CharTy; // Create an ElementRegion of bytes.
       SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
-      ER = MRMgr.getElementRegion(EleTy, ZeroIdx, AR, getContext());
+      ER = MRMgr.getElementRegion(EleTy, ZeroIdx, AR, Ctx);
       break;
     }
 
@@ -891,13 +883,13 @@ SVal RegionStoreManager::EvalBinOp(BinaryOperator::Opcode Op, Loc L, NonLoc R,
                 cast<nonloc::ConcreteInt>(ValMgr.convertToArrayIndex(*Offset)));
       const MemRegion* NewER =
         MRMgr.getElementRegion(ER->getElementType(), NewIdx,
-                               ER->getSuperRegion(), getContext());
+                               ER->getSuperRegion(), Ctx);
       return ValMgr.makeLoc(NewER);
     }
     if (0 == Base->getValue()) {
       const MemRegion* NewER =
         MRMgr.getElementRegion(ER->getElementType(), R,
-                               ER->getSuperRegion(), getContext());
+                               ER->getSuperRegion(), Ctx);
       return ValMgr.makeLoc(NewER);
     }
   }
@@ -922,7 +914,7 @@ Optional<SVal> RegionStoreManager::getDefaultBinding(RegionBindings B,
                                                      const MemRegion *R) {
   if (R->isBoundable())
     if (const TypedRegion *TR = dyn_cast<TypedRegion>(R))
-      if (TR->getValueType(getContext())->isUnionType())
+      if (TR->getValueType()->isUnionType())
         return UnknownVal();
 
   if (const SVal *V = Lookup(B, R, BindingKey::Default))
@@ -931,38 +923,6 @@ Optional<SVal> RegionStoreManager::getDefaultBinding(RegionBindings B,
   return Optional<SVal>();
 }
 
-Optional<SVal> RegionStoreManager::getBinding(RegionBindings B,
-                                              const MemRegion *R) {
-
-  if (const Optional<SVal> &V = getDirectBinding(B, R))
-    return V;
-
-  return getDefaultBinding(B, R);
-}
-
-static bool IsReinterpreted(QualType RTy, QualType UsedTy, ASTContext &Ctx) {
-  RTy = Ctx.getCanonicalType(RTy);
-  UsedTy = Ctx.getCanonicalType(UsedTy);
-
-  if (RTy == UsedTy)
-    return false;
-
-
-  // Recursively check the types.  We basically want to see if a pointer value
-  // is ever reinterpreted as a non-pointer, e.g. void** and intptr_t*
-  // represents a reinterpretation.
-  if (Loc::IsLocType(RTy) && Loc::IsLocType(UsedTy)) {
-    const PointerType *PRTy = RTy->getAs<PointerType>();
-    const PointerType *PUsedTy = UsedTy->getAs<PointerType>();
-
-    return PUsedTy && PRTy &&
-           IsReinterpreted(PRTy->getPointeeType(),
-                           PUsedTy->getPointeeType(), Ctx);
-  }
-
-  return true;
-}
-
 SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   assert(!isa<UnknownVal>(L) && "location unknown");
   assert(!isa<UndefinedVal>(L) && "location undefined");
@@ -977,7 +937,7 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   if (isa<AllocaRegion>(MR) || isa<SymbolicRegion>(MR)) {
     if (T.isNull()) {
       const SymbolicRegion *SR = cast<SymbolicRegion>(MR);
-      T = SR->getSymbol()->getType(getContext());
+      T = SR->getSymbol()->getType(Ctx);
     }
     MR = GetElementZeroRegion(MR, T);
   }
@@ -990,7 +950,7 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   // FIXME: Perhaps this method should just take a 'const MemRegion*' argument
   //  instead of 'Loc', and have the other Loc cases handled at a higher level.
   const TypedRegion *R = cast<TypedRegion>(MR);
-  QualType RTy = R->getValueType(getContext());
+  QualType RTy = R->getValueType();
 
   // FIXME: We should eventually handle funny addressing.  e.g.:
   //
@@ -1001,17 +961,6 @@ SVal RegionStoreManager::Retrieve(Store store, Loc L, QualType T) {
   //
   // Such funny addressing will occur due to layering of regions.
 
-#if 0
-  ASTContext &Ctx = getContext();
-  if (!T.isNull() && IsReinterpreted(RTy, T, Ctx)) {
-    SVal ZeroIdx = ValMgr.makeZeroArrayIndex();
-    R = MRMgr.getElementRegion(T, ZeroIdx, R, Ctx);
-    RTy = T;
-    assert(Ctx.getCanonicalType(RTy) ==
-           Ctx.getCanonicalType(R->getValueType(Ctx)));
-  }
-#endif
-
   if (RTy->isStructureOrClassType())
     return RetrieveStruct(store, R);
 
@@ -1121,8 +1070,7 @@ SVal RegionStoreManager::RetrieveElement(Store store,
   if (const StringRegion *StrR=dyn_cast<StringRegion>(superR)) {
     // FIXME: Handle loads from strings where the literal is treated as
     // an integer, e.g., *((unsigned int*)"hello")
-    ASTContext &Ctx = getContext();
-    QualType T = Ctx.getAsArrayType(StrR->getValueType(Ctx))->getElementType();
+    QualType T = Ctx.getAsArrayType(StrR->getValueType())->getElementType();
     if (T != Ctx.getCanonicalType(R->getElementType()))
       return UnknownVal();
 
@@ -1131,16 +1079,18 @@ SVal RegionStoreManager::RetrieveElement(Store store,
     if (nonloc::ConcreteInt *CI = dyn_cast<nonloc::ConcreteInt>(&Idx)) {
       int64_t i = CI->getValue().getSExtValue();
       int64_t byteLength = Str->getByteLength();
-      if (i > byteLength) {
-        // Buffer overflow checking in GRExprEngine should handle this case,
-        // but we shouldn't rely on it to not overflow here if that checking
-        // is disabled.
-        return UnknownVal();
-      }
-      char c = (i == byteLength) ? '\0' : Str->getStrData()[i];
+      // Technically, only i == byteLength is guaranteed to be null.
+      // However, such overflows should be caught before reaching this point;
+      // the only time such an access would be made is if a string literal was
+      // used to initialize a larger array.
+      char c = (i >= byteLength) ? '\0' : Str->getString()[i];
       return ValMgr.makeIntVal(c, T);
     }
   }
+  
+  // Check for loads from a code text region.  For such loads, just give up.
+  if (isa<CodeTextRegion>(superR))
+    return UnknownVal();
 
   // Handle the case where we are indexing into a larger scalar object.
   // For example, this handles:
@@ -1148,9 +1098,9 @@ SVal RegionStoreManager::RetrieveElement(Store store,
   //   char *y = &x;
   //   return *y;
   // FIXME: This is a hack, and doesn't do anything really intelligent yet.
-  const RegionRawOffset &O = R->getAsRawOffset();
+  const RegionRawOffset &O = R->getAsArrayOffset();
   if (const TypedRegion *baseR = dyn_cast_or_null<TypedRegion>(O.getRegion())) {
-    QualType baseT = baseR->getValueType(Ctx);
+    QualType baseT = baseR->getValueType();
     if (baseT->isScalarType()) {
       QualType elemT = R->getElementType();
       if (elemT->isScalarType()) {
@@ -1180,7 +1130,7 @@ SVal RegionStoreManager::RetrieveField(Store store,
   if (const Optional<SVal> &V = getDirectBinding(B, R))
     return *V;
 
-  QualType Ty = R->getValueType(getContext());
+  QualType Ty = R->getValueType();
   return RetrieveFieldOrElementCommon(store, R, Ty, R->getSuperRegion());
 }
 
@@ -1243,13 +1193,18 @@ SVal RegionStoreManager::RetrieveFieldOrElementCommon(Store store,
   }
 
   if (R->hasStackNonParametersStorage()) {
-    if (isa<ElementRegion>(R)) {
+    if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
       // Currently we don't reason specially about Clang-style vectors.  Check
       // if superR is a vector and if so return Unknown.
       if (const TypedRegion *typedSuperR = dyn_cast<TypedRegion>(superR)) {
-        if (typedSuperR->getValueType(getContext())->isVectorType())
+        if (typedSuperR->getValueType()->isVectorType())
           return UnknownVal();
       }
+      
+      // FIXME: We also need to take ElementRegions with symbolic indexes into
+      // account.
+      if (!ER->getIndex().isConstant())
+        return UnknownVal();
     }
 
     return UndefinedVal();
@@ -1332,21 +1287,18 @@ SVal RegionStoreManager::RetrieveVar(Store store, const VarRegion *R) {
 }
 
 SVal RegionStoreManager::RetrieveLazySymbol(const TypedRegion *R) {
-
-  QualType valTy = R->getValueType(getContext());
-
   // All other values are symbolic.
   return ValMgr.getRegionValueSymbolVal(R);
 }
 
 SVal RegionStoreManager::RetrieveStruct(Store store, const TypedRegion* R) {
-  QualType T = R->getValueType(getContext());
+  QualType T = R->getValueType();
   assert(T->isStructureOrClassType());
   return ValMgr.makeLazyCompoundVal(store, R);
 }
 
 SVal RegionStoreManager::RetrieveArray(Store store, const TypedRegion * R) {
-  assert(isa<ConstantArrayType>(R->getValueType(getContext())));
+  assert(isa<ConstantArrayType>(R->getValueType()));
   return ValMgr.makeLazyCompoundVal(store, R);
 }
 
@@ -1371,38 +1323,26 @@ Store RegionStoreManager::Bind(Store store, Loc L, SVal V) {
 
   // Check if the region is a struct region.
   if (const TypedRegion* TR = dyn_cast<TypedRegion>(R))
-    if (TR->getValueType(getContext())->isStructureOrClassType())
+    if (TR->getValueType()->isStructureOrClassType())
       return BindStruct(store, TR, V);
 
-  // Special case: the current region represents a cast and it and the super
-  // region both have pointer types or intptr_t types.  If so, perform the
-  // bind to the super region.
-  // This is needed to support OSAtomicCompareAndSwap and friends or other
-  // loads that treat integers as pointers and vis versa.
   if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
     if (ER->getIndex().isZeroConstant()) {
       if (const TypedRegion *superR =
             dyn_cast<TypedRegion>(ER->getSuperRegion())) {
-        ASTContext &Ctx = getContext();
-        QualType superTy = superR->getValueType(Ctx);
-        QualType erTy = ER->getValueType(Ctx);
-
-        if (IsAnyPointerOrIntptr(superTy, Ctx) &&
-            IsAnyPointerOrIntptr(erTy, Ctx)) {
-          V = ValMgr.getSValuator().EvalCast(V, superTy, erTy);
-          return Bind(store, loc::MemRegionVal(superR), V);
-        }
+        QualType superTy = superR->getValueType();
         // For now, just invalidate the fields of the struct/union/class.
+        // This is for test rdar_test_7185607 in misc-ps-region-store.m.
         // FIXME: Precisely handle the fields of the record.
-        if (superTy->isRecordType())
-          return InvalidateRegion(store, superR, NULL, 0, NULL);
+        if (superTy->isStructureOrClassType())
+          return KillStruct(store, superR, UnknownVal());
       }
     }
   }
   else if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(R)) {
     // Binding directly to a symbolic region should be treated as binding
     // to element 0.
-    QualType T = SR->getSymbol()->getType(getContext());
+    QualType T = SR->getSymbol()->getType(Ctx);
 
     // FIXME: Is this the right way to handle symbols that are references?
     if (const PointerType *PT = T->getAs<PointerType>())
@@ -1454,7 +1394,7 @@ Store RegionStoreManager::setImplicitDefaultValue(Store store,
   else if (T->isStructureOrClassType() || T->isArrayType()) {
     // Set the default value to a zero constant when it is a structure
     // or array.  The type doesn't really matter.
-    V = ValMgr.makeZeroVal(ValMgr.getContext().IntTy);
+    V = ValMgr.makeZeroVal(Ctx.IntTy);
   }
   else {
     return store;
@@ -1466,44 +1406,21 @@ Store RegionStoreManager::setImplicitDefaultValue(Store store,
 Store RegionStoreManager::BindArray(Store store, const TypedRegion* R,
                                     SVal Init) {
 
-  ASTContext &Ctx = getContext();
-  const ArrayType *AT =
-    cast<ArrayType>(Ctx.getCanonicalType(R->getValueType(Ctx)));
+  const ArrayType *AT =cast<ArrayType>(Ctx.getCanonicalType(R->getValueType()));
   QualType ElementTy = AT->getElementType();
   Optional<uint64_t> Size;
 
   if (const ConstantArrayType* CAT = dyn_cast<ConstantArrayType>(AT))
     Size = CAT->getSize().getZExtValue();
 
-  // Check if the init expr is a StringLiteral.
-  if (isa<loc::MemRegionVal>(Init)) {
-    const MemRegion* InitR = cast<loc::MemRegionVal>(Init).getRegion();
-    const StringLiteral* S = cast<StringRegion>(InitR)->getStringLiteral();
-    const char* str = S->getStrData();
-    unsigned len = S->getByteLength();
-    unsigned j = 0;
-
-    // Copy bytes from the string literal into the target array. Trailing bytes
-    // in the array that are not covered by the string literal are initialized
-    // to zero.
+  // Check if the init expr is a string literal.
+  if (loc::MemRegionVal *MRV = dyn_cast<loc::MemRegionVal>(&Init)) {
+    const StringRegion *S = cast<StringRegion>(MRV->getRegion());
 
-    // We assume that string constants are bound to
-    // constant arrays.
-    uint64_t size = Size.getValue();
-
-    for (uint64_t i = 0; i < size; ++i, ++j) {
-      if (j >= len)
-        break;
-
-      SVal Idx = ValMgr.makeArrayIndex(i);
-      const ElementRegion* ER = MRMgr.getElementRegion(ElementTy, Idx, R,
-                                                       getContext());
-
-      SVal V = ValMgr.makeIntVal(str[j], sizeof(char)*8, true);
-      store = Bind(store, loc::MemRegionVal(ER), V);
-    }
-
-    return store;
+    // Treat the string as a lazy compound value.
+    nonloc::LazyCompoundVal LCV =
+      cast<nonloc::LazyCompoundVal>(ValMgr.makeLazyCompoundVal(store, S));
+    return CopyLazyBindings(LCV, store, R);
   }
 
   // Handle lazy compound values.
@@ -1525,10 +1442,12 @@ Store RegionStoreManager::BindArray(Store store, const TypedRegion* R,
       break;
 
     SVal Idx = ValMgr.makeArrayIndex(i);
-    const ElementRegion *ER = MRMgr.getElementRegion(ElementTy, Idx, R, getContext());
+    const ElementRegion *ER = MRMgr.getElementRegion(ElementTy, Idx, R, Ctx);
 
     if (ElementTy->isStructureOrClassType())
       store = BindStruct(store, ER, *VI);
+    else if (ElementTy->isArrayType())
+      store = BindArray(store, ER, *VI);
     else
       store = Bind(store, ValMgr.makeLoc(ER), *VI);
   }
@@ -1547,7 +1466,7 @@ Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
   if (!Features.supportsFields())
     return store;
 
-  QualType T = R->getValueType(getContext());
+  QualType T = R->getValueType();
   assert(T->isStructureOrClassType());
 
   const RecordType* RT = T->getAs<RecordType>();
@@ -1560,10 +1479,13 @@ Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
   if (const nonloc::LazyCompoundVal *LCV=dyn_cast<nonloc::LazyCompoundVal>(&V))
     return CopyLazyBindings(*LCV, store, R);
 
-  // We may get non-CompoundVal accidentally due to imprecise cast logic.
-  // Ignore them and kill the field values.
-  if (V.isUnknown() || !isa<nonloc::CompoundVal>(V))
-    return KillStruct(store, R);
+  // We may get non-CompoundVal accidentally due to imprecise cast logic or
+  // that we are binding symbolic struct value. Kill the field values, and if
+  // the value is symbolic go and bind it as a "default" binding.
+  if (V.isUnknown() || !isa<nonloc::CompoundVal>(V)) {
+    SVal SV = isa<nonloc::SymbolVal>(V) ? V : UnknownVal();
+    return KillStruct(store, R, SV);
+  }
 
   nonloc::CompoundVal& CV = cast<nonloc::CompoundVal>(V);
   nonloc::CompoundVal::iterator VI = CV.begin(), VE = CV.end();
@@ -1596,14 +1518,15 @@ Store RegionStoreManager::BindStruct(Store store, const TypedRegion* R,
   return store;
 }
 
-Store RegionStoreManager::KillStruct(Store store, const TypedRegion* R) {
+Store RegionStoreManager::KillStruct(Store store, const TypedRegion* R,
+                                     SVal DefaultVal) {
   RegionBindings B = GetRegionBindings(store);
   llvm::OwningPtr<RegionStoreSubRegionMap>
     SubRegions(getRegionStoreSubRegionMap(store));
   RemoveSubRegionBindings(B, R, *SubRegions);
 
   // Set the default value of the struct region to "unknown".
-  return Add(B, R, BindingKey::Default, UnknownVal()).getRoot();
+  return Add(B, R, BindingKey::Default, DefaultVal).getRoot();
 }
 
 Store RegionStoreManager::CopyLazyBindings(nonloc::LazyCompoundVal V,
@@ -1627,21 +1550,10 @@ Store RegionStoreManager::CopyLazyBindings(nonloc::LazyCompoundVal V,
 // "Raw" retrievals and bindings.
 //===----------------------------------------------------------------------===//
 
-BindingKey BindingKey::Make(const MemRegion *R, Kind k) {
-  if (const ElementRegion *ER = dyn_cast<ElementRegion>(R)) {
-    const RegionRawOffset &O = ER->getAsRawOffset();
-
-    if (O.getRegion())
-      return BindingKey(O.getRegion(), O.getByteOffset(), k);
-
-    // FIXME: There are some ElementRegions for which we cannot compute
-    // raw offsets yet, including regions with symbolic offsets.
-  }
-
-  return BindingKey(R, 0, k);
-}
 
 RegionBindings RegionStoreManager::Add(RegionBindings B, BindingKey K, SVal V) {
+  if (!K.isValid())
+    return B;
   return RBFactory.Add(B, K, V);
 }
 
@@ -1651,6 +1563,8 @@ RegionBindings RegionStoreManager::Add(RegionBindings B, const MemRegion *R,
 }
 
 const SVal *RegionStoreManager::Lookup(RegionBindings B, BindingKey K) {
+  if (!K.isValid())
+    return NULL;
   return B.lookup(K);
 }
 
@@ -1661,6 +1575,8 @@ const SVal *RegionStoreManager::Lookup(RegionBindings B,
 }
 
 RegionBindings RegionStoreManager::Remove(RegionBindings B, BindingKey K) {
+  if (!K.isValid())
+    return B;
   return RBFactory.Remove(B, K);
 }
 
@@ -1669,11 +1585,6 @@ RegionBindings RegionStoreManager::Remove(RegionBindings B, const MemRegion *R,
   return Remove(B, BindingKey::Make(R, k));
 }
 
-Store RegionStoreManager::Remove(Store store, BindingKey K) {
-  RegionBindings B = GetRegionBindings(store);
-  return Remove(B, K).getRoot();
-}
-
 //===----------------------------------------------------------------------===//
 // State pruning.
 //===----------------------------------------------------------------------===//
@@ -1818,12 +1729,12 @@ bool RemoveDeadBindingsWorker::UpdatePostponed() {
   return changed;
 }
 
-const GRState *RegionStoreManager::RemoveDeadBindings(GRState &state,
+Store RegionStoreManager::RemoveDeadBindings(Store store,
                                              const StackFrameContext *LCtx,
                                              SymbolReaper& SymReaper,
                            llvm::SmallVectorImpl<const MemRegion*>& RegionRoots)
 {
-  RegionBindings B = GetRegionBindings(state.getStore());
+  RegionBindings B = GetRegionBindings(store);
   RemoveDeadBindingsWorker W(*this, StateMgr, B, SymReaper, LCtx);
   W.GenerateClusters();
 
@@ -1856,14 +1767,13 @@ const GRState *RegionStoreManager::RemoveDeadBindings(GRState &state,
     for (; SI != SE; ++SI)
       SymReaper.maybeDead(*SI);
   }
-  state.setStore(B.getRoot());
-  const GRState *s = StateMgr.getPersistentState(state);
-  return s;
+
+  return B.getRoot();
 }
 
 
-GRState const *RegionStoreManager::EnterStackFrame(GRState const *state,
-                                               StackFrameContext const *frame) {
+Store RegionStoreManager::EnterStackFrame(const GRState *state,
+                                          const StackFrameContext *frame) {
   FunctionDecl const *FD = cast<FunctionDecl>(frame->getDecl());
   FunctionDecl::param_const_iterator PI = FD->param_begin();
   Store store = state->getStore();
@@ -1887,9 +1797,9 @@ GRState const *RegionStoreManager::EnterStackFrame(GRState const *state,
       store = Bind(store, ValMgr.makeLoc(MRMgr.getVarRegion(*PI,frame)),ArgVal);
     }
   } else
-    assert(0 && "Unhandled call expression.");
+    llvm_unreachable("Unhandled call expression.");
 
-  return state->makeWithStore(store);
+  return store;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Checker/ReturnPointerRangeChecker.cpp b/lib/Checker/ReturnPointerRangeChecker.cpp
index 14edf5668983..a9eb5ce1a700 100644
--- a/lib/Checker/ReturnPointerRangeChecker.cpp
+++ b/lib/Checker/ReturnPointerRangeChecker.cpp
@@ -66,7 +66,7 @@ void ReturnPointerRangeChecker::PreVisitReturnStmt(CheckerContext &C,
 
   DefinedOrUnknownSVal NumElements
     = C.getStoreManager().getSizeInElements(state, ER->getSuperRegion(),
-                                           ER->getValueType(C.getASTContext()));
+                                           ER->getValueType());
 
   const GRState *StInBound = state->AssumeInBound(Idx, NumElements, true);
   const GRState *StOutBound = state->AssumeInBound(Idx, NumElements, false);
diff --git a/lib/Checker/ReturnUndefChecker.cpp b/lib/Checker/ReturnUndefChecker.cpp
index 52a0b3076b6a..73d1890f336f 100644
--- a/lib/Checker/ReturnUndefChecker.cpp
+++ b/lib/Checker/ReturnUndefChecker.cpp
@@ -61,6 +61,7 @@ void ReturnUndefChecker::PreVisitReturnStmt(CheckerContext &C,
   EnhancedBugReport *report = 
     new EnhancedBugReport(*BT, BT->getDescription(), N);
 
+  report->addRange(RetE->getSourceRange());
   report->addVisitorCreator(bugreporter::registerTrackNullOrUndefValue, RetE);
 
   C.EmitReport(report);
diff --git a/lib/Checker/SVals.cpp b/lib/Checker/SVals.cpp
index 7a99e8681df9..97ba74e94878 100644
--- a/lib/Checker/SVals.cpp
+++ b/lib/Checker/SVals.cpp
@@ -62,6 +62,9 @@ const FunctionDecl *SVal::getAsFunctionDecl() const {
 ///  wraps a symbol, return that SymbolRef.  Otherwise return 0.
 // FIXME: should we consider SymbolRef wrapped in CodeTextRegion?
 SymbolRef SVal::getAsLocSymbol() const {
+  if (const nonloc::LocAsInteger *X = dyn_cast<nonloc::LocAsInteger>(this))
+    return X->getLoc().getAsLocSymbol();
+
   if (const loc::MemRegionVal *X = dyn_cast<loc::MemRegionVal>(this)) {
     const MemRegion *R = X->StripCasts();
     if (const SymbolicRegion *SymR = dyn_cast<SymbolicRegion>(R))
@@ -247,8 +250,8 @@ SVal loc::ConcreteInt::EvalBinOp(BasicValueFactory& BasicVals,
                                  BinaryOperator::Opcode Op,
                                  const loc::ConcreteInt& R) const {
 
-  assert (Op == BinaryOperator::Add || Op == BinaryOperator::Sub ||
-          (Op >= BinaryOperator::LT && Op <= BinaryOperator::NE));
+  assert (Op == BO_Add || Op == BO_Sub ||
+          (Op >= BO_LT && Op <= BO_NE));
 
   const llvm::APSInt* X = BasicVals.EvaluateAPSInt(Op, getValue(), R.getValue());
 
diff --git a/lib/Checker/SValuator.cpp b/lib/Checker/SValuator.cpp
index a7e15fc3fd73..273e5742a8e2 100644
--- a/lib/Checker/SValuator.cpp
+++ b/lib/Checker/SValuator.cpp
@@ -37,7 +37,7 @@ SVal SValuator::EvalBinOp(const GRState *ST, BinaryOperator::Opcode Op,
   if (isa<Loc>(R)) {
     // Support pointer arithmetic where the addend is on the left
     // and the pointer on the right.
-    assert(Op == BinaryOperator::Add);
+    assert(Op == BO_Add);
 
     // Commute the operands.
     return EvalBinOpLN(ST, Op, cast<Loc>(R), cast<NonLoc>(L), T);
@@ -49,7 +49,7 @@ SVal SValuator::EvalBinOp(const GRState *ST, BinaryOperator::Opcode Op,
 DefinedOrUnknownSVal SValuator::EvalEQ(const GRState *ST,
                                        DefinedOrUnknownSVal L,
                                        DefinedOrUnknownSVal R) {
-  return cast<DefinedOrUnknownSVal>(EvalBinOp(ST, BinaryOperator::EQ, L, R,
+  return cast<DefinedOrUnknownSVal>(EvalBinOp(ST, BO_EQ, L, R,
                                               ValMgr.getContext().IntTy));
 }
 
diff --git a/lib/Checker/SimpleConstraintManager.cpp b/lib/Checker/SimpleConstraintManager.cpp
index 321381b045ad..04496e1e3332 100644
--- a/lib/Checker/SimpleConstraintManager.cpp
+++ b/lib/Checker/SimpleConstraintManager.cpp
@@ -31,17 +31,17 @@ bool SimpleConstraintManager::canReasonAbout(SVal X) const {
     if (const SymIntExpr *SIE = dyn_cast<SymIntExpr>(SE)) {
       switch (SIE->getOpcode()) {
           // We don't reason yet about bitwise-constraints on symbolic values.
-        case BinaryOperator::And:
-        case BinaryOperator::Or:
-        case BinaryOperator::Xor:
+        case BO_And:
+        case BO_Or:
+        case BO_Xor:
           return false;
         // We don't reason yet about these arithmetic constraints on
         // symbolic values.
-        case BinaryOperator::Mul:
-        case BinaryOperator::Div:
-        case BinaryOperator::Rem:
-        case BinaryOperator::Shl:
-        case BinaryOperator::Shr:
+        case BO_Mul:
+        case BO_Div:
+        case BO_Rem:
+        case BO_Shl:
+        case BO_Shr:
           return false;
         // All other cases.
         default:
@@ -125,12 +125,12 @@ static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   switch (op) {
   default:
     assert(false && "Invalid opcode.");
-  case BinaryOperator::LT: return BinaryOperator::GE;
-  case BinaryOperator::GT: return BinaryOperator::LE;
-  case BinaryOperator::LE: return BinaryOperator::GT;
-  case BinaryOperator::GE: return BinaryOperator::LT;
-  case BinaryOperator::EQ: return BinaryOperator::NE;
-  case BinaryOperator::NE: return BinaryOperator::EQ;
+  case BO_LT: return BO_GE;
+  case BO_GT: return BO_LE;
+  case BO_LE: return BO_GT;
+  case BO_GE: return BO_LT;
+  case BO_EQ: return BO_NE;
+  case BO_NE: return BO_EQ;
   }
 }
 
@@ -178,7 +178,7 @@ const GRState *SimpleConstraintManager::AssumeAux(const GRState *state,
     if (!BinaryOperator::isComparisonOp(op)) {
       QualType T = SymMgr.getType(SE);
       const llvm::APSInt &zero = BasicVals.getValue(0, T);
-      op = (Assumption ? BinaryOperator::NE : BinaryOperator::EQ);
+      op = (Assumption ? BO_NE : BO_EQ);
       return AssumeSymRel(state, SE, op, zero);
     }
 
@@ -238,10 +238,10 @@ const GRState *SimpleConstraintManager::AssumeSymRel(const GRState *state,
 
     // Get the constant out of the expression "($sym+constant1)".
     switch (SE->getOpcode()) {
-    case BinaryOperator::Add:
+    case BO_Add:
       Adjustment = SE->getRHS();
       break;
-    case BinaryOperator::Sub:
+    case BO_Sub:
       Adjustment = -SE->getRHS();
       break;
     default:
@@ -276,48 +276,24 @@ const GRState *SimpleConstraintManager::AssumeSymRel(const GRState *state,
     // No logic yet for other operators.  Assume the constraint is feasible.
     return state;
 
-  case BinaryOperator::EQ:
+  case BO_EQ:
     return AssumeSymEQ(state, Sym, ConvertedInt, Adjustment);
 
-  case BinaryOperator::NE:
+  case BO_NE:
     return AssumeSymNE(state, Sym, ConvertedInt, Adjustment);
 
-  case BinaryOperator::GT:
+  case BO_GT:
     return AssumeSymGT(state, Sym, ConvertedInt, Adjustment);
 
-  case BinaryOperator::GE:
+  case BO_GE:
     return AssumeSymGE(state, Sym, ConvertedInt, Adjustment);
 
-  case BinaryOperator::LT:
+  case BO_LT:
     return AssumeSymLT(state, Sym, ConvertedInt, Adjustment);
 
-  case BinaryOperator::LE:
+  case BO_LE:
     return AssumeSymLE(state, Sym, ConvertedInt, Adjustment);
   } // end switch
 }
 
-const GRState *SimpleConstraintManager::AssumeInBound(const GRState *state,
-                                                      DefinedSVal Idx,
-                                                      DefinedSVal UpperBound,
-                                                      bool Assumption) {
-
-  // Only support ConcreteInt for now.
-  if (!(isa<nonloc::ConcreteInt>(Idx) && isa<nonloc::ConcreteInt>(UpperBound)))
-    return state;
-
-  const llvm::APSInt& Zero = state->getBasicVals().getZeroWithPtrWidth(false);
-  llvm::APSInt IdxV = cast<nonloc::ConcreteInt>(Idx).getValue();
-  // IdxV might be too narrow.
-  if (IdxV.getBitWidth() < Zero.getBitWidth())
-    IdxV.extend(Zero.getBitWidth());
-  // UBV might be too narrow, too.
-  llvm::APSInt UBV = cast<nonloc::ConcreteInt>(UpperBound).getValue();
-  if (UBV.getBitWidth() < Zero.getBitWidth())
-    UBV.extend(Zero.getBitWidth());
-
-  bool InBound = (Zero <= IdxV) && (IdxV < UBV);
-  bool isFeasible = Assumption ? InBound : !InBound;
-  return isFeasible ? state : NULL;
-}
-
 }  // end of namespace clang
diff --git a/lib/Checker/SimpleConstraintManager.h b/lib/Checker/SimpleConstraintManager.h
index 45057e64f31f..96811b3e36e6 100644
--- a/lib/Checker/SimpleConstraintManager.h
+++ b/lib/Checker/SimpleConstraintManager.h
@@ -43,10 +43,6 @@ public:
                               BinaryOperator::Opcode op,
                               const llvm::APSInt& Int);
 
-  const GRState *AssumeInBound(const GRState *state, DefinedSVal Idx,
-                               DefinedSVal UpperBound,
-                               bool Assumption);
-
 protected:
 
   //===------------------------------------------------------------------===//
diff --git a/lib/Checker/SimpleSValuator.cpp b/lib/Checker/SimpleSValuator.cpp
index 3bc4ee7d0613..782cd4f5e68b 100644
--- a/lib/Checker/SimpleSValuator.cpp
+++ b/lib/Checker/SimpleSValuator.cpp
@@ -168,12 +168,12 @@ static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
   switch (op) {
   default:
     assert(false && "Invalid opcode.");
-  case BinaryOperator::LT: return BinaryOperator::GE;
-  case BinaryOperator::GT: return BinaryOperator::LE;
-  case BinaryOperator::LE: return BinaryOperator::GT;
-  case BinaryOperator::GE: return BinaryOperator::LT;
-  case BinaryOperator::EQ: return BinaryOperator::NE;
-  case BinaryOperator::NE: return BinaryOperator::EQ;
+  case BO_LT: return BO_GE;
+  case BO_GT: return BO_LE;
+  case BO_LE: return BO_GT;
+  case BO_GE: return BO_LT;
+  case BO_EQ: return BO_NE;
+  case BO_NE: return BO_EQ;
   }
 }
 
@@ -181,12 +181,12 @@ static BinaryOperator::Opcode ReverseComparison(BinaryOperator::Opcode op) {
   switch (op) {
   default:
     assert(false && "Invalid opcode.");
-  case BinaryOperator::LT: return BinaryOperator::GT;
-  case BinaryOperator::GT: return BinaryOperator::LT;
-  case BinaryOperator::LE: return BinaryOperator::GE;
-  case BinaryOperator::GE: return BinaryOperator::LE;
-  case BinaryOperator::EQ:
-  case BinaryOperator::NE:
+  case BO_LT: return BO_GT;
+  case BO_GT: return BO_LT;
+  case BO_LE: return BO_GE;
+  case BO_GE: return BO_LE;
+  case BO_EQ:
+  case BO_NE:
     return op;
   }
 }
@@ -202,14 +202,14 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
   default:
     // We can't reduce this case; just treat it normally.
     break;
-  case BinaryOperator::Mul:
+  case BO_Mul:
     // a*0 and a*1
     if (RHS == 0)
       return ValMgr.makeIntVal(0, resultTy);
     else if (RHS == 1)
       isIdempotent = true;
     break;
-  case BinaryOperator::Div:
+  case BO_Div:
     // a/0 and a/1
     if (RHS == 0)
       // This is also handled elsewhere.
@@ -217,7 +217,7 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
     else if (RHS == 1)
       isIdempotent = true;
     break;
-  case BinaryOperator::Rem:
+  case BO_Rem:
     // a%0 and a%1
     if (RHS == 0)
       // This is also handled elsewhere.
@@ -225,23 +225,23 @@ SVal SimpleSValuator::MakeSymIntVal(const SymExpr *LHS,
     else if (RHS == 1)
       return ValMgr.makeIntVal(0, resultTy);
     break;
-  case BinaryOperator::Add:
-  case BinaryOperator::Sub:
-  case BinaryOperator::Shl:
-  case BinaryOperator::Shr:
-  case BinaryOperator::Xor:
+  case BO_Add:
+  case BO_Sub:
+  case BO_Shl:
+  case BO_Shr:
+  case BO_Xor:
     // a+0, a-0, a<<0, a>>0, a^0
     if (RHS == 0)
       isIdempotent = true;
     break;
-  case BinaryOperator::And:
+  case BO_And:
     // a&0 and a&(~0)
     if (RHS == 0)
       return ValMgr.makeIntVal(0, resultTy);
     else if (RHS.isAllOnesValue())
       isIdempotent = true;
     break;
-  case BinaryOperator::Or:
+  case BO_Or:
     // a|0 and a|(~0)
     if (RHS == 0)
       isIdempotent = true;
@@ -275,19 +275,19 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
     switch (op) {
       default:
         break;
-      case BinaryOperator::EQ:
-      case BinaryOperator::LE:
-      case BinaryOperator::GE:
+      case BO_EQ:
+      case BO_LE:
+      case BO_GE:
         return ValMgr.makeTruthVal(true, resultTy);
-      case BinaryOperator::LT:
-      case BinaryOperator::GT:
-      case BinaryOperator::NE:
+      case BO_LT:
+      case BO_GT:
+      case BO_NE:
         return ValMgr.makeTruthVal(false, resultTy);
-      case BinaryOperator::Xor:
-      case BinaryOperator::Sub:
+      case BO_Xor:
+      case BO_Sub:
         return ValMgr.makeIntVal(0, resultTy);
-      case BinaryOperator::Or:
-      case BinaryOperator::And:
+      case BO_Or:
+      case BO_And:
         return EvalCastNL(lhs, resultTy);
     }
 
@@ -312,9 +312,9 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
         }
         default:
           switch (op) {
-            case BinaryOperator::EQ:
+            case BO_EQ:
               return ValMgr.makeTruthVal(false, resultTy);
-            case BinaryOperator::NE:
+            case BO_NE:
               return ValMgr.makeTruthVal(true, resultTy);
             default:
               // This case also handles pointer arithmetic.
@@ -333,7 +333,7 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
         return UnknownVal();
 
       // Is this a logical not? (!x is represented as x == 0.)
-      if (op == BinaryOperator::EQ && rhs.isZeroConstant()) {
+      if (op == BO_EQ && rhs.isZeroConstant()) {
         // We know how to negate certain expressions. Simplify them here.
 
         BinaryOperator::Opcode opc = symIntExpr->getOpcode();
@@ -342,34 +342,34 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
           // We don't know how to negate this operation.
           // Just handle it as if it were a normal comparison to 0.
           break;
-        case BinaryOperator::LAnd:
-        case BinaryOperator::LOr:
+        case BO_LAnd:
+        case BO_LOr:
           assert(false && "Logical operators handled by branching logic.");
           return UnknownVal();
-        case BinaryOperator::Assign:
-        case BinaryOperator::MulAssign:
-        case BinaryOperator::DivAssign:
-        case BinaryOperator::RemAssign:
-        case BinaryOperator::AddAssign:
-        case BinaryOperator::SubAssign:
-        case BinaryOperator::ShlAssign:
-        case BinaryOperator::ShrAssign:
-        case BinaryOperator::AndAssign:
-        case BinaryOperator::XorAssign:
-        case BinaryOperator::OrAssign:
-        case BinaryOperator::Comma:
+        case BO_Assign:
+        case BO_MulAssign:
+        case BO_DivAssign:
+        case BO_RemAssign:
+        case BO_AddAssign:
+        case BO_SubAssign:
+        case BO_ShlAssign:
+        case BO_ShrAssign:
+        case BO_AndAssign:
+        case BO_XorAssign:
+        case BO_OrAssign:
+        case BO_Comma:
           assert(false && "'=' and ',' operators handled by GRExprEngine.");
           return UnknownVal();
-        case BinaryOperator::PtrMemD:
-        case BinaryOperator::PtrMemI:
+        case BO_PtrMemD:
+        case BO_PtrMemI:
           assert(false && "Pointer arithmetic not handled here.");
           return UnknownVal();
-        case BinaryOperator::LT:
-        case BinaryOperator::GT:
-        case BinaryOperator::LE:
-        case BinaryOperator::GE:
-        case BinaryOperator::EQ:
-        case BinaryOperator::NE:
+        case BO_LT:
+        case BO_GT:
+        case BO_LE:
+        case BO_GE:
+        case BO_EQ:
+        case BO_NE:
           // Negate the comparison and make a value.
           opc = NegateComparison(opc);
           assert(symIntExpr->getType(ValMgr.getContext()) == resultTy);
@@ -402,9 +402,9 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
 
           const llvm::APSInt *newRHS;
           if (lop == op)
-            newRHS = BVF.EvaluateAPSInt(BinaryOperator::Add, first, second);
+            newRHS = BVF.EvaluateAPSInt(BO_Add, first, second);
           else
-            newRHS = BVF.EvaluateAPSInt(BinaryOperator::Sub, first, second);
+            newRHS = BVF.EvaluateAPSInt(BO_Sub, first, second);
           return MakeSymIntVal(symIntExpr->getLHS(), lop, *newRHS, resultTy);
         }
       }
@@ -429,26 +429,26 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
         lhs = tmp;
 
         switch (op) {
-          case BinaryOperator::LT:
-          case BinaryOperator::GT:
-          case BinaryOperator::LE:
-          case BinaryOperator::GE:
+          case BO_LT:
+          case BO_GT:
+          case BO_LE:
+          case BO_GE:
             op = ReverseComparison(op);
             continue;
-          case BinaryOperator::EQ:
-          case BinaryOperator::NE:
-          case BinaryOperator::Add:
-          case BinaryOperator::Mul:
-          case BinaryOperator::And:
-          case BinaryOperator::Xor:
-          case BinaryOperator::Or:
+          case BO_EQ:
+          case BO_NE:
+          case BO_Add:
+          case BO_Mul:
+          case BO_And:
+          case BO_Xor:
+          case BO_Or:
             continue;
-          case BinaryOperator::Shr:
+          case BO_Shr:
             if (lhsValue.isAllOnesValue() && lhsValue.isSigned())
               // At this point lhs and rhs have been swapped.
               return rhs;
             // FALL-THROUGH
-          case BinaryOperator::Shl:
+          case BO_Shl:
             if (lhsValue == 0)
               // At this point lhs and rhs have been swapped.
               return rhs;
@@ -461,10 +461,12 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
     case nonloc::SymbolValKind: {
       nonloc::SymbolVal *slhs = cast<nonloc::SymbolVal>(&lhs);
       SymbolRef Sym = slhs->getSymbol();
-      
+
+      ASTContext& Ctx = ValMgr.getContext();
+
       // Does the symbol simplify to a constant?  If so, "fold" the constant
       // by setting 'lhs' to a ConcreteInt and try again.
-      if (Sym->getType(ValMgr.getContext())->isIntegerType())
+      if (Sym->getType(Ctx)->isIntegerType())
         if (const llvm::APSInt *Constant = state->getSymVal(Sym)) {
           // The symbol evaluates to a constant. If necessary, promote the
           // folded constant (LHS) to the result type.
@@ -474,7 +476,7 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
           
           // Also promote the RHS (if necessary).
 
-          // For shifts, it necessary promote the RHS to the result type.
+          // For shifts, it is not necessary to promote the RHS.
           if (BinaryOperator::isShiftOp(op))
             continue;
           
@@ -486,7 +488,20 @@ SVal SimpleSValuator::EvalBinOpNN(const GRState *state,
           
           continue;
         }
-      
+
+      // Is the RHS a symbol we can simplify?
+      if (const nonloc::SymbolVal *srhs = dyn_cast<nonloc::SymbolVal>(&rhs)) {
+        SymbolRef RSym = srhs->getSymbol();
+        if (RSym->getType(Ctx)->isIntegerType()) {
+          if (const llvm::APSInt *Constant = state->getSymVal(RSym)) {
+            // The symbol evaluates to a constant.
+            BasicValueFactory &BVF = ValMgr.getBasicValueFactory();
+            const llvm::APSInt &rhs_I = BVF.Convert(resultTy, *Constant);
+            rhs = nonloc::ConcreteInt(rhs_I);
+          }
+        }
+      }
+
       if (isa<nonloc::ConcreteInt>(rhs)) {
         return MakeSymIntVal(slhs->getSymbol(), op,
                              cast<nonloc::ConcreteInt>(rhs).getValue(),
@@ -510,7 +525,7 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
   // calling this function with another operation (PR7527). We don't attempt to
   // model this for now, but it could be useful, particularly when the
   // "location" is actually an integer value that's been passed through a void*.
-  if (!(BinaryOperator::isComparisonOp(op) || op == BinaryOperator::Sub))
+  if (!(BinaryOperator::isComparisonOp(op) || op == BO_Sub))
     return UnknownVal();
 
   // Special cases for when both sides are identical.
@@ -519,15 +534,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
     default:
       assert(false && "Unimplemented operation for two identical values");
       return UnknownVal();
-    case BinaryOperator::Sub:
+    case BO_Sub:
       return ValMgr.makeZeroVal(resultTy);
-    case BinaryOperator::EQ:
-    case BinaryOperator::LE:
-    case BinaryOperator::GE:
+    case BO_EQ:
+    case BO_LE:
+    case BO_GE:
       return ValMgr.makeTruthVal(true, resultTy);
-    case BinaryOperator::NE:
-    case BinaryOperator::LT:
-    case BinaryOperator::GT:
+    case BO_NE:
+    case BO_LT:
+    case BO_GT:
       return ValMgr.makeTruthVal(false, resultTy);
     }
   }
@@ -543,15 +558,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       switch (op) {
       default:
         break;
-      case BinaryOperator::Sub:
+      case BO_Sub:
         return EvalCastL(lhs, resultTy);
-      case BinaryOperator::EQ:
-      case BinaryOperator::LE:
-      case BinaryOperator::LT:
+      case BO_EQ:
+      case BO_LE:
+      case BO_LT:
         return ValMgr.makeTruthVal(false, resultTy);
-      case BinaryOperator::NE:
-      case BinaryOperator::GT:
-      case BinaryOperator::GE:
+      case BO_NE:
+      case BO_GT:
+      case BO_GE:
         return ValMgr.makeTruthVal(true, resultTy);
       }
     }
@@ -594,13 +609,13 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       switch (op) {
       default:
         break;
-      case BinaryOperator::EQ:
-      case BinaryOperator::GT:
-      case BinaryOperator::GE:
+      case BO_EQ:
+      case BO_GT:
+      case BO_GE:
         return ValMgr.makeTruthVal(false, resultTy);
-      case BinaryOperator::NE:
-      case BinaryOperator::LT:
-      case BinaryOperator::LE:
+      case BO_NE:
+      case BO_LT:
+      case BO_LE:
         return ValMgr.makeTruthVal(true, resultTy);
       }
     }
@@ -624,15 +639,15 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
         switch (op) {
         default:
           break;
-        case BinaryOperator::Sub:
+        case BO_Sub:
           return EvalCastL(lhs, resultTy);
-        case BinaryOperator::EQ:
-        case BinaryOperator::LT:
-        case BinaryOperator::LE:
+        case BO_EQ:
+        case BO_LT:
+        case BO_LE:
           return ValMgr.makeTruthVal(false, resultTy);
-        case BinaryOperator::NE:
-        case BinaryOperator::GT:
-        case BinaryOperator::GE:
+        case BO_NE:
+        case BO_GT:
+        case BO_GE:
           return ValMgr.makeTruthVal(true, resultTy);
         }
       }
@@ -660,9 +675,9 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       switch (op) {
       default:
         return UnknownVal();
-      case BinaryOperator::EQ:
+      case BO_EQ:
         return ValMgr.makeTruthVal(false, resultTy);
-      case BinaryOperator::NE:
+      case BO_NE:
         return ValMgr.makeTruthVal(true, resultTy);
       }
     }
@@ -711,8 +726,8 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
       }
 
       // If the element indexes aren't comparable, see if the raw offsets are.
-      RegionRawOffset LeftOffset = LeftER->getAsRawOffset();
-      RegionRawOffset RightOffset = RightER->getAsRawOffset();
+      RegionRawOffset LeftOffset = LeftER->getAsArrayOffset();
+      RegionRawOffset RightOffset = RightER->getAsArrayOffset();
 
       if (LeftOffset.getRegion() != NULL &&
           LeftOffset.getRegion() == RightOffset.getRegion()) {
@@ -722,17 +737,17 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
         switch (op) {
         default:
           return UnknownVal();
-        case BinaryOperator::LT:
+        case BO_LT:
           return ValMgr.makeTruthVal(left < right, resultTy);
-        case BinaryOperator::GT:
+        case BO_GT:
           return ValMgr.makeTruthVal(left > right, resultTy);
-        case BinaryOperator::LE:
+        case BO_LE:
           return ValMgr.makeTruthVal(left <= right, resultTy);
-        case BinaryOperator::GE:
+        case BO_GE:
           return ValMgr.makeTruthVal(left >= right, resultTy);
-        case BinaryOperator::EQ:
+        case BO_EQ:
           return ValMgr.makeTruthVal(left == right, resultTy);
-        case BinaryOperator::NE:
+        case BO_NE:
           return ValMgr.makeTruthVal(left != right, resultTy);
         }
       }
@@ -770,16 +785,16 @@ SVal SimpleSValuator::EvalBinOpLL(const GRState *state,
 
       // We know for sure that the two fields are not the same, since that
       // would have given us the same SVal.
-      if (op == BinaryOperator::EQ)
+      if (op == BO_EQ)
         return ValMgr.makeTruthVal(false, resultTy);
-      if (op == BinaryOperator::NE)
+      if (op == BO_NE)
         return ValMgr.makeTruthVal(true, resultTy);
 
       // Iterate through the fields and see which one comes first.
       // [C99 6.7.2.1.13] "Within a structure object, the non-bit-field
       // members and the units in which bit-fields reside have addresses that
       // increase in the order in which they are declared."
-      bool leftFirst = (op == BinaryOperator::LT || op == BinaryOperator::LE);
+      bool leftFirst = (op == BO_LT || op == BO_LE);
       for (RecordDecl::field_iterator I = RD->field_begin(),
            E = RD->field_end(); I!=E; ++I) {
         if (*I == LeftFD)
@@ -818,8 +833,41 @@ SVal SimpleSValuator::EvalBinOpLN(const GRState *state,
       }
     }
   }
+  
+  // We are dealing with pointer arithmetic.
+
+  // Handle pointer arithmetic on constant values.
+  if (nonloc::ConcreteInt *rhsInt = dyn_cast<nonloc::ConcreteInt>(&rhs)) {
+    if (loc::ConcreteInt *lhsInt = dyn_cast<loc::ConcreteInt>(&lhs)) {
+      const llvm::APSInt &leftI = lhsInt->getValue();
+      assert(leftI.isUnsigned());
+      llvm::APSInt rightI(rhsInt->getValue(), /* isUnsigned */ true);
+
+      // Convert the bitwidth of rightI.  This should deal with overflow
+      // since we are dealing with concrete values.
+      rightI.extOrTrunc(leftI.getBitWidth());
+
+      // Offset the increment by the pointer size.
+      llvm::APSInt Multiplicand(rightI.getBitWidth(), /* isUnsigned */ true);
+      rightI *= Multiplicand;
+      
+      // Compute the adjusted pointer.
+      switch (op) {
+        case BO_Add:
+          rightI = leftI + rightI;
+          break;
+        case BO_Sub:
+          rightI = leftI - rightI;
+          break;
+        default:
+          llvm_unreachable("Invalid pointer arithmetic operation");
+      }
+      return loc::ConcreteInt(ValMgr.getBasicValueFactory().getValue(rightI));
+    }
+  }
+  
 
-  // Delegate pointer arithmetic to the StoreManager.
+  // Delegate remaining pointer arithmetic to the StoreManager.
   return state->getStateManager().getStoreManager().EvalBinOp(op, lhs,
                                                               rhs, resultTy);
 }
diff --git a/lib/Checker/StackAddrLeakChecker.cpp b/lib/Checker/StackAddrLeakChecker.cpp
index f4a9db62df4b..c67a81dced0a 100644
--- a/lib/Checker/StackAddrLeakChecker.cpp
+++ b/lib/Checker/StackAddrLeakChecker.cpp
@@ -108,7 +108,7 @@ void StackAddrLeakChecker::EmitStackError(CheckerContext &C, const MemRegion *R,
     report->addRange(range);
 
   C.EmitReport(report);
-}	
+}
 
 void StackAddrLeakChecker::PreVisitReturnStmt(CheckerContext &C,
                                               const ReturnStmt *RS) {
diff --git a/lib/Checker/Store.cpp b/lib/Checker/Store.cpp
index b12833170506..1cb5cd70cae6 100644
--- a/lib/Checker/Store.cpp
+++ b/lib/Checker/Store.cpp
@@ -21,6 +21,11 @@ StoreManager::StoreManager(GRStateManager &stateMgr)
   : ValMgr(stateMgr.getValueManager()), StateMgr(stateMgr),
     MRMgr(ValMgr.getRegionManager()), Ctx(stateMgr.getContext()) {}
 
+Store StoreManager::EnterStackFrame(const GRState *state,
+                                    const StackFrameContext *frame) {
+  return state->getStore();
+}
+
 const MemRegion *StoreManager::MakeElementRegion(const MemRegion *Base,
                                               QualType EleTy, uint64_t index) {
   SVal idx = ValMgr.makeArrayIndex(index);
@@ -78,7 +83,7 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
   // Handle casts from compatible types.
   if (R->isBoundable())
     if (const TypedRegion *TR = dyn_cast<TypedRegion>(R)) {
-      QualType ObjTy = Ctx.getCanonicalType(TR->getValueType(Ctx));
+      QualType ObjTy = Ctx.getCanonicalType(TR->getValueType());
       if (CanonPointeeTy == ObjTy)
         return R;
     }
@@ -96,17 +101,10 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
       assert(0 && "Invalid region cast");
       break;
     }
-    
+
     case MemRegion::FunctionTextRegionKind:
     case MemRegion::BlockTextRegionKind:
-    case MemRegion::BlockDataRegionKind: {
-      // CodeTextRegion should be cast to only a function or block pointer type,
-      // although they can in practice be casted to anything, e.g, void*, char*,
-      // etc.  
-      // Just return the region.
-      return R;
-    }
-
+    case MemRegion::BlockDataRegionKind:
     case MemRegion::StringRegionKind:
       // FIXME: Need to handle arbitrary downcasts.
     case MemRegion::SymbolicRegionKind:
@@ -139,7 +137,7 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
       // FIXME: Handle symbolic raw offsets.
 
       const ElementRegion *elementR = cast<ElementRegion>(R);
-      const RegionRawOffset &rawOff = elementR->getAsRawOffset();
+      const RegionRawOffset &rawOff = elementR->getAsArrayOffset();
       const MemRegion *baseR = rawOff.getRegion();
 
       // If we cannot compute a raw offset, throw up our hands and return
@@ -154,7 +152,7 @@ const MemRegion *StoreManager::CastRegion(const MemRegion *R, QualType CastToTy)
         // check to see if type we are casting to is the same as the base
         // region.  If so, just return the base region.
         if (const TypedRegion *TR = dyn_cast<TypedRegion>(baseR)) {
-          QualType ObjTy = Ctx.getCanonicalType(TR->getValueType(Ctx));
+          QualType ObjTy = Ctx.getCanonicalType(TR->getValueType());
           QualType CanonPointeeTy = Ctx.getCanonicalType(PointeeTy);
           if (CanonPointeeTy == ObjTy)
             return baseR;
@@ -217,7 +215,7 @@ SVal StoreManager::CastRetrievedVal(SVal V, const TypedRegion *R,
 
   if (performTestOnly) {  
     // Automatically translate references to pointers.
-    QualType T = R->getValueType(Ctx);
+    QualType T = R->getValueType();
     if (const ReferenceType *RT = T->getAs<ReferenceType>())
       T = Ctx.getPointerType(RT->getPointeeType());
     
@@ -279,10 +277,6 @@ SVal StoreManager::getLValueElement(QualType elementType, SVal Offset,
   if (Base.isUnknownOrUndef() || isa<loc::ConcreteInt>(Base))
     return Base;
 
-  // Only handle integer offsets... for now.
-  if (!isa<nonloc::ConcreteInt>(Offset))
-    return UnknownVal();
-
   const MemRegion* BaseRegion = cast<loc::MemRegionVal>(Base).getRegion();
 
   // Pointer of any type can be cast and used as array base.
@@ -311,6 +305,19 @@ SVal StoreManager::getLValueElement(QualType elementType, SVal Offset,
     return UnknownVal();
 
   const llvm::APSInt& BaseIdxI = cast<nonloc::ConcreteInt>(BaseIdx).getValue();
+
+  // Only allow non-integer offsets if the base region has no offset itself.
+  // FIXME: This is a somewhat arbitrary restriction. We should be using
+  // SValuator here to add the two offsets without checking their types.
+  if (!isa<nonloc::ConcreteInt>(Offset)) {
+    if (isa<ElementRegion>(BaseRegion->StripCasts()))
+      return UnknownVal();
+
+    return loc::MemRegionVal(MRMgr.getElementRegion(elementType, Offset,
+                                                    ElemR->getSuperRegion(),
+                                                    Ctx));
+  }
+
   const llvm::APSInt& OffI = cast<nonloc::ConcreteInt>(Offset).getValue();
   assert(BaseIdxI.isSigned());
 
diff --git a/lib/Checker/StreamChecker.cpp b/lib/Checker/StreamChecker.cpp
index c527ca24496f..8553875a24ff 100644
--- a/lib/Checker/StreamChecker.cpp
+++ b/lib/Checker/StreamChecker.cpp
@@ -23,18 +23,49 @@ using namespace clang;
 
 namespace {
 
+struct StreamState {
+  enum Kind { Opened, Closed, OpenFailed, Escaped } K;
+  const Stmt *S;
+
+  StreamState(Kind k, const Stmt *s) : K(k), S(s) {}
+
+  bool isOpened() const { return K == Opened; }
+  bool isClosed() const { return K == Closed; }
+  //bool isOpenFailed() const { return K == OpenFailed; }
+  //bool isEscaped() const { return K == Escaped; }
+
+  bool operator==(const StreamState &X) const {
+    return K == X.K && S == X.S;
+  }
+
+  static StreamState getOpened(const Stmt *s) { return StreamState(Opened, s); }
+  static StreamState getClosed(const Stmt *s) { return StreamState(Closed, s); }
+  static StreamState getOpenFailed(const Stmt *s) { 
+    return StreamState(OpenFailed, s); 
+  }
+  static StreamState getEscaped(const Stmt *s) {
+    return StreamState(Escaped, s);
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    ID.AddInteger(K);
+    ID.AddPointer(S);
+  }
+};
+
 class StreamChecker : public CheckerVisitor<StreamChecker> {
-  IdentifierInfo *II_fopen, *II_fread, *II_fwrite, 
+  IdentifierInfo *II_fopen, *II_tmpfile, *II_fclose, *II_fread, *II_fwrite, 
                  *II_fseek, *II_ftell, *II_rewind, *II_fgetpos, *II_fsetpos,  
                  *II_clearerr, *II_feof, *II_ferror, *II_fileno;
-  BuiltinBug *BT_nullfp, *BT_illegalwhence;
+  BuiltinBug *BT_nullfp, *BT_illegalwhence, *BT_doubleclose, *BT_ResourceLeak;
 
 public:
   StreamChecker() 
-    : II_fopen(0), II_fread(0), II_fwrite(0), 
+    : II_fopen(0), II_tmpfile(0) ,II_fclose(0), II_fread(0), II_fwrite(0), 
       II_fseek(0), II_ftell(0), II_rewind(0), II_fgetpos(0), II_fsetpos(0), 
       II_clearerr(0), II_feof(0), II_ferror(0), II_fileno(0), 
-      BT_nullfp(0), BT_illegalwhence(0) {}
+      BT_nullfp(0), BT_illegalwhence(0), BT_doubleclose(0), 
+      BT_ResourceLeak(0) {}
 
   static void *getTag() {
     static int x;
@@ -42,9 +73,14 @@ public:
   }
 
   virtual bool EvalCallExpr(CheckerContext &C, const CallExpr *CE);
+  void EvalDeadSymbols(CheckerContext &C, SymbolReaper &SymReaper);
+  void EvalEndPath(GREndPathNodeBuilder &B, void *tag, GRExprEngine &Eng);
+  void PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S);
 
 private:
   void Fopen(CheckerContext &C, const CallExpr *CE);
+  void Tmpfile(CheckerContext &C, const CallExpr *CE);
+  void Fclose(CheckerContext &C, const CallExpr *CE);
   void Fread(CheckerContext &C, const CallExpr *CE);
   void Fwrite(CheckerContext &C, const CallExpr *CE);
   void Fseek(CheckerContext &C, const CallExpr *CE);
@@ -56,14 +92,25 @@ private:
   void Feof(CheckerContext &C, const CallExpr *CE);
   void Ferror(CheckerContext &C, const CallExpr *CE);
   void Fileno(CheckerContext &C, const CallExpr *CE);
+
+  void OpenFileAux(CheckerContext &C, const CallExpr *CE);
   
-  // Return true indicates the stream pointer is NULL.
   const GRState *CheckNullStream(SVal SV, const GRState *state, 
                                  CheckerContext &C);
+  const GRState *CheckDoubleClose(const CallExpr *CE, const GRState *state, 
+                                 CheckerContext &C);
 };
 
 } // end anonymous namespace
 
+namespace clang {
+  template <>
+  struct GRStateTrait<StreamState> 
+    : public GRStatePartialTrait<llvm::ImmutableMap<SymbolRef, StreamState> > {
+    static void *GDMIndex() { return StreamChecker::getTag(); }
+  };
+}
+
 void clang::RegisterStreamChecker(GRExprEngine &Eng) {
   Eng.registerCheck(new StreamChecker());
 }
@@ -79,6 +126,10 @@ bool StreamChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
   ASTContext &Ctx = C.getASTContext();
   if (!II_fopen)
     II_fopen = &Ctx.Idents.get("fopen");
+  if (!II_tmpfile)
+    II_tmpfile = &Ctx.Idents.get("tmpfile");
+  if (!II_fclose)
+    II_fclose = &Ctx.Idents.get("fclose");
   if (!II_fread)
     II_fread = &Ctx.Idents.get("fread");
   if (!II_fwrite)
@@ -106,6 +157,14 @@ bool StreamChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
     Fopen(C, CE);
     return true;
   }
+  if (FD->getIdentifier() == II_tmpfile) {
+    Tmpfile(C, CE);
+    return true;
+  }
+  if (FD->getIdentifier() == II_fclose) {
+    Fclose(C, CE);
+    return true;
+  }
   if (FD->getIdentifier() == II_fread) {
     Fread(C, CE);
     return true;
@@ -155,21 +214,43 @@ bool StreamChecker::EvalCallExpr(CheckerContext &C, const CallExpr *CE) {
 }
 
 void StreamChecker::Fopen(CheckerContext &C, const CallExpr *CE) {
+  OpenFileAux(C, CE);
+}
+
+void StreamChecker::Tmpfile(CheckerContext &C, const CallExpr *CE) {
+  OpenFileAux(C, CE);
+}
+
+void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) {
   const GRState *state = C.getState();
   unsigned Count = C.getNodeBuilder().getCurrentBlockCount();
   ValueManager &ValMgr = C.getValueManager();
   DefinedSVal RetVal = cast<DefinedSVal>(ValMgr.getConjuredSymbolVal(0, CE, 
                                                                      Count));
   state = state->BindExpr(CE, RetVal);
-
+  
   ConstraintManager &CM = C.getConstraintManager();
   // Bifurcate the state into two: one with a valid FILE* pointer, the other
   // with a NULL.
   const GRState *stateNotNull, *stateNull;
   llvm::tie(stateNotNull, stateNull) = CM.AssumeDual(state, RetVal);
+  
+  if (SymbolRef Sym = RetVal.getAsSymbol()) {
+    // if RetVal is not NULL, set the symbol's state to Opened.
+    stateNotNull =
+      stateNotNull->set<StreamState>(Sym,StreamState::getOpened(CE));
+    stateNull =
+      stateNull->set<StreamState>(Sym, StreamState::getOpenFailed(CE));
+
+    C.addTransition(stateNotNull);
+    C.addTransition(stateNull);
+  }
+}
 
-  C.addTransition(stateNotNull);
-  C.addTransition(stateNull);
+void StreamChecker::Fclose(CheckerContext &C, const CallExpr *CE) {
+  const GRState *state = CheckDoubleClose(CE, C.getState(), C);
+  if (state)
+    C.addTransition(state);
 }
 
 void StreamChecker::Fread(CheckerContext &C, const CallExpr *CE) {
@@ -285,3 +366,103 @@ const GRState *StreamChecker::CheckNullStream(SVal SV, const GRState *state,
   }
   return stateNotNull;
 }
+
+const GRState *StreamChecker::CheckDoubleClose(const CallExpr *CE,
+                                               const GRState *state,
+                                               CheckerContext &C) {
+  SymbolRef Sym = state->getSVal(CE->getArg(0)).getAsSymbol();
+  if (!Sym)
+    return state;
+  
+  const StreamState *SS = state->get<StreamState>(Sym);
+
+  // If the file stream is not tracked, return.
+  if (!SS)
+    return state;
+  
+  // Check: Double close a File Descriptor could cause undefined behaviour.
+  // Conforming to man-pages
+  if (SS->isClosed()) {
+    ExplodedNode *N = C.GenerateSink();
+    if (N) {
+      if (!BT_doubleclose)
+        BT_doubleclose = new BuiltinBug("Double fclose",
+                                        "Try to close a file Descriptor already"
+                                        " closed. Cause undefined behaviour.");
+      BugReport *R = new BugReport(*BT_doubleclose,
+                                   BT_doubleclose->getDescription(), N);
+      C.EmitReport(R);
+    }
+    return NULL;
+  }
+  
+  // Close the File Descriptor.
+  return state->set<StreamState>(Sym, StreamState::getClosed(CE));
+}
+
+void StreamChecker::EvalDeadSymbols(CheckerContext &C,SymbolReaper &SymReaper) {
+  for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
+         E = SymReaper.dead_end(); I != E; ++I) {
+    SymbolRef Sym = *I;
+    const GRState *state = C.getState();
+    const StreamState *SS = state->get<StreamState>(Sym);
+    if (!SS)
+      return;
+
+    if (SS->isOpened()) {
+      ExplodedNode *N = C.GenerateSink();
+      if (N) {
+        if (!BT_ResourceLeak)
+          BT_ResourceLeak = new BuiltinBug("Resource Leak", 
+                          "Opened File never closed. Potential Resource leak.");
+        BugReport *R = new BugReport(*BT_ResourceLeak, 
+                                     BT_ResourceLeak->getDescription(), N);
+        C.EmitReport(R);
+      }
+    }
+  }
+}
+
+void StreamChecker::EvalEndPath(GREndPathNodeBuilder &B, void *tag,
+                                GRExprEngine &Eng) {
+  SaveAndRestore<bool> OldHasGen(B.HasGeneratedNode);
+  const GRState *state = B.getState();
+  typedef llvm::ImmutableMap<SymbolRef, StreamState> SymMap;
+  SymMap M = state->get<StreamState>();
+  
+  for (SymMap::iterator I = M.begin(), E = M.end(); I != E; ++I) {
+    StreamState SS = I->second;
+    if (SS.isOpened()) {
+      ExplodedNode *N = B.generateNode(state, tag, B.getPredecessor());
+      if (N) {
+        if (!BT_ResourceLeak)
+          BT_ResourceLeak = new BuiltinBug("Resource Leak", 
+                          "Opened File never closed. Potential Resource leak.");
+        BugReport *R = new BugReport(*BT_ResourceLeak, 
+                                     BT_ResourceLeak->getDescription(), N);
+        Eng.getBugReporter().EmitReport(R);
+      }
+    }
+  }
+}
+
+void StreamChecker::PreVisitReturnStmt(CheckerContext &C, const ReturnStmt *S) {
+  const Expr *RetE = S->getRetValue();
+  if (!RetE)
+    return;
+  
+  const GRState *state = C.getState();
+  SymbolRef Sym = state->getSVal(RetE).getAsSymbol();
+  
+  if (!Sym)
+    return;
+  
+  const StreamState *SS = state->get<StreamState>(Sym);
+  if(!SS)
+    return;
+
+  if (SS->isOpened())
+    state = state->set<StreamState>(Sym, StreamState::getEscaped(S));
+
+  C.addTransition(state);
+}
diff --git a/lib/Checker/SymbolManager.cpp b/lib/Checker/SymbolManager.cpp
index c2b557ea57db..3b1bb6d98d95 100644
--- a/lib/Checker/SymbolManager.cpp
+++ b/lib/Checker/SymbolManager.cpp
@@ -28,22 +28,22 @@ static void print(llvm::raw_ostream& os, BinaryOperator::Opcode Op) {
     default:
       assert(false && "operator printing not implemented");
       break;
-    case BinaryOperator::Mul: os << '*'  ; break;
-    case BinaryOperator::Div: os << '/'  ; break;
-    case BinaryOperator::Rem: os << '%'  ; break;
-    case BinaryOperator::Add: os << '+'  ; break;
-    case BinaryOperator::Sub: os << '-'  ; break;
-    case BinaryOperator::Shl: os << "<<" ; break;
-    case BinaryOperator::Shr: os << ">>" ; break;
-    case BinaryOperator::LT:  os << "<"  ; break;
-    case BinaryOperator::GT:  os << '>'  ; break;
-    case BinaryOperator::LE:  os << "<=" ; break;
-    case BinaryOperator::GE:  os << ">=" ; break;
-    case BinaryOperator::EQ:  os << "==" ; break;
-    case BinaryOperator::NE:  os << "!=" ; break;
-    case BinaryOperator::And: os << '&'  ; break;
-    case BinaryOperator::Xor: os << '^'  ; break;
-    case BinaryOperator::Or:  os << '|'  ; break;
+    case BO_Mul: os << '*'  ; break;
+    case BO_Div: os << '/'  ; break;
+    case BO_Rem: os << '%'  ; break;
+    case BO_Add: os << '+'  ; break;
+    case BO_Sub: os << '-'  ; break;
+    case BO_Shl: os << "<<" ; break;
+    case BO_Shr: os << ">>" ; break;
+    case BO_LT:  os << "<"  ; break;
+    case BO_GT:  os << '>'  ; break;
+    case BO_LE:  os << "<=" ; break;
+    case BO_GE:  os << ">=" ; break;
+    case BO_EQ:  os << "==" ; break;
+    case BO_NE:  os << "!=" ; break;
+    case BO_And: os << '&'  ; break;
+    case BO_Xor: os << '^'  ; break;
+    case BO_Or:  os << '|'  ; break;
   }
 }
 
@@ -78,6 +78,11 @@ void SymbolExtent::dumpToStream(llvm::raw_ostream& os) const {
   os << "extent_$" << getSymbolID() << '{' << getRegion() << '}';
 }
 
+void SymbolMetadata::dumpToStream(llvm::raw_ostream& os) const {
+  os << "meta_$" << getSymbolID() << '{'
+     << getRegion() << ',' << T.getAsString() << '}';
+}
+
 void SymbolRegionValue::dumpToStream(llvm::raw_ostream& os) const {
   os << "reg_$" << getSymbolID() << "<" << R << ">";
 }
@@ -150,6 +155,24 @@ SymbolManager::getExtentSymbol(const SubRegion *R) {
   return cast<SymbolExtent>(SD);
 }
 
+const SymbolMetadata*
+SymbolManager::getMetadataSymbol(const MemRegion* R, const Stmt* S, QualType T,
+                                 unsigned Count, const void* SymbolTag) {
+
+  llvm::FoldingSetNodeID profile;
+  SymbolMetadata::Profile(profile, R, S, T, Count, SymbolTag);
+  void* InsertPos;
+  SymExpr *SD = DataSet.FindNodeOrInsertPos(profile, InsertPos);
+  if (!SD) {
+    SD = (SymExpr*) BPAlloc.Allocate<SymbolMetadata>();
+    new (SD) SymbolMetadata(SymbolCounter, R, S, T, Count, SymbolTag);
+    DataSet.InsertNode(SD, InsertPos);
+    ++SymbolCounter;
+  }
+
+  return cast<SymbolMetadata>(SD);
+}
+
 const SymIntExpr *SymbolManager::getSymIntExpr(const SymExpr *lhs,
                                                BinaryOperator::Opcode op,
                                                const llvm::APSInt& v,
@@ -191,21 +214,34 @@ QualType SymbolConjured::getType(ASTContext&) const {
 }
 
 QualType SymbolDerived::getType(ASTContext& Ctx) const {
-  return R->getValueType(Ctx);
+  return R->getValueType();
 }
 
 QualType SymbolExtent::getType(ASTContext& Ctx) const {
   return Ctx.getSizeType();
 }
 
+QualType SymbolMetadata::getType(ASTContext&) const {
+  return T;
+}
+
 QualType SymbolRegionValue::getType(ASTContext& C) const {
-  return R->getValueType(C);
+  return R->getValueType();
 }
 
 SymbolManager::~SymbolManager() {}
 
 bool SymbolManager::canSymbolicate(QualType T) {
-  return Loc::IsLocType(T) || (T->isIntegerType() && T->isScalarType());
+  if (Loc::IsLocType(T))
+    return true;
+
+  if (T->isIntegerType())
+    return T->isScalarType();
+
+  if (T->isRecordType())
+    return true;
+
+  return false;
 }
 
 void SymbolReaper::markLive(SymbolRef sym) {
@@ -213,6 +249,11 @@ void SymbolReaper::markLive(SymbolRef sym) {
   TheDead.erase(sym);
 }
 
+void SymbolReaper::markInUse(SymbolRef sym) {
+  if (isa<SymbolMetadata>(sym))
+    MetadataInUse.insert(sym);
+}
+
 bool SymbolReaper::maybeDead(SymbolRef sym) {
   if (isLive(sym))
     return false;
@@ -221,6 +262,31 @@ bool SymbolReaper::maybeDead(SymbolRef sym) {
   return true;
 }
 
+static bool IsLiveRegion(SymbolReaper &Reaper, const MemRegion *MR) {
+  MR = MR->getBaseRegion();
+
+  if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(MR))
+    return Reaper.isLive(SR->getSymbol());
+
+  if (const VarRegion *VR = dyn_cast<VarRegion>(MR))
+    return Reaper.isLive(VR);
+
+  // FIXME: This is a gross over-approximation. What we really need is a way to
+  // tell if anything still refers to this region. Unlike SymbolicRegions,
+  // AllocaRegions don't have associated symbols, though, so we don't actually
+  // have a way to track their liveness.
+  if (isa<AllocaRegion>(MR))
+    return true;
+
+  if (isa<CXXThisRegion>(MR))
+    return true;
+
+  if (isa<MemSpaceRegion>(MR))
+    return true;
+
+  return false;
+}
+
 bool SymbolReaper::isLive(SymbolRef sym) {
   if (TheLiving.count(sym))
     return true;
@@ -234,11 +300,21 @@ bool SymbolReaper::isLive(SymbolRef sym) {
   }
 
   if (const SymbolExtent *extent = dyn_cast<SymbolExtent>(sym)) {
-    const MemRegion *Base = extent->getRegion()->getBaseRegion();
-    if (const VarRegion *VR = dyn_cast<VarRegion>(Base))
-      return isLive(VR);
-    if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(Base))
-      return isLive(SR->getSymbol());
+    if (IsLiveRegion(*this, extent->getRegion())) {
+      markLive(sym);
+      return true;
+    }
+    return false;
+  }
+
+  if (const SymbolMetadata *metadata = dyn_cast<SymbolMetadata>(sym)) {
+    if (MetadataInUse.count(sym)) {
+      if (IsLiveRegion(*this, metadata->getRegion())) {
+        markLive(sym);
+        MetadataInUse.erase(sym);
+        return true;
+      }
+    }
     return false;
   }
 
@@ -248,16 +324,19 @@ bool SymbolReaper::isLive(SymbolRef sym) {
 }
 
 bool SymbolReaper::isLive(const Stmt* ExprVal) const {
-  return LCtx->getLiveVariables()->isLive(Loc, ExprVal);
+  return LCtx->getAnalysisContext()->getRelaxedLiveVariables()->
+      isLive(Loc, ExprVal);
 }
 
 bool SymbolReaper::isLive(const VarRegion *VR) const {
-  const StackFrameContext *SFC = VR->getStackFrame();
+  const StackFrameContext *VarContext = VR->getStackFrame();
+  const StackFrameContext *CurrentContext = LCtx->getCurrentStackFrame();
+
+  if (VarContext == CurrentContext)
+    return LCtx->getAnalysisContext()->getRelaxedLiveVariables()->
+        isLive(Loc, VR->getDecl());
 
-  if (SFC == LCtx->getCurrentStackFrame())
-    return LCtx->getLiveVariables()->isLive(Loc, VR->getDecl());
-  else
-    return SFC->isParentOf(LCtx->getCurrentStackFrame());
+  return VarContext->isParentOf(CurrentContext);
 }
 
 SymbolVisitor::~SymbolVisitor() {}
diff --git a/lib/Checker/UndefBranchChecker.cpp b/lib/Checker/UndefBranchChecker.cpp
index 90883456b17c..1ff0641be5df 100644
--- a/lib/Checker/UndefBranchChecker.cpp
+++ b/lib/Checker/UndefBranchChecker.cpp
@@ -29,27 +29,28 @@ class UndefBranchChecker : public Checker {
 
     FindUndefExpr(GRStateManager& V, const GRState* S) : VM(V), St(S) {}
 
-    Expr* FindExpr(Expr* Ex) {
+    const Expr* FindExpr(const Expr* Ex) {
       if (!MatchesCriteria(Ex))
         return 0;
 
-      for (Stmt::child_iterator I=Ex->child_begin(), E=Ex->child_end();I!=E;++I)
-        if (Expr* ExI = dyn_cast_or_null<Expr>(*I)) {
-          Expr* E2 = FindExpr(ExI);
+      for (Stmt::const_child_iterator I = Ex->child_begin(), 
+                                      E = Ex->child_end();I!=E;++I)
+        if (const Expr* ExI = dyn_cast_or_null<Expr>(*I)) {
+          const Expr* E2 = FindExpr(ExI);
           if (E2) return E2;
         }
 
       return Ex;
     }
 
-    bool MatchesCriteria(Expr* Ex) { return St->getSVal(Ex).isUndef(); }
+    bool MatchesCriteria(const Expr* Ex) { return St->getSVal(Ex).isUndef(); }
   };
 
 public:
   UndefBranchChecker() : BT(0) {}
   static void *getTag();
   void VisitBranchCondition(GRBranchNodeBuilder &Builder, GRExprEngine &Eng,
-                            Stmt *Condition, void *tag);
+                            const Stmt *Condition, void *tag);
 };
 
 }
@@ -65,7 +66,7 @@ void *UndefBranchChecker::getTag() {
 
 void UndefBranchChecker::VisitBranchCondition(GRBranchNodeBuilder &Builder, 
                                               GRExprEngine &Eng,
-                                              Stmt *Condition, void *tag) {
+                                              const Stmt *Condition, void *tag){
   const GRState *state = Builder.getState();
   SVal X = state->getSVal(Condition);
   if (X.isUndef()) {
@@ -81,7 +82,7 @@ void UndefBranchChecker::VisitBranchCondition(GRBranchNodeBuilder &Builder,
       // subexpressions and roughly look for the most nested subexpression
       // that binds to Undefined.  We then highlight that expression's range.
       BlockEdge B = cast<BlockEdge>(N->getLocation());
-      Expr* Ex = cast<Expr>(B.getSrc()->getTerminatorCondition());
+      const Expr* Ex = cast<Expr>(B.getSrc()->getTerminatorCondition());
       assert (Ex && "Block must have a terminator.");
 
       // Get the predecessor node and check if is a PostStmt with the Stmt
diff --git a/lib/Checker/UndefinedAssignmentChecker.cpp b/lib/Checker/UndefinedAssignmentChecker.cpp
index 6cef60eaee29..ccc97489e631 100644
--- a/lib/Checker/UndefinedAssignmentChecker.cpp
+++ b/lib/Checker/UndefinedAssignmentChecker.cpp
@@ -25,9 +25,8 @@ class UndefinedAssignmentChecker
 public:
   UndefinedAssignmentChecker() : BT(0) {}
   static void *getTag();
-  virtual void PreVisitBind(CheckerContext &C, const Stmt *AssignE,
-                            const Stmt *StoreE, SVal location,
-                            SVal val);
+  virtual void PreVisitBind(CheckerContext &C, const Stmt *StoreE,
+                            SVal location, SVal val);
 };
 }
 
@@ -41,7 +40,6 @@ void *UndefinedAssignmentChecker::getTag() {
 }
 
 void UndefinedAssignmentChecker::PreVisitBind(CheckerContext &C,
-                                              const Stmt *AssignE,
                                               const Stmt *StoreE,
                                               SVal location,
                                               SVal val) {
@@ -61,8 +59,8 @@ void UndefinedAssignmentChecker::PreVisitBind(CheckerContext &C,
   // Generate a report for this bug.
   const Expr *ex = 0;
 
-  while (AssignE) {
-    if (const BinaryOperator *B = dyn_cast<BinaryOperator>(AssignE)) {
+  while (StoreE) {
+    if (const BinaryOperator *B = dyn_cast<BinaryOperator>(StoreE)) {
       if (B->isCompoundAssignmentOp()) {
         const GRState *state = C.getState();
         if (state->getSVal(B->getLHS()).isUndef()) {
@@ -77,7 +75,7 @@ void UndefinedAssignmentChecker::PreVisitBind(CheckerContext &C,
       break;
     }
 
-    if (const DeclStmt *DS = dyn_cast<DeclStmt>(AssignE)) {
+    if (const DeclStmt *DS = dyn_cast<DeclStmt>(StoreE)) {
       const VarDecl* VD = dyn_cast<VarDecl>(DS->getSingleDecl());
       ex = VD->getInit();
     }
diff --git a/lib/Checker/UnixAPIChecker.cpp b/lib/Checker/UnixAPIChecker.cpp
index e9b8f0966ae1..de7346d6273a 100644
--- a/lib/Checker/UnixAPIChecker.cpp
+++ b/lib/Checker/UnixAPIChecker.cpp
@@ -100,7 +100,7 @@ static void CheckOpen(CheckerContext &C, UnixAPIChecker &UC,
   NonLoc ocreateFlag =
     cast<NonLoc>(C.getValueManager().makeIntVal(UC.Val_O_CREAT.getValue(),
                                                 oflagsEx->getType()));
-  SVal maskedFlagsUC = C.getSValuator().EvalBinOpNN(state, BinaryOperator::And,
+  SVal maskedFlagsUC = C.getSValuator().EvalBinOpNN(state, BO_And,
                                                     oflags, ocreateFlag,
                                                     oflagsEx->getType());
   if (maskedFlagsUC.isUnknownOrUndef())
diff --git a/lib/Checker/UnreachableCodeChecker.cpp b/lib/Checker/UnreachableCodeChecker.cpp
new file mode 100644
index 000000000000..7a56c7f46b48
--- /dev/null
+++ b/lib/Checker/UnreachableCodeChecker.cpp
@@ -0,0 +1,226 @@
+//==- UnreachableCodeChecker.cpp - Generalized dead code checker -*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// This file implements a generalized unreachable code checker using a
+// path-sensitive analysis. We mark any path visited, and then walk the CFG as a
+// post-analysis to determine what was never visited.
+//
+// A similar flow-sensitive only check exists in Analysis/ReachableCode.cpp
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ParentMap.h"
+#include "clang/Basic/Builtins.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Checker/PathSensitive/CheckerVisitor.h"
+#include "clang/Checker/PathSensitive/ExplodedGraph.h"
+#include "clang/Checker/PathSensitive/SVals.h"
+#include "clang/Checker/PathSensitive/CheckerHelpers.h"
+#include "clang/Checker/BugReporter/BugReporter.h"
+#include "GRExprEngineExperimentalChecks.h"
+#include "llvm/ADT/SmallPtrSet.h"
+
+// The number of CFGBlock pointers we want to reserve memory for. This is used
+// once for each function we analyze.
+#define DEFAULT_CFGBLOCKS 256
+
+using namespace clang;
+
+namespace {
+class UnreachableCodeChecker : public CheckerVisitor<UnreachableCodeChecker> {
+public:
+  static void *getTag();
+  void VisitEndAnalysis(ExplodedGraph &G,
+                        BugReporter &B,
+                        GRExprEngine &Eng);
+private:
+  static inline const Stmt *getUnreachableStmt(const CFGBlock *CB);
+  void FindUnreachableEntryPoints(const CFGBlock *CB);
+  static bool isInvalidPath(const CFGBlock *CB, const ParentMap &PM);
+  static inline bool isEmptyCFGBlock(const CFGBlock *CB);
+
+  llvm::SmallSet<unsigned, DEFAULT_CFGBLOCKS> reachable;
+  llvm::SmallSet<unsigned, DEFAULT_CFGBLOCKS> visited;
+};
+}
+
+void *UnreachableCodeChecker::getTag() {
+  static int x = 0;
+  return &x;
+}
+
+void clang::RegisterUnreachableCodeChecker(GRExprEngine &Eng) {
+  Eng.registerCheck(new UnreachableCodeChecker());
+}
+
+void UnreachableCodeChecker::VisitEndAnalysis(ExplodedGraph &G,
+                                              BugReporter &B,
+                                              GRExprEngine &Eng) {
+  // Bail out if we didn't cover all paths
+  if (Eng.hasWorkRemaining())
+    return;
+
+  CFG *C = 0;
+  ParentMap *PM = 0;
+  // Iterate over ExplodedGraph
+  for (ExplodedGraph::node_iterator I = G.nodes_begin(), E = G.nodes_end();
+      I != E; ++I) {
+    const ProgramPoint &P = I->getLocation();
+    const LocationContext *LC = P.getLocationContext();
+
+    // Save the CFG if we don't have it already
+    if (!C)
+      C = LC->getAnalysisContext()->getUnoptimizedCFG();
+    if (!PM)
+      PM = &LC->getParentMap();
+
+    if (const BlockEntrance *BE = dyn_cast<BlockEntrance>(&P)) {
+      const CFGBlock *CB = BE->getBlock();
+      reachable.insert(CB->getBlockID());
+    }
+  }
+
+  // Bail out if we didn't get the CFG or the ParentMap.
+  if (!C || !PM)
+    return;
+
+  ASTContext &Ctx = B.getContext();
+
+  // Find CFGBlocks that were not covered by any node
+  for (CFG::const_iterator I = C->begin(); I != C->end(); ++I) {
+    const CFGBlock *CB = *I;
+    // Check if the block is unreachable
+    if (reachable.count(CB->getBlockID()))
+      continue;
+
+    // Check if the block is empty (an artificial block)
+    if (isEmptyCFGBlock(CB))
+      continue;
+
+    // Find the entry points for this block
+    FindUnreachableEntryPoints(CB);
+
+    // This block may have been pruned; check if we still want to report it
+    if (reachable.count(CB->getBlockID()))
+      continue;
+
+    // Check for false positives
+    if (CB->size() > 0 && isInvalidPath(CB, *PM))
+      continue;
+
+    // Special case for __builtin_unreachable.
+    // FIXME: This should be extended to include other unreachable markers,
+    // such as llvm_unreachable.
+    if (!CB->empty()) {
+      const Stmt *First = CB->front();
+      if (const CallExpr *CE = dyn_cast<CallExpr>(First)) {
+        if (CE->isBuiltinCall(Ctx) == Builtin::BI__builtin_unreachable)
+          continue;
+      }
+    }
+
+    // We found a block that wasn't covered - find the statement to report
+    SourceRange SR;
+    SourceLocation SL;
+    if (const Stmt *S = getUnreachableStmt(CB)) {
+      SR = S->getSourceRange();
+      SL = S->getLocStart();
+      if (SR.isInvalid() || SL.isInvalid())
+        continue;
+    }
+    else
+      continue;
+
+    // Check if the SourceLocation is in a system header
+    const SourceManager &SM = B.getSourceManager();
+    if (SM.isInSystemHeader(SL) || SM.isInExternCSystemHeader(SL))
+      continue;
+
+    B.EmitBasicReport("Unreachable code", "Dead code", "This statement is never"
+        " executed", SL, SR);
+  }
+}
+
+// Recursively finds the entry point(s) for this dead CFGBlock.
+void UnreachableCodeChecker::FindUnreachableEntryPoints(const CFGBlock *CB) {
+  bool allPredecessorsReachable = true;
+
+  visited.insert(CB->getBlockID());
+
+  for (CFGBlock::const_pred_iterator I = CB->pred_begin(); I != CB->pred_end();
+      ++I) {
+    // Recurse over all unreachable blocks
+    if (!reachable.count((*I)->getBlockID())) {
+      // At least one predeccessor was unreachable
+      allPredecessorsReachable = false;
+
+      // Only visit the block once
+      if (!visited.count((*I)->getBlockID()))
+        FindUnreachableEntryPoints(*I);
+    }
+  }
+
+  // If at least one predecessor is unreachable, mark this block as reachable
+  // so we don't report this block.
+  if (!allPredecessorsReachable) {
+    reachable.insert(CB->getBlockID());
+  }
+}
+
+// Find the Stmt* in a CFGBlock for reporting a warning
+const Stmt *UnreachableCodeChecker::getUnreachableStmt(const CFGBlock *CB) {
+  if (CB->size() > 0)
+    return CB->front().getStmt();
+  else if (const Stmt *S = CB->getTerminator())
+    return S;
+  else
+    return 0;
+}
+
+// Determines if the path to this CFGBlock contained an element that infers this
+// block is a false positive. We assume that FindUnreachableEntryPoints has
+// already marked only the entry points to any dead code, so we need only to
+// find the condition that led to this block (the predecessor of this block.)
+// There will never be more than one predecessor.
+bool UnreachableCodeChecker::isInvalidPath(const CFGBlock *CB,
+                                           const ParentMap &PM) {
+  // We only expect a predecessor size of 0 or 1. If it is >1, then an external
+  // condition has broken our assumption (for example, a sink being placed by
+  // another check). In these cases, we choose not to report.
+  if (CB->pred_size() > 1)
+    return true;
+
+  // If there are no predecessors, then this block is trivially unreachable
+  if (CB->pred_size() == 0)
+    return false;
+
+  const CFGBlock *pred = *CB->pred_begin();
+
+  // Get the predecessor block's terminator conditon
+  const Stmt *cond = pred->getTerminatorCondition();
+
+  //assert(cond && "CFGBlock's predecessor has a terminator condition");
+  // The previous assertion is invalid in some cases (eg do/while). Leaving
+  // reporting of these situations on at the moment to help triage these cases.
+  if (!cond)
+    return false;
+
+  // Run each of the checks on the conditions
+  if (containsMacro(cond) || containsEnum(cond)
+      || containsStaticLocal(cond) || containsBuiltinOffsetOf(cond)
+      || containsStmt<SizeOfAlignOfExpr>(cond))
+    return true;
+
+  return false;
+}
+
+// Returns true if the given CFGBlock is empty
+bool UnreachableCodeChecker::isEmptyCFGBlock(const CFGBlock *CB) {
+  return CB->getLabel() == 0       // No labels
+      && CB->size() == 0           // No statements
+      && CB->getTerminator() == 0; // No terminator
+}
diff --git a/lib/Checker/VLASizeChecker.cpp b/lib/Checker/VLASizeChecker.cpp
index 936991d6133c..0800b8be1881 100644
--- a/lib/Checker/VLASizeChecker.cpp
+++ b/lib/Checker/VLASizeChecker.cpp
@@ -117,7 +117,7 @@ void VLASizeChecker::PreVisitDeclStmt(CheckerContext &C, const DeclStmt *DS) {
   SVal EleSizeVal = ValMgr.makeIntVal(EleSize.getQuantity(), SizeTy);
 
   // Multiply the array length by the element size.
-  SVal ArraySizeVal = SV.EvalBinOpNN(state, BinaryOperator::Mul, ArrayLength,
+  SVal ArraySizeVal = SV.EvalBinOpNN(state, BO_Mul, ArrayLength,
                                      cast<NonLoc>(EleSizeVal), SizeTy);
 
   // Finally, Assume that the array's extent matches the given size.
diff --git a/lib/Checker/ValueManager.cpp b/lib/Checker/ValueManager.cpp
index aa0c3c877dde..8b7cd7bbdb68 100644
--- a/lib/Checker/ValueManager.cpp
+++ b/lib/Checker/ValueManager.cpp
@@ -72,7 +72,7 @@ SVal ValueManager::convertToArrayIndex(SVal V) {
 
 DefinedOrUnknownSVal 
 ValueManager::getRegionValueSymbolVal(const TypedRegion* R) {
-  QualType T = R->getValueType(SymMgr.getContext());
+  QualType T = R->getValueType();
 
   if (!SymbolManager::canSymbolicate(T))
     return UnknownVal();
@@ -117,11 +117,24 @@ DefinedOrUnknownSVal ValueManager::getConjuredSymbolVal(const void *SymbolTag,
   return nonloc::SymbolVal(sym);
 }
 
+DefinedSVal ValueManager::getMetadataSymbolVal(const void *SymbolTag,
+                                               const MemRegion *MR,
+                                               const Expr *E, QualType T,
+                                               unsigned Count) {
+  assert(SymbolManager::canSymbolicate(T) && "Invalid metadata symbol type");
+
+  SymbolRef sym = SymMgr.getMetadataSymbol(MR, E, T, Count, SymbolTag);
+
+  if (Loc::IsLocType(T))
+    return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
+
+  return nonloc::SymbolVal(sym);
+}
 
 DefinedOrUnknownSVal
 ValueManager::getDerivedRegionValueSymbolVal(SymbolRef parentSymbol,
                                              const TypedRegion *R) {
-  QualType T = R->getValueType(R->getContext());
+  QualType T = R->getValueType();
 
   if (!SymbolManager::canSymbolicate(T))
     return UnknownVal();
diff --git a/lib/CodeGen/ABIInfo.h b/lib/CodeGen/ABIInfo.h
index 85524acbe1f7..91b7742557f5 100644
--- a/lib/CodeGen/ABIInfo.h
+++ b/lib/CodeGen/ABIInfo.h
@@ -16,52 +16,52 @@
 namespace llvm {
   class Value;
   class LLVMContext;
+  class TargetData;
 }
 
 namespace clang {
   class ASTContext;
 
-  // FIXME: This is a layering issue if we want to move ABIInfo
-  // down. Fortunately CGFunctionInfo has no real tie to CodeGen.
   namespace CodeGen {
     class CGFunctionInfo;
     class CodeGenFunction;
+    class CodeGenTypes;
   }
 
-  /* FIXME: All of this stuff should be part of the target interface
-     somehow. It is currently here because it is not clear how to factor
-     the targets to support this, since the Targets currently live in a
-     layer below types n'stuff.
-  */
+  // FIXME: All of this stuff should be part of the target interface
+  // somehow. It is currently here because it is not clear how to factor
+  // the targets to support this, since the Targets currently live in a
+  // layer below types n'stuff.
 
   /// ABIArgInfo - Helper class to encapsulate information about how a
   /// specific C type should be passed to or returned from a function.
   class ABIArgInfo {
   public:
     enum Kind {
-      Direct,    /// Pass the argument directly using the normal
-                 /// converted LLVM type. Complex and structure types
-                 /// are passed using first class aggregates.
-
-      Extend,    /// Valid only for integer argument types. Same as 'direct'
-                 /// but also emit a zero/sign extension attribute.
-
-      Indirect,  /// Pass the argument indirectly via a hidden pointer
-                 /// with the specified alignment (0 indicates default
-                 /// alignment).
-
-      Ignore,    /// Ignore the argument (treat as void). Useful for
-                 /// void and empty structs.
-
-      Coerce,    /// Only valid for aggregate return types, the argument
-                 /// should be accessed by coercion to a provided type.
-
-      Expand,    /// Only valid for aggregate argument types. The
-                 /// structure should be expanded into consecutive
-                 /// arguments for its constituent fields. Currently
-                 /// expand is only allowed on structures whose fields
-                 /// are all scalar types or are themselves expandable
-                 /// types.
+      /// Direct - Pass the argument directly using the normal converted LLVM
+      /// type, or by coercing to another specified type stored in
+      /// 'CoerceToType').  If an offset is specified (in UIntData), then the
+      /// argument passed is offset by some number of bytes in the memory
+      /// representation.
+      Direct,
+
+      /// Extend - Valid only for integer argument types. Same as 'direct'
+      /// but also emit a zero/sign extension attribute.
+      Extend,
+
+      /// Indirect - Pass the argument indirectly via a hidden pointer
+      /// with the specified alignment (0 indicates default alignment).
+      Indirect,
+
+      /// Ignore - Ignore the argument (treat as void). Useful for void and
+      /// empty structs.
+      Ignore,
+
+      /// Expand - Only valid for aggregate argument types. The structure should
+      /// be expanded into consecutive arguments for its constituent fields.
+      /// Currently expand is only allowed on structures whose fields
+      /// are all scalar types or are themselves expandable types.
+      Expand,
 
       KindFirst=Direct, KindLast=Expand
     };
@@ -79,18 +79,15 @@ namespace clang {
   public:
     ABIArgInfo() : TheKind(Direct), TypeData(0), UIntData(0) {}
 
-    static ABIArgInfo getDirect() {
-      return ABIArgInfo(Direct);
+    static ABIArgInfo getDirect(const llvm::Type *T = 0, unsigned Offset = 0) {
+      return ABIArgInfo(Direct, T, Offset);
     }
-    static ABIArgInfo getExtend() {
-      return ABIArgInfo(Extend);
+    static ABIArgInfo getExtend(const llvm::Type *T = 0) {
+      return ABIArgInfo(Extend, T, 0);
     }
     static ABIArgInfo getIgnore() {
       return ABIArgInfo(Ignore);
     }
-    static ABIArgInfo getCoerce(const llvm::Type *T) {
-      return ABIArgInfo(Coerce, T);
-    }
     static ABIArgInfo getIndirect(unsigned Alignment, bool ByVal = true) {
       return ABIArgInfo(Indirect, 0, Alignment, ByVal);
     }
@@ -102,16 +99,28 @@ namespace clang {
     bool isDirect() const { return TheKind == Direct; }
     bool isExtend() const { return TheKind == Extend; }
     bool isIgnore() const { return TheKind == Ignore; }
-    bool isCoerce() const { return TheKind == Coerce; }
     bool isIndirect() const { return TheKind == Indirect; }
     bool isExpand() const { return TheKind == Expand; }
 
-    // Coerce accessors
+    bool canHaveCoerceToType() const {
+      return TheKind == Direct || TheKind == Extend;
+    }
+    
+    // Direct/Extend accessors
+    unsigned getDirectOffset() const {
+      assert((isDirect() || isExtend()) && "Not a direct or extend kind");
+      return UIntData;
+    }
     const llvm::Type *getCoerceToType() const {
-      assert(TheKind == Coerce && "Invalid kind!");
+      assert(canHaveCoerceToType() && "Invalid kind!");
       return TypeData;
     }
-
+    
+    void setCoerceToType(const llvm::Type *T) {
+      assert(canHaveCoerceToType() && "Invalid kind!");
+      TypeData = T;
+    }
+    
     // Indirect accessors
     unsigned getIndirectAlign() const {
       assert(TheKind == Indirect && "Invalid kind!");
@@ -130,15 +139,16 @@ namespace clang {
   /// passed or returned from functions.
   class ABIInfo {
   public:
+    CodeGen::CodeGenTypes &CGT;
+    
+    ABIInfo(CodeGen::CodeGenTypes &cgt) : CGT(cgt) {}
     virtual ~ABIInfo();
+    
+    ASTContext &getContext() const;
+    llvm::LLVMContext &getVMContext() const;
+    const llvm::TargetData &getTargetData() const;
 
-    virtual void computeInfo(CodeGen::CGFunctionInfo &FI,
-                             ASTContext &Ctx,
-                             llvm::LLVMContext &VMContext,
-                             // This is the preferred type for argument lowering
-                             // which can be used to generate better IR.
-                             const llvm::Type *const *PrefTypes = 0,
-                             unsigned NumPrefTypes = 0) const = 0;
+    virtual void computeInfo(CodeGen::CGFunctionInfo &FI) const = 0;
 
     /// EmitVAArg - Emit the target dependent code to load a value of
     /// \arg Ty from the va_list pointed to by \arg VAListAddr.
diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index cb9e63622185..04f1ef24b297 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -24,36 +24,6 @@
 using namespace clang;
 using namespace CodeGen;
 
-/// CGBlockInfo - Information to generate a block literal.
-class clang::CodeGen::CGBlockInfo {
-public:
-  /// Name - The name of the block, kindof.
-  const char *Name;
-
-  /// DeclRefs - Variables from parent scopes that have been
-  /// imported into this block.
-  llvm::SmallVector<const BlockDeclRefExpr *, 8> DeclRefs;
-
-  /// InnerBlocks - This block and the blocks it encloses.
-  llvm::SmallPtrSet<const DeclContext *, 4> InnerBlocks;
-
-  /// CXXThisRef - Non-null if 'this' was required somewhere, in
-  /// which case this is that expression.
-  const CXXThisExpr *CXXThisRef;
-
-  /// NeedsObjCSelf - True if something in this block has an implicit
-  /// reference to 'self'.
-  bool NeedsObjCSelf;
-
-  /// These are initialized by GenerateBlockFunction.
-  bool BlockHasCopyDispose;
-  CharUnits BlockSize;
-  CharUnits BlockAlign;
-  llvm::SmallVector<const Expr*, 8> BlockLayout;
-
-  CGBlockInfo(const char *Name);
-};
-
 CGBlockInfo::CGBlockInfo(const char *N)
   : Name(N), CXXThisRef(0), NeedsObjCSelf(false) {
     
@@ -64,9 +34,12 @@ CGBlockInfo::CGBlockInfo(const char *N)
 
 
 llvm::Constant *CodeGenFunction::
-BuildDescriptorBlockDecl(const BlockExpr *BE, bool BlockHasCopyDispose, CharUnits Size,
+BuildDescriptorBlockDecl(const BlockExpr *BE, const CGBlockInfo &Info,
                          const llvm::StructType* Ty,
+                         llvm::Constant *BlockVarLayout,
                          std::vector<HelperInfo> *NoteForHelper) {
+  bool BlockHasCopyDispose = Info.BlockHasCopyDispose;
+  CharUnits Size = Info.BlockSize;
   const llvm::Type *UnsignedLongTy
     = CGM.getTypes().ConvertType(getContext().UnsignedLongTy);
   llvm::Constant *C;
@@ -100,7 +73,8 @@ BuildDescriptorBlockDecl(const BlockExpr *BE, bool BlockHasCopyDispose, CharUnit
           CGM.GetAddrOfConstantCString(BlockTypeEncoding), PtrToInt8Ty));
   
   // Layout.
-  C = llvm::ConstantInt::get(UnsignedLongTy, 0);
+  C = BlockVarLayout;
+    
   Elts.push_back(C);
 
   C = llvm::ConstantStruct::get(VMContext, Elts, false);
@@ -111,20 +85,6 @@ BuildDescriptorBlockDecl(const BlockExpr *BE, bool BlockHasCopyDispose, CharUnit
   return C;
 }
 
-llvm::Constant *BlockModule::getNSConcreteGlobalBlock() {
-  if (NSConcreteGlobalBlock == 0)
-    NSConcreteGlobalBlock = CGM.CreateRuntimeVariable(PtrToInt8Ty,
-                                                      "_NSConcreteGlobalBlock");
-  return NSConcreteGlobalBlock;
-}
-
-llvm::Constant *BlockModule::getNSConcreteStackBlock() {
-  if (NSConcreteStackBlock == 0)
-    NSConcreteStackBlock = CGM.CreateRuntimeVariable(PtrToInt8Ty,
-                                                     "_NSConcreteStackBlock");
-  return NSConcreteStackBlock;
-}
-
 static void CollectBlockDeclRefInfo(const Stmt *S, CGBlockInfo &Info) {
   for (Stmt::const_child_iterator I = S->child_begin(), E = S->child_end();
        I != E; ++I)
@@ -198,6 +158,21 @@ static void AllocateAllBlockDeclRefs(CodeGenFunction &CGF, CGBlockInfo &Info) {
   }
 }
 
+static unsigned computeBlockFlag(CodeGenModule &CGM,
+                                 const BlockExpr *BE, unsigned flags) {
+  QualType BPT = BE->getType();
+  const FunctionType *ftype = BPT->getPointeeType()->getAs<FunctionType>();
+  QualType ResultType = ftype->getResultType();
+  
+  CallArgList Args;
+  CodeGenTypes &Types = CGM.getTypes();
+  const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, Args,
+                                                       FunctionType::ExtInfo());
+  if (CGM.ReturnTypeUsesSRet(FnInfo))
+    flags |= CodeGenFunction::BLOCK_USE_STRET;
+  return flags;
+}
+
 // FIXME: Push most into CGM, passing down a few bits, like current function
 // name.
 llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
@@ -221,6 +196,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
   llvm::Value *V;
 
   {
+    llvm::Constant *BlockVarLayout;
     // C = BuildBlockStructInitlist();
     unsigned int flags = BLOCK_HAS_SIGNATURE;
 
@@ -229,6 +205,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
     // __invoke
     llvm::Function *Fn
       = CodeGenFunction(CGM).GenerateBlockFunction(CurGD, BE, Info, CurFuncDecl,
+                                                   BlockVarLayout,
                                                    LocalDeclMap);
     BlockHasCopyDispose |= Info.BlockHasCopyDispose;
     Elts[3] = Fn;
@@ -244,18 +221,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
     Elts[0] = C;
 
     // __flags
-    {
-      QualType BPT = BE->getType();
-      const FunctionType *ftype = BPT->getPointeeType()->getAs<FunctionType>();
-      QualType ResultType = ftype->getResultType();
-    
-      CallArgList Args;
-      CodeGenTypes &Types = CGM.getTypes();
-      const CGFunctionInfo &FnInfo = Types.getFunctionInfo(ResultType, Args,
-                                                       FunctionType::ExtInfo());
-      if (CGM.ReturnTypeUsesSRet(FnInfo))
-        flags |= BLOCK_USE_STRET;
-    }
+    flags = computeBlockFlag(CGM, BE, flags);
     const llvm::IntegerType *IntTy = cast<llvm::IntegerType>(
       CGM.getTypes().ConvertType(CGM.getContext().IntTy));
     C = llvm::ConstantInt::get(IntTy, flags);
@@ -267,8 +233,8 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
 
     if (Info.BlockLayout.empty()) {
       // __descriptor
-      Elts[4] = BuildDescriptorBlockDecl(BE, Info.BlockHasCopyDispose,
-                                         Info.BlockSize, 0, 0);
+      C = llvm::Constant::getNullValue(PtrToInt8Ty);
+      Elts[4] = BuildDescriptorBlockDecl(BE, Info, 0, C, 0);
 
       // Optimize to being a global block.
       Elts[0] = CGM.getNSConcreteGlobalBlock();
@@ -371,7 +337,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
                                             SourceLocation());
               if (VD->getType()->isReferenceType()) {
                 E = new (getContext())
-                    UnaryOperator(const_cast<Expr*>(E), UnaryOperator::AddrOf,
+                    UnaryOperator(const_cast<Expr*>(E), UO_AddrOf,
                                 getContext().getPointerType(E->getType()),
                                 SourceLocation());
               } 
@@ -381,7 +347,7 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
 
       if (BDRE->isByRef()) {
         E = new (getContext())
-          UnaryOperator(const_cast<Expr*>(E), UnaryOperator::AddrOf,
+          UnaryOperator(const_cast<Expr*>(E), UO_AddrOf,
                         getContext().getPointerType(E->getType()),
                         SourceLocation());
       }
@@ -422,9 +388,8 @@ llvm::Value *CodeGenFunction::BuildBlockLiteralTmp(const BlockExpr *BE) {
     NoteForHelper.resize(NumHelpers);
 
     // __descriptor
-    llvm::Value *Descriptor = BuildDescriptorBlockDecl(BE,
-                                                       Info.BlockHasCopyDispose,
-                                                       Info.BlockSize, Ty,
+    llvm::Value *Descriptor = BuildDescriptorBlockDecl(BE, Info, Ty,
+                                                       BlockVarLayout,
                                                        &NoteForHelper);
     Descriptor = Builder.CreateBitCast(Descriptor, PtrToInt8Ty);
     Builder.CreateStore(Descriptor, Builder.CreateStructGEP(V, 4, "block.tmp"));
@@ -693,18 +658,23 @@ BlockModule::GetAddrOfGlobalBlock(const BlockExpr *BE, const char * n) {
   std::vector<llvm::Constant*> LiteralFields(FieldCount);
 
   CGBlockInfo Info(n);
+  llvm::Constant *BlockVarLayout;
   llvm::DenseMap<const Decl*, llvm::Value*> LocalDeclMap;
   llvm::Function *Fn
-    = CodeGenFunction(CGM).GenerateBlockFunction(GlobalDecl(), BE, Info, 0, LocalDeclMap);
+    = CodeGenFunction(CGM).GenerateBlockFunction(GlobalDecl(), BE, 
+                                                 Info, 0, BlockVarLayout,
+                                                 LocalDeclMap);
   assert(Info.BlockSize == BlockLiteralSize
          && "no imports allowed for global block");
 
   // isa
-  LiteralFields[0] = getNSConcreteGlobalBlock();
+  LiteralFields[0] = CGM.getNSConcreteGlobalBlock();
 
-  // Flags
+  // __flags
+  unsigned flags = computeBlockFlag(CGM, BE,
+                                    (BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE));
   LiteralFields[1] =
-    llvm::ConstantInt::get(IntTy, BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE);
+    llvm::ConstantInt::get(IntTy, flags);
 
   // Reserved
   LiteralFields[2] = llvm::Constant::getNullValue(IntTy);
@@ -737,6 +707,7 @@ llvm::Function *
 CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
                                        CGBlockInfo &Info,
                                        const Decl *OuterFuncDecl,
+                                       llvm::Constant *& BlockVarLayout,
                                   llvm::DenseMap<const Decl*, llvm::Value*> ldm) {
 
   // Check if we should generate debug info for this block.
@@ -751,7 +722,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
        ++i) {
     const VarDecl *VD = dyn_cast<VarDecl>(i->first);
 
-    if (VD->getStorageClass() == VarDecl::Static || VD->hasExternalStorage())
+    if (VD->getStorageClass() == SC_Static || VD->hasExternalStorage())
       LocalDeclMap[VD] = i->second;
   }
 
@@ -784,6 +755,12 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
   // Build the block struct now.
   AllocateAllBlockDeclRefs(*this, Info);
 
+  // Capture block layout info. here.
+  if (CGM.getContext().getLangOptions().ObjC1)
+    BlockVarLayout = CGM.getObjCRuntime().GCBlockLayout(*this, Info.DeclRefs);
+  else
+    BlockVarLayout = llvm::Constant::getNullValue(PtrToInt8Ty);
+  
   QualType ParmTy = getContext().getBlockParmType(BlockHasCopyDispose,
                                                   BlockLayout);
 
@@ -813,7 +790,11 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
                            Name.getString(), &CGM.getModule());
 
   CGM.SetInternalFunctionAttributes(BD, Fn, FI);
-
+  StartFunction(BD, ResultType, Fn, Args,
+                BExpr->getBody()->getLocEnd());
+  
+  CurFuncDecl = OuterFuncDecl;
+  
   QualType FnType(BlockFunctionType, 0);
   bool HasPrototype = isa<FunctionProtoType>(BlockFunctionType);
   
@@ -822,15 +803,22 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD, const BlockExpr *BExpr,
                                      getContext().getTranslationUnitDecl(),
                                      SourceLocation(), ID, FnType, 
                                      0,
-                                     FunctionDecl::Static,
-                                     FunctionDecl::None,
+                                     SC_Static,
+                                     SC_None,
                                      false, HasPrototype);
+  if (FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FnType)) {
+    const FunctionDecl *CFD = dyn_cast<FunctionDecl>(CurCodeDecl);
+    FunctionDecl *FD = const_cast<FunctionDecl *>(CFD);
+    llvm::SmallVector<ParmVarDecl*, 16> Params;
+    for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i)
+      Params.push_back(ParmVarDecl::Create(getContext(), FD, 
+                                           SourceLocation(), 0,
+                                           FT->getArgType(i), /*TInfo=*/0,
+                                           SC_None, SC_None, 0));
+    FD->setParams(Params.data(), Params.size());
+  }
+  
   
-  StartFunction(BD, ResultType, Fn, Args,
-                BExpr->getBody()->getLocEnd());
-
-  CurFuncDecl = OuterFuncDecl;
-
   // If we have a C++ 'this' reference, go ahead and force it into
   // existence now.
   if (Info.CXXThisRef) {
@@ -928,7 +916,7 @@ CharUnits BlockFunction::getBlockOffset(CharUnits Size, CharUnits Align) {
                                          getContext().getTranslationUnitDecl(),
                                          SourceLocation(),
                                          0, QualType(PadTy), 0,
-                                         VarDecl::None, VarDecl::None);
+                                         SC_None, SC_None);
     Expr *E = new (getContext()) DeclRefExpr(PadDecl, PadDecl->getType(),
                                              SourceLocation());
     BlockLayout.push_back(E);
@@ -974,8 +962,8 @@ GenerateCopyHelperFunction(bool BlockHasCopyDispose, const llvm::StructType *T,
   FunctionDecl *FD = FunctionDecl::Create(getContext(),
                                           getContext().getTranslationUnitDecl(),
                                           SourceLocation(), II, R, 0,
-                                          FunctionDecl::Static,
-                                          FunctionDecl::None,
+                                          SC_Static,
+                                          SC_None,
                                           false,
                                           true);
   CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
@@ -1012,7 +1000,7 @@ GenerateCopyHelperFunction(bool BlockHasCopyDispose, const llvm::StructType *T,
         Dstv = Builder.CreateBitCast(Dstv, PtrToInt8Ty);
 
         llvm::Value *N = llvm::ConstantInt::get(CGF.Int32Ty, flag);
-        llvm::Value *F = getBlockObjectAssign();
+        llvm::Value *F = CGM.getBlockObjectAssign();
         Builder.CreateCall3(F, Dstv, Srcv, N);
       }
     }
@@ -1056,8 +1044,8 @@ GenerateDestroyHelperFunction(bool BlockHasCopyDispose,
   FunctionDecl *FD = FunctionDecl::Create(getContext(),
                                           getContext().getTranslationUnitDecl(),
                                           SourceLocation(), II, R, 0,
-                                          FunctionDecl::Static,
-                                          FunctionDecl::None,
+                                          SC_Static,
+                                          SC_None,
                                           false, true);
   CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
 
@@ -1141,8 +1129,8 @@ GeneratebyrefCopyHelperFunction(const llvm::Type *T, int flag) {
   FunctionDecl *FD = FunctionDecl::Create(getContext(),
                                           getContext().getTranslationUnitDecl(),
                                           SourceLocation(), II, R, 0,
-                                          FunctionDecl::Static,
-                                          FunctionDecl::None,
+                                          SC_Static,
+                                          SC_None,
                                           false, true);
   CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
 
@@ -1164,7 +1152,7 @@ GeneratebyrefCopyHelperFunction(const llvm::Type *T, int flag) {
   flag |= BLOCK_BYREF_CALLER;
 
   llvm::Value *N = llvm::ConstantInt::get(CGF.Int32Ty, flag);
-  llvm::Value *F = getBlockObjectAssign();
+  llvm::Value *F = CGM.getBlockObjectAssign();
   Builder.CreateCall3(F, DstObj, SrcObj, N);
 
   CGF.FinishFunction();
@@ -1205,8 +1193,8 @@ BlockFunction::GeneratebyrefDestroyHelperFunction(const llvm::Type *T,
   FunctionDecl *FD = FunctionDecl::Create(getContext(),
                                           getContext().getTranslationUnitDecl(),
                                           SourceLocation(), II, R, 0,
-                                          FunctionDecl::Static,
-                                          FunctionDecl::None,
+                                          SC_Static,
+                                          SC_None,
                                           false, true);
   CGF.StartFunction(FD, R, Fn, Args, SourceLocation());
 
@@ -1259,37 +1247,8 @@ llvm::Constant *BlockFunction::BuildbyrefDestroyHelper(const llvm::Type *T,
   return Entry=CodeGenFunction(CGM).GeneratebyrefDestroyHelperFunction(T, Flag);
 }
 
-llvm::Value *BlockFunction::getBlockObjectDispose() {
-  if (CGM.BlockObjectDispose == 0) {
-    const llvm::FunctionType *FTy;
-    std::vector<const llvm::Type*> ArgTys;
-    const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
-    ArgTys.push_back(PtrToInt8Ty);
-    ArgTys.push_back(CGF.Int32Ty);
-    FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
-    CGM.BlockObjectDispose
-      = CGM.CreateRuntimeFunction(FTy, "_Block_object_dispose");
-  }
-  return CGM.BlockObjectDispose;
-}
-
-llvm::Value *BlockFunction::getBlockObjectAssign() {
-  if (CGM.BlockObjectAssign == 0) {
-    const llvm::FunctionType *FTy;
-    std::vector<const llvm::Type*> ArgTys;
-    const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
-    ArgTys.push_back(PtrToInt8Ty);
-    ArgTys.push_back(PtrToInt8Ty);
-    ArgTys.push_back(CGF.Int32Ty);
-    FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
-    CGM.BlockObjectAssign
-      = CGM.CreateRuntimeFunction(FTy, "_Block_object_assign");
-  }
-  return CGM.BlockObjectAssign;
-}
-
 void BlockFunction::BuildBlockRelease(llvm::Value *V, int flag) {
-  llvm::Value *F = getBlockObjectDispose();
+  llvm::Value *F = CGM.getBlockObjectDispose();
   llvm::Value *N;
   V = Builder.CreateBitCast(V, PtrToInt8Ty);
   N = llvm::ConstantInt::get(CGF.Int32Ty, flag);
diff --git a/lib/CodeGen/CGBlocks.h b/lib/CodeGen/CGBlocks.h
index 772a62c24f1f..743e3c83be33 100644
--- a/lib/CodeGen/CGBlocks.h
+++ b/lib/CodeGen/CGBlocks.h
@@ -73,8 +73,6 @@ class BlockModule : public BlockBase {
   CodeGenTypes &getTypes() { return Types; }
   const llvm::TargetData &getTargetData() const { return TheTargetData; }
 public:
-  llvm::Constant *getNSConcreteGlobalBlock();
-  llvm::Constant *getNSConcreteStackBlock();
   int getGlobalUniqueCount() { return ++Block.GlobalUniqueCount; }
   const llvm::Type *getBlockDescriptorType();
 
@@ -82,14 +80,6 @@ public:
 
   llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, const char *);
 
-  /// NSConcreteGlobalBlock - Cached reference to the class pointer for global
-  /// blocks.
-  llvm::Constant *NSConcreteGlobalBlock;
-
-  /// NSConcreteStackBlock - Cached reference to the class poinnter for stack
-  /// blocks.
-  llvm::Constant *NSConcreteStackBlock;
-
   const llvm::Type *BlockDescriptorType;
   const llvm::Type *GenericBlockLiteralType;
 
@@ -97,8 +87,6 @@ public:
     int GlobalUniqueCount;
   } Block;
 
-  llvm::Value *BlockObjectAssign;
-  llvm::Value *BlockObjectDispose;
   const llvm::PointerType *PtrToInt8Ty;
 
   std::map<uint64_t, llvm::Constant *> AssignCache;
@@ -108,9 +96,7 @@ public:
               CodeGenTypes &T, CodeGenModule &CodeGen)
     : Context(C), TheModule(M), TheTargetData(TD), Types(T),
       CGM(CodeGen), VMContext(M.getContext()),
-      NSConcreteGlobalBlock(0), NSConcreteStackBlock(0), BlockDescriptorType(0),
-      GenericBlockLiteralType(0),
-      BlockObjectAssign(0), BlockObjectDispose(0) {
+      BlockDescriptorType(0), GenericBlockLiteralType(0) {
     Block.GlobalUniqueCount = 0;
     PtrToInt8Ty = llvm::Type::getInt8PtrTy(M.getContext());
   }
@@ -207,8 +193,6 @@ public:
   llvm::Constant *BuildbyrefDestroyHelper(const llvm::Type *T, int flag,
                                           unsigned Align);
 
-  llvm::Value *getBlockObjectAssign();
-  llvm::Value *getBlockObjectDispose();
   void BuildBlockRelease(llvm::Value *DeclPtr, int flag = BLOCK_FIELD_IS_BYREF);
 
   bool BlockRequiresCopying(QualType Ty)
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index fff4bacab6b5..986f621f64f5 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -41,6 +41,31 @@ static void EmitMemoryBarrier(CodeGenFunction &CGF,
                          C, C + 5);
 }
 
+static Value *EmitCastToInt(CodeGenFunction &CGF,
+                            const llvm::Type *ToType, Value *Val) {
+  if (Val->getType()->isPointerTy()) {
+    return CGF.Builder.CreatePtrToInt(Val, ToType);
+  }
+  assert(Val->getType()->isIntegerTy() &&
+         "Used a non-integer and non-pointer type with atomic builtin");
+  assert(Val->getType()->getScalarSizeInBits() <=
+         ToType->getScalarSizeInBits() && "Integer type too small");
+  return CGF.Builder.CreateSExtOrBitCast(Val, ToType);
+}
+
+static Value *EmitCastFromInt(CodeGenFunction &CGF, QualType ToQualType,
+                              Value *Val) {
+  const llvm::Type *ToType = CGF.ConvertType(ToQualType);
+  if (ToType->isPointerTy()) {
+    return CGF.Builder.CreateIntToPtr(Val, ToType);
+  }
+  assert(Val->getType()->isIntegerTy() &&
+         "Used a non-integer and non-pointer type with atomic builtin");
+  assert(Val->getType()->getScalarSizeInBits() >=
+         ToType->getScalarSizeInBits() && "Integer type too small");
+  return CGF.Builder.CreateTruncOrBitCast(Val, ToType);
+}
+
 // The atomic builtins are also full memory barriers. This is a utility for
 // wrapping a call to the builtins with memory barriers.
 static Value *EmitCallWithBarrier(CodeGenFunction &CGF, Value *Fn,
@@ -60,13 +85,20 @@ static Value *EmitCallWithBarrier(CodeGenFunction &CGF, Value *Fn,
 /// and the expression node.
 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
                                Intrinsic::ID Id, const CallExpr *E) {
-  Value *Args[2] = { CGF.EmitScalarExpr(E->getArg(0)),
-                     CGF.EmitScalarExpr(E->getArg(1)) };
-  const llvm::Type *ResType[2];
-  ResType[0] = CGF.ConvertType(E->getType());
-  ResType[1] = CGF.ConvertType(E->getArg(0)->getType());
-  Value *AtomF = CGF.CGM.getIntrinsic(Id, ResType, 2);
-  return RValue::get(EmitCallWithBarrier(CGF, AtomF, Args, Args + 2));
+  const llvm::Type *ValueType =
+    llvm::IntegerType::get(CGF.getLLVMContext(),
+                           CGF.getContext().getTypeSize(E->getType()));
+  const llvm::Type *PtrType = ValueType->getPointerTo();
+  const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
+  Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes, 2);
+
+  Value *Args[2] = { CGF.Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
+                                               PtrType),
+                     EmitCastToInt(CGF, ValueType,
+                                   CGF.EmitScalarExpr(E->getArg(1))) };
+  return RValue::get(EmitCastFromInt(CGF, E->getType(),
+                                     EmitCallWithBarrier(CGF, AtomF, Args,
+                                                         Args + 2)));
 }
 
 /// Utility to insert an atomic instruction based Instrinsic::ID and
@@ -75,14 +107,21 @@ static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
                                    Intrinsic::ID Id, const CallExpr *E,
                                    Instruction::BinaryOps Op) {
-  const llvm::Type *ResType[2];
-  ResType[0] = CGF.ConvertType(E->getType());
-  ResType[1] = CGF.ConvertType(E->getArg(0)->getType());
-  Value *AtomF = CGF.CGM.getIntrinsic(Id, ResType, 2);
-  Value *Args[2] = { CGF.EmitScalarExpr(E->getArg(0)),
-                     CGF.EmitScalarExpr(E->getArg(1)) };
+  const llvm::Type *ValueType =
+    llvm::IntegerType::get(CGF.getLLVMContext(),
+                           CGF.getContext().getTypeSize(E->getType()));
+  const llvm::Type *PtrType = ValueType->getPointerTo();
+  const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
+  Value *AtomF = CGF.CGM.getIntrinsic(Id, IntrinsicTypes, 2);
+
+  Value *Args[2] = { CGF.Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
+                                               PtrType),
+                     EmitCastToInt(CGF, ValueType,
+                                   CGF.EmitScalarExpr(E->getArg(1))) };
   Value *Result = EmitCallWithBarrier(CGF, AtomF, Args, Args + 2);
-  return RValue::get(CGF.Builder.CreateBinOp(Op, Result, Args[1]));
+  return RValue::get(EmitCastFromInt(CGF, E->getType(),
+                                     CGF.Builder.CreateBinOp(Op, Result,
+                                                             Args[1])));
 }
 
 /// EmitFAbs - Emit a call to fabs/fabsf/fabsl, depending on the type of ValTy,
@@ -245,9 +284,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
                                      "cast");
     return RValue::get(Result);
   }
-  case Builtin::BI__builtin_expect:
+  case Builtin::BI__builtin_expect: {
     // FIXME: pass expect through to LLVM
+    if (E->getArg(1)->HasSideEffects(getContext()))
+      (void)EmitScalarExpr(E->getArg(1));
     return RValue::get(EmitScalarExpr(E->getArg(0)));
+  }
   case Builtin::BI__builtin_bswap32:
   case Builtin::BI__builtin_bswap64: {
     Value *ArgValue = EmitScalarExpr(E->getArg(0));
@@ -746,14 +788,23 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_val_compare_and_swap_4:
   case Builtin::BI__sync_val_compare_and_swap_8:
   case Builtin::BI__sync_val_compare_and_swap_16: {
-    const llvm::Type *ResType[2];
-    ResType[0]= ConvertType(E->getType());
-    ResType[1] = ConvertType(E->getArg(0)->getType());
-    Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap, ResType, 2);
-    Value *Args[3] = { EmitScalarExpr(E->getArg(0)),
-                       EmitScalarExpr(E->getArg(1)),
-                       EmitScalarExpr(E->getArg(2)) };
-    return RValue::get(EmitCallWithBarrier(*this, AtomF, Args, Args + 3));
+    const llvm::Type *ValueType =
+      llvm::IntegerType::get(CGF.getLLVMContext(),
+                             CGF.getContext().getTypeSize(E->getType()));
+    const llvm::Type *PtrType = ValueType->getPointerTo();
+    const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
+    Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap,
+                                    IntrinsicTypes, 2);
+
+    Value *Args[3] = { Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
+                                             PtrType),
+                       EmitCastToInt(CGF, ValueType,
+                                     CGF.EmitScalarExpr(E->getArg(1))),
+                       EmitCastToInt(CGF, ValueType,
+                                     CGF.EmitScalarExpr(E->getArg(2))) };
+    return RValue::get(EmitCastFromInt(CGF, E->getType(),
+                                       EmitCallWithBarrier(CGF, AtomF, Args,
+                                                           Args + 3)));
   }
 
   case Builtin::BI__sync_bool_compare_and_swap_1:
@@ -761,14 +812,22 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_bool_compare_and_swap_4:
   case Builtin::BI__sync_bool_compare_and_swap_8:
   case Builtin::BI__sync_bool_compare_and_swap_16: {
-    const llvm::Type *ResType[2];
-    ResType[0]= ConvertType(E->getArg(1)->getType());
-    ResType[1] = llvm::PointerType::getUnqual(ResType[0]);
-    Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap, ResType, 2);
-    Value *OldVal = EmitScalarExpr(E->getArg(1));
-    Value *Args[3] = { EmitScalarExpr(E->getArg(0)),
-                       OldVal,
-                       EmitScalarExpr(E->getArg(2)) };
+    const llvm::Type *ValueType =
+      llvm::IntegerType::get(
+        CGF.getLLVMContext(),
+        CGF.getContext().getTypeSize(E->getArg(1)->getType()));
+    const llvm::Type *PtrType = ValueType->getPointerTo();
+    const llvm::Type *IntrinsicTypes[2] = { ValueType, PtrType };
+    Value *AtomF = CGM.getIntrinsic(Intrinsic::atomic_cmp_swap,
+                                    IntrinsicTypes, 2);
+
+    Value *Args[3] = { Builder.CreateBitCast(CGF.EmitScalarExpr(E->getArg(0)),
+                                             PtrType),
+                       EmitCastToInt(CGF, ValueType,
+                                     CGF.EmitScalarExpr(E->getArg(1))),
+                       EmitCastToInt(CGF, ValueType,
+                                     CGF.EmitScalarExpr(E->getArg(2))) };
+    Value *OldVal = Args[1];
     Value *PrevVal = EmitCallWithBarrier(*this, AtomF, Args, Args + 3);
     Value *Result = Builder.CreateICmpEQ(PrevVal, OldVal);
     // zext bool to int.
@@ -953,8 +1012,10 @@ const llvm::VectorType *GetNeonType(LLVMContext &C, unsigned type, bool q) {
   return 0;
 }
 
-Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
+Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C, bool widen) {
   unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements();
+  if (widen)
+    nElts <<= 1;
   SmallVector<Constant*, 16> Indices(nElts, C);
   Value* SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
   return Builder.CreateShuffleVector(V, V, SV, "lane");
@@ -989,6 +1050,28 @@ Value *CodeGenFunction::EmitNeonShiftVector(Value *V, const llvm::Type *Ty,
   return llvm::ConstantVector::get(CV.begin(), CV.size());
 }
 
+/// GetPointeeAlignment - Given an expression with a pointer type, find the
+/// alignment of the type referenced by the pointer.  Skip over implicit
+/// casts.
+static Value *GetPointeeAlignment(CodeGenFunction &CGF, const Expr *Addr) {
+  unsigned Align = 1;
+  // Check if the type is a pointer.  The implicit cast operand might not be.
+  while (Addr->getType()->isPointerType()) {
+    QualType PtTy = Addr->getType()->getPointeeType();
+    unsigned NewA = CGF.getContext().getTypeAlignInChars(PtTy).getQuantity();
+    if (NewA > Align)
+      Align = NewA;
+
+    // If the address is an implicit cast, repeat with the cast operand.
+    if (const ImplicitCastExpr *CastAddr = dyn_cast<ImplicitCastExpr>(Addr)) {
+      Addr = CastAddr->getSubExpr();
+      continue;
+    }
+    break;
+  }
+  return llvm::ConstantInt::get(CGF.Int32Ty, Align);
+}
+
 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E) {
   if (BuiltinID == ARM::BI__clear_cache) {
@@ -1002,9 +1085,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
                                a, b);
   }
 
-  // Determine the type of this overloaded NEON intrinsic.
-  assert(BuiltinID > ARM::BI__builtin_thread_pointer);
-
   llvm::SmallVector<Value*, 4> Ops;
   for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++)
     Ops.push_back(EmitScalarExpr(E->getArg(i)));
@@ -1014,6 +1094,25 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   if (!Arg->isIntegerConstantExpr(Result, getContext()))
     return 0;
 
+  if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
+      BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
+    // Determine the overloaded type of this builtin.
+    const llvm::Type *Ty;
+    if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
+      Ty = llvm::Type::getFloatTy(VMContext);
+    else
+      Ty = llvm::Type::getDoubleTy(VMContext);
+    
+    // Determine whether this is an unsigned conversion or not.
+    bool usgn = Result.getZExtValue() == 1;
+    unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
+
+    // Call the appropriate intrinsic.
+    Function *F = CGM.getIntrinsic(Int, &Ty, 1);
+    return Builder.CreateCall(F, Ops.begin(), Ops.end(), "vcvtr");
+  }
+  
+  // Determine the type of this overloaded NEON intrinsic.
   unsigned type = Result.getZExtValue();
   bool usgn = type & 0x08;
   bool quad = type & 0x10;
@@ -1029,19 +1128,36 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   switch (BuiltinID) {
   default: return 0;
   case ARM::BI__builtin_neon_vaba_v:
-  case ARM::BI__builtin_neon_vabaq_v:
-    Int = usgn ? Intrinsic::arm_neon_vabau : Intrinsic::arm_neon_vabas;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vaba");
-  case ARM::BI__builtin_neon_vabal_v:
-    Int = usgn ? Intrinsic::arm_neon_vabalu : Intrinsic::arm_neon_vabals;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vabal");
+  case ARM::BI__builtin_neon_vabaq_v: {
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    SmallVector<Value*, 2> Args;
+    Args.push_back(Ops[1]);
+    Args.push_back(Ops[2]);
+    Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
+    Ops[1] = EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Args, "vaba");
+    return Builder.CreateAdd(Ops[0], Ops[1], "vaba");
+  }
+  case ARM::BI__builtin_neon_vabal_v: {
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    SmallVector<Value*, 2> Args;
+    Args.push_back(Ops[1]);
+    Args.push_back(Ops[2]);
+    Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[1] = EmitNeonCall(CGM.getIntrinsic(Int, &DTy, 1), Args, "vabal");
+    Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+    return Builder.CreateAdd(Ops[0], Ops[1], "vabal");
+  }
   case ARM::BI__builtin_neon_vabd_v:
   case ARM::BI__builtin_neon_vabdq_v:
     Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vabd");
-  case ARM::BI__builtin_neon_vabdl_v:
-    Int = usgn ? Intrinsic::arm_neon_vabdlu : Intrinsic::arm_neon_vabdls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vabdl");
+  case ARM::BI__builtin_neon_vabdl_v: {
+    Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, &DTy, 1), Ops, "vabdl");
+    return Builder.CreateZExt(Ops[0], Ty, "vabdl");
+  }
   case ARM::BI__builtin_neon_vabs_v:
   case ARM::BI__builtin_neon_vabsq_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vabs, &Ty, 1),
@@ -1049,12 +1165,29 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vaddhn_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vaddhn, &Ty, 1),
                         Ops, "vaddhn");
-  case ARM::BI__builtin_neon_vaddl_v:
-    Int = usgn ? Intrinsic::arm_neon_vaddlu : Intrinsic::arm_neon_vaddls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vaddl");
-  case ARM::BI__builtin_neon_vaddw_v:
-    Int = usgn ? Intrinsic::arm_neon_vaddws : Intrinsic::arm_neon_vaddwu;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vaddw");
+  case ARM::BI__builtin_neon_vaddl_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    if (usgn) {
+      Ops[0] = Builder.CreateZExt(Ops[0], Ty);
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+    } else {
+      Ops[0] = Builder.CreateSExt(Ops[0], Ty);
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+    }
+    return Builder.CreateAdd(Ops[0], Ops[1], "vaddl");
+  }
+  case ARM::BI__builtin_neon_vaddw_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    if (usgn)
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+    else
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+    return Builder.CreateAdd(Ops[0], Ops[1], "vaddw");
+  }
   case ARM::BI__builtin_neon_vcale_v:
     std::swap(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vcage_v: {
@@ -1126,6 +1259,12 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Int, Tys, 2);
     return EmitNeonCall(F, Ops, "vcvt_n");
   }
+  case ARM::BI__builtin_neon_vdup_lane_v: 
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    return EmitNeonSplat(Ops[0], cast<Constant>(Ops[1]));
+  case ARM::BI__builtin_neon_vdupq_lane_v:
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    return EmitNeonSplat(Ops[0], cast<Constant>(Ops[1]), true);
   case ARM::BI__builtin_neon_vext_v:
   case ARM::BI__builtin_neon_vextq_v: {
     ConstantInt *C = dyn_cast<ConstantInt>(Ops[2]);
@@ -1161,6 +1300,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vhsub");
   case ARM::BI__builtin_neon_vld1_v:
   case ARM::BI__builtin_neon_vld1q_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vld1, &Ty, 1),
                         Ops, "vld1");
   case ARM::BI__builtin_neon_vld1_lane_v:
@@ -1183,7 +1323,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld2_v:
   case ARM::BI__builtin_neon_vld2q_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2, &Ty, 1);
-    Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
+    Value *Align = GetPointeeAlignment(*this, E->getArg(1));
+    Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld2");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     return Builder.CreateStore(Ops[1], Ops[0]);
@@ -1191,7 +1332,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld3_v:
   case ARM::BI__builtin_neon_vld3q_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld3, &Ty, 1);
-    Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
+    Value *Align = GetPointeeAlignment(*this, E->getArg(1));
+    Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld3");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     return Builder.CreateStore(Ops[1], Ops[0]);
@@ -1199,7 +1341,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld4_v:
   case ARM::BI__builtin_neon_vld4q_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld4, &Ty, 1);
-    Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
+    Value *Align = GetPointeeAlignment(*this, E->getArg(1));
+    Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld4");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     return Builder.CreateStore(Ops[1], Ops[0]);
@@ -1209,6 +1352,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2lane, &Ty, 1);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(1)));
     Ops[1] = Builder.CreateCall(F, Ops.begin() + 1, Ops.end(), "vld2_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1220,6 +1364,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(1)));
     Ops[1] = Builder.CreateCall(F, Ops.begin() + 1, Ops.end(), "vld3_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1232,6 +1377,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
     Ops[5] = Builder.CreateBitCast(Ops[5], Ty);
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(1)));
     Ops[1] = Builder.CreateCall(F, Ops.begin() + 1, Ops.end(), "vld3_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1261,6 +1407,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
 
     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
     Args.push_back(CI);
+    Args.push_back(GetPointeeAlignment(*this, E->getArg(1)));
     
     Ops[1] = Builder.CreateCall(F, Args.begin(), Args.end(), "vld_dup");
     // splat lane 0 to all elts in each vector of the result.
@@ -1283,28 +1430,79 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vminq_v:
     Int = usgn ? Intrinsic::arm_neon_vminu : Intrinsic::arm_neon_vmins;
     return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vmin");
-  case ARM::BI__builtin_neon_vmlal_lane_v:
-    splat = true;
-  case ARM::BI__builtin_neon_vmlal_v:
-    Int = usgn ? Intrinsic::arm_neon_vmlalu : Intrinsic::arm_neon_vmlals;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vmlal", splat);
-  case ARM::BI__builtin_neon_vmlsl_lane_v:
-    splat = true;
-  case ARM::BI__builtin_neon_vmlsl_v:
-    Int = usgn ? Intrinsic::arm_neon_vmlslu : Intrinsic::arm_neon_vmlsls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vmlsl", splat);
-  case ARM::BI__builtin_neon_vmovl_v:
-    Int = usgn ? Intrinsic::arm_neon_vmovlu : Intrinsic::arm_neon_vmovls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vmovl");
-  case ARM::BI__builtin_neon_vmovn_v:
-    return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmovn, &Ty, 1),
-                        Ops, "vmovn");
-  case ARM::BI__builtin_neon_vmull_lane_v:
-    splat = true;
-  case ARM::BI__builtin_neon_vmull_v:
-    Int = usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
-    Int = poly ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vmlal", splat);
+  case ARM::BI__builtin_neon_vmlal_lane_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
+    Ops[2] = EmitNeonSplat(Ops[2], cast<Constant>(Ops[3]));
+  }
+  case ARM::BI__builtin_neon_vmlal_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
+    if (usgn) {
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+      Ops[2] = Builder.CreateZExt(Ops[2], Ty);
+    } else {
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+      Ops[2] = Builder.CreateSExt(Ops[2], Ty);
+    }
+    Ops[1] = Builder.CreateMul(Ops[1], Ops[2]);
+    return Builder.CreateAdd(Ops[0], Ops[1], "vmlal");
+  }
+  case ARM::BI__builtin_neon_vmlsl_lane_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
+    Ops[2] = EmitNeonSplat(Ops[2], cast<Constant>(Ops[3]));
+  }
+  case ARM::BI__builtin_neon_vmlsl_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    Ops[2] = Builder.CreateBitCast(Ops[2], DTy);
+    if (usgn) {
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+      Ops[2] = Builder.CreateZExt(Ops[2], Ty);
+    } else {
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+      Ops[2] = Builder.CreateSExt(Ops[2], Ty);
+    }
+    Ops[1] = Builder.CreateMul(Ops[1], Ops[2]);
+    return Builder.CreateSub(Ops[0], Ops[1], "vmlsl");
+  }
+  case ARM::BI__builtin_neon_vmovl_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
+    if (usgn)
+      return Builder.CreateZExt(Ops[0], Ty, "vmovl");
+    return Builder.CreateSExt(Ops[0], Ty, "vmovl");
+  }
+  case ARM::BI__builtin_neon_vmovn_v: {
+    const llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
+    return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
+  }
+  case ARM::BI__builtin_neon_vmull_lane_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    Ops[1] = EmitNeonSplat(Ops[1], cast<Constant>(Ops[2]));
+  }
+  case ARM::BI__builtin_neon_vmull_v: {
+    if (poly)
+      return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vmullp, &Ty, 1),
+                          Ops, "vmull");
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    if (usgn) {
+      Ops[0] = Builder.CreateZExt(Ops[0], Ty);
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+    } else {
+      Ops[0] = Builder.CreateSExt(Ops[0], Ty);
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+    }
+    return Builder.CreateMul(Ops[0], Ops[1], "vmull");
+  }
   case ARM::BI__builtin_neon_vpadal_v:
   case ARM::BI__builtin_neon_vpadalq_v:
     Int = usgn ? Intrinsic::arm_neon_vpadalu : Intrinsic::arm_neon_vpadals;
@@ -1503,6 +1701,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return Builder.CreateAdd(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vst1_v:
   case ARM::BI__builtin_neon_vst1q_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vst1_lane_v:
@@ -1513,37 +1712,60 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty));
   case ARM::BI__builtin_neon_vst2_v:
   case ARM::BI__builtin_neon_vst2q_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vst2_lane_v:
   case ARM::BI__builtin_neon_vst2q_lane_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2lane, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vst3_v:
   case ARM::BI__builtin_neon_vst3q_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vst3_lane_v:
   case ARM::BI__builtin_neon_vst3q_lane_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3lane, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vst4_v:
   case ARM::BI__builtin_neon_vst4q_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vst4_lane_v:
   case ARM::BI__builtin_neon_vst4q_lane_v:
+    Ops.push_back(GetPointeeAlignment(*this, E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4lane, &Ty, 1),
                         Ops, "");
   case ARM::BI__builtin_neon_vsubhn_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vsubhn, &Ty, 1),
                         Ops, "vsubhn");
-  case ARM::BI__builtin_neon_vsubl_v:
-    Int = usgn ? Intrinsic::arm_neon_vsublu : Intrinsic::arm_neon_vsubls;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vsubl");
-  case ARM::BI__builtin_neon_vsubw_v:
-    Int = usgn ? Intrinsic::arm_neon_vsubws : Intrinsic::arm_neon_vsubwu;
-    return EmitNeonCall(CGM.getIntrinsic(Int, &Ty, 1), Ops, "vsubw");
+  case ARM::BI__builtin_neon_vsubl_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    if (usgn) {
+      Ops[0] = Builder.CreateZExt(Ops[0], Ty);
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+    } else {
+      Ops[0] = Builder.CreateSExt(Ops[0], Ty);
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+    }
+    return Builder.CreateSub(Ops[0], Ops[1], "vsubl");
+  }
+  case ARM::BI__builtin_neon_vsubw_v: {
+    const llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    Ops[1] = Builder.CreateBitCast(Ops[1], DTy);
+    if (usgn)
+      Ops[1] = Builder.CreateZExt(Ops[1], Ty);
+    else
+      Ops[1] = Builder.CreateSExt(Ops[1], Ty);
+    return Builder.CreateSub(Ops[0], Ops[1], "vsubw");
+  }
   case ARM::BI__builtin_neon_vtbl1_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
                         Ops, "vtbl1");
@@ -1626,8 +1848,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     for (unsigned vi = 0; vi != 2; ++vi) {
       SmallVector<Constant*, 16> Indices;
       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
-        Indices.push_back(ConstantInt::get(Int32Ty, (i >> 1)));
-        Indices.push_back(ConstantInt::get(Int32Ty, (i >> 1)+e));
+        Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
+        Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
       }
       Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
       SV = llvm::ConstantVector::get(Indices.begin(), Indices.size());
diff --git a/lib/CodeGen/CGCXX.cpp b/lib/CodeGen/CGCXX.cpp
index 7b7be9a260ed..179716f631a8 100644
--- a/lib/CodeGen/CGCXX.cpp
+++ b/lib/CodeGen/CGCXX.cpp
@@ -287,44 +287,6 @@ CodeGenModule::GetAddrOfCXXDestructor(const CXXDestructorDecl *D,
   return cast<llvm::Function>(GetOrCreateLLVMFunction(Name, FTy, GD));
 }
 
-llvm::Constant *
-CodeGenModule::GetCXXMemberFunctionPointerValue(const CXXMethodDecl *MD) {
-  assert(MD->isInstance() && "Member function must not be static!");
-    
-  MD = MD->getCanonicalDecl();
-
-  const llvm::Type *PtrDiffTy = Types.ConvertType(Context.getPointerDiffType());
-
-  // Get the function pointer (or index if this is a virtual function).
-  if (MD->isVirtual()) {
-    uint64_t Index = VTables.getMethodVTableIndex(MD);
-
-    // FIXME: We shouldn't use / 8 here.
-    uint64_t PointerWidthInBytes = Context.Target.getPointerWidth(0) / 8;
-
-    // Itanium C++ ABI 2.3:
-    //   For a non-virtual function, this field is a simple function pointer. 
-    //   For a virtual function, it is 1 plus the virtual table offset 
-    //   (in bytes) of the function, represented as a ptrdiff_t. 
-    return llvm::ConstantInt::get(PtrDiffTy, (Index * PointerWidthInBytes) + 1);
-  }
-
-  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
-  const llvm::Type *Ty;
-  // Check whether the function has a computable LLVM signature.
-  if (!CodeGenTypes::VerifyFuncTypeComplete(FPT)) {
-    // The function has a computable LLVM signature; use the correct type.
-    Ty = Types.GetFunctionType(Types.getFunctionInfo(MD), FPT->isVariadic());
-  } else {
-    // Use an arbitrary non-function type to tell GetAddrOfFunction that the
-    // function type is incomplete.
-    Ty = PtrDiffTy;
-  }
-
-  llvm::Constant *FuncPtr = GetAddrOfFunction(MD, Ty);
-  return llvm::ConstantExpr::getPtrToInt(FuncPtr, PtrDiffTy);
-}
-
 static llvm::Value *BuildVirtualCall(CodeGenFunction &CGF, uint64_t VTableIndex, 
                                      llvm::Value *This, const llvm::Type *Ty) {
   Ty = Ty->getPointerTo()->getPointerTo()->getPointerTo();
@@ -356,4 +318,154 @@ CodeGenFunction::BuildVirtualCall(const CXXDestructorDecl *DD, CXXDtorType Type,
   return ::BuildVirtualCall(*this, VTableIndex, This, Ty);
 }
 
-CXXABI::~CXXABI() {}
+/// Implementation for CGCXXABI.  Possibly this should be moved into
+/// the incomplete ABI implementations?
+
+CGCXXABI::~CGCXXABI() {}
+
+static void ErrorUnsupportedABI(CodeGenFunction &CGF,
+                                llvm::StringRef S) {
+  Diagnostic &Diags = CGF.CGM.getDiags();
+  unsigned DiagID = Diags.getCustomDiagID(Diagnostic::Error,
+                                          "cannot yet compile %1 in this ABI");
+  Diags.Report(CGF.getContext().getFullLoc(CGF.CurCodeDecl->getLocation()),
+               DiagID)
+    << S;
+}
+
+static llvm::Constant *GetBogusMemberPointer(CodeGenModule &CGM,
+                                             QualType T) {
+  return llvm::Constant::getNullValue(CGM.getTypes().ConvertType(T));
+}
+
+const llvm::Type *
+CGCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
+  return CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
+}
+
+llvm::Value *CGCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
+                                                       llvm::Value *&This,
+                                                       llvm::Value *MemPtr,
+                                                 const MemberPointerType *MPT) {
+  ErrorUnsupportedABI(CGF, "calls through member pointers");
+
+  const FunctionProtoType *FPT = 
+    MPT->getPointeeType()->getAs<FunctionProtoType>();
+  const CXXRecordDecl *RD = 
+    cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
+  const llvm::FunctionType *FTy = 
+    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
+                                   FPT->isVariadic());
+  return llvm::Constant::getNullValue(FTy->getPointerTo());
+}
+
+llvm::Value *CGCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
+                                                    llvm::Value *Base,
+                                                    llvm::Value *MemPtr,
+                                              const MemberPointerType *MPT) {
+  ErrorUnsupportedABI(CGF, "loads of member pointers");
+  const llvm::Type *Ty = CGF.ConvertType(MPT->getPointeeType())->getPointerTo();
+  return llvm::Constant::getNullValue(Ty);
+}
+
+llvm::Value *CGCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
+                                                   const CastExpr *E,
+                                                   llvm::Value *Src) {
+  ErrorUnsupportedABI(CGF, "member function pointer conversions");
+  return GetBogusMemberPointer(CGM, E->getType());
+}
+
+llvm::Value *
+CGCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
+                                      llvm::Value *L,
+                                      llvm::Value *R,
+                                      const MemberPointerType *MPT,
+                                      bool Inequality) {
+  ErrorUnsupportedABI(CGF, "member function pointer comparison");
+  return CGF.Builder.getFalse();
+}
+
+llvm::Value *
+CGCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
+                                     llvm::Value *MemPtr,
+                                     const MemberPointerType *MPT) {
+  ErrorUnsupportedABI(CGF, "member function pointer null testing");
+  return CGF.Builder.getFalse();
+}
+
+llvm::Constant *
+CGCXXABI::EmitMemberPointerConversion(llvm::Constant *C, const CastExpr *E) {
+  return GetBogusMemberPointer(CGM, E->getType());
+}
+
+llvm::Constant *
+CGCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
+  return GetBogusMemberPointer(CGM, QualType(MPT, 0));
+}
+
+llvm::Constant *CGCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
+  return GetBogusMemberPointer(CGM,
+                         CGM.getContext().getMemberPointerType(MD->getType(),
+                                         MD->getParent()->getTypeForDecl()));
+}
+
+llvm::Constant *CGCXXABI::EmitMemberPointer(const FieldDecl *FD) {
+  return GetBogusMemberPointer(CGM,
+                         CGM.getContext().getMemberPointerType(FD->getType(),
+                                         FD->getParent()->getTypeForDecl()));
+}
+
+bool CGCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
+  // Fake answer.
+  return true;
+}
+
+void CGCXXABI::BuildThisParam(CodeGenFunction &CGF, FunctionArgList &Params) {
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(CGF.CurGD.getDecl());
+
+  // FIXME: I'm not entirely sure I like using a fake decl just for code
+  // generation. Maybe we can come up with a better way?
+  ImplicitParamDecl *ThisDecl
+    = ImplicitParamDecl::Create(CGM.getContext(), 0, MD->getLocation(),
+                                &CGM.getContext().Idents.get("this"),
+                                MD->getThisType(CGM.getContext()));
+  Params.push_back(std::make_pair(ThisDecl, ThisDecl->getType()));
+  getThisDecl(CGF) = ThisDecl;
+}
+
+void CGCXXABI::EmitThisParam(CodeGenFunction &CGF) {
+  /// Initialize the 'this' slot.
+  assert(getThisDecl(CGF) && "no 'this' variable for function");
+  getThisValue(CGF)
+    = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(getThisDecl(CGF)),
+                             "this");
+}
+
+void CGCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
+                                   RValue RV, QualType ResultType) {
+  CGF.EmitReturnOfRValue(RV, ResultType);
+}
+
+CharUnits CGCXXABI::GetArrayCookieSize(QualType ElementType) {
+  return CharUnits::Zero();
+}
+
+llvm::Value *CGCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+                                             llvm::Value *NewPtr,
+                                             llvm::Value *NumElements,
+                                             QualType ElementType) {
+  // Should never be called.
+  ErrorUnsupportedABI(CGF, "array cookie initialization");
+  return 0;
+}
+
+void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                               QualType ElementType, llvm::Value *&NumElements,
+                               llvm::Value *&AllocPtr, CharUnits &CookieSize) {
+  ErrorUnsupportedABI(CGF, "array cookie reading");
+
+  // This should be enough to avoid assertions.
+  NumElements = 0;
+  AllocPtr = llvm::Constant::getNullValue(CGF.Builder.getInt8PtrTy());
+  CookieSize = CharUnits::Zero();
+}
diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h
index e1bbb0a79cce..367e345918a2 100644
--- a/lib/CodeGen/CGCXXABI.h
+++ b/lib/CodeGen/CGCXXABI.h
@@ -15,23 +15,215 @@
 #ifndef CLANG_CODEGEN_CXXABI_H
 #define CLANG_CODEGEN_CXXABI_H
 
+#include "CodeGenFunction.h"
+
+namespace llvm {
+  class Constant;
+  class Type;
+  class Value;
+
+  template <class T> class SmallVectorImpl;
+}
+
 namespace clang {
+  class CastExpr;
+  class CXXConstructorDecl;
+  class CXXDestructorDecl;
+  class CXXMethodDecl;
+  class CXXRecordDecl;
+  class FieldDecl;
+
 namespace CodeGen {
+  class CodeGenFunction;
   class CodeGenModule;
   class MangleContext;
 
 /// Implements C++ ABI-specific code generation functions.
-class CXXABI {
+class CGCXXABI {
+protected:
+  CodeGenModule &CGM;
+
+  CGCXXABI(CodeGenModule &CGM) : CGM(CGM) {}
+
+protected:
+  ImplicitParamDecl *&getThisDecl(CodeGenFunction &CGF) {
+    return CGF.CXXThisDecl;
+  }
+  llvm::Value *&getThisValue(CodeGenFunction &CGF) {
+    return CGF.CXXThisValue;
+  }
+
+  ImplicitParamDecl *&getVTTDecl(CodeGenFunction &CGF) {
+    return CGF.CXXVTTDecl;
+  }
+  llvm::Value *&getVTTValue(CodeGenFunction &CGF) {
+    return CGF.CXXVTTValue;
+  }
+
+  /// Build a parameter variable suitable for 'this'.
+  void BuildThisParam(CodeGenFunction &CGF, FunctionArgList &Params);
+
+  /// Perform prolog initialization of the parameter variable suitable
+  /// for 'this' emitted by BuildThisParam.
+  void EmitThisParam(CodeGenFunction &CGF);
+
+  ASTContext &getContext() const { return CGM.getContext(); }
+
 public:
-  virtual ~CXXABI();
+
+  virtual ~CGCXXABI();
 
   /// Gets the mangle context.
   virtual MangleContext &getMangleContext() = 0;
+
+  /// Find the LLVM type used to represent the given member pointer
+  /// type.
+  virtual const llvm::Type *
+  ConvertMemberPointerType(const MemberPointerType *MPT);
+
+  /// Load a member function from an object and a member function
+  /// pointer.  Apply the this-adjustment and set 'This' to the
+  /// adjusted value.
+  virtual llvm::Value *
+  EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
+                                  llvm::Value *&This,
+                                  llvm::Value *MemPtr,
+                                  const MemberPointerType *MPT);
+
+  /// Calculate an l-value from an object and a data member pointer.
+  virtual llvm::Value *EmitMemberDataPointerAddress(CodeGenFunction &CGF,
+                                                    llvm::Value *Base,
+                                                    llvm::Value *MemPtr,
+                                            const MemberPointerType *MPT);
+
+  /// Perform a derived-to-base or base-to-derived member pointer
+  /// conversion.
+  virtual llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
+                                                   const CastExpr *E,
+                                                   llvm::Value *Src);
+
+  /// Perform a derived-to-base or base-to-derived member pointer
+  /// conversion on a constant member pointer.
+  virtual llvm::Constant *EmitMemberPointerConversion(llvm::Constant *C,
+                                                      const CastExpr *E);
+
+  /// Return true if the given member pointer can be zero-initialized
+  /// (in the C++ sense) with an LLVM zeroinitializer.
+  virtual bool isZeroInitializable(const MemberPointerType *MPT);
+
+  /// Create a null member pointer of the given type.
+  virtual llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT);
+
+  /// Create a member pointer for the given method.
+  virtual llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD);
+
+  /// Create a member pointer for the given field.
+  virtual llvm::Constant *EmitMemberPointer(const FieldDecl *FD);
+
+  /// Emit a comparison between two member pointers.  Returns an i1.
+  virtual llvm::Value *
+  EmitMemberPointerComparison(CodeGenFunction &CGF,
+                              llvm::Value *L,
+                              llvm::Value *R,
+                              const MemberPointerType *MPT,
+                              bool Inequality);
+
+  /// Determine if a member pointer is non-null.  Returns an i1.
+  virtual llvm::Value *
+  EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
+                             llvm::Value *MemPtr,
+                             const MemberPointerType *MPT);
+
+  /// Build the signature of the given constructor variant by adding
+  /// any required parameters.  For convenience, ResTy has been
+  /// initialized to 'void', and ArgTys has been initialized with the
+  /// type of 'this' (although this may be changed by the ABI) and
+  /// will have the formal parameters added to it afterwards.
+  ///
+  /// If there are ever any ABIs where the implicit parameters are
+  /// intermixed with the formal parameters, we can address those
+  /// then.
+  virtual void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                         CXXCtorType T,
+                                         CanQualType &ResTy,
+                               llvm::SmallVectorImpl<CanQualType> &ArgTys) = 0;
+
+  /// Build the signature of the given destructor variant by adding
+  /// any required parameters.  For convenience, ResTy has been
+  /// initialized to 'void' and ArgTys has been initialized with the
+  /// type of 'this' (although this may be changed by the ABI).
+  virtual void BuildDestructorSignature(const CXXDestructorDecl *Dtor,
+                                        CXXDtorType T,
+                                        CanQualType &ResTy,
+                               llvm::SmallVectorImpl<CanQualType> &ArgTys) = 0;
+
+  /// Build the ABI-specific portion of the parameter list for a
+  /// function.  This generally involves a 'this' parameter and
+  /// possibly some extra data for constructors and destructors.
+  ///
+  /// ABIs may also choose to override the return type, which has been
+  /// initialized with the formal return type of the function.
+  virtual void BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                           QualType &ResTy,
+                                           FunctionArgList &Params) = 0;
+
+  /// Emit the ABI-specific prolog for the function.
+  virtual void EmitInstanceFunctionProlog(CodeGenFunction &CGF) = 0;
+
+  virtual void EmitReturnFromThunk(CodeGenFunction &CGF,
+                                   RValue RV, QualType ResultType);
+
+  /**************************** Array cookies ******************************/
+
+  /// Returns the extra size required in order to store the array
+  /// cookie for the given type.  May return 0 to indicate that no
+  /// array cookie is required.
+  ///
+  /// Several cases are filtered out before this method is called:
+  ///   - non-array allocations never need a cookie
+  ///   - calls to ::operator new(size_t, void*) never need a cookie
+  ///
+  /// \param ElementType - the allocated type of the expression,
+  ///   i.e. the pointee type of the expression result type
+  virtual CharUnits GetArrayCookieSize(QualType ElementType);
+
+  /// Initialize the array cookie for the given allocation.
+  ///
+  /// \param NewPtr - a char* which is the presumed-non-null
+  ///   return value of the allocation function
+  /// \param NumElements - the computed number of elements,
+  ///   potentially collapsed from the multidimensional array case
+  /// \param ElementType - the base element allocated type,
+  ///   i.e. the allocated type after stripping all array types
+  virtual llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
+                                             llvm::Value *NewPtr,
+                                             llvm::Value *NumElements,
+                                             QualType ElementType);
+
+  /// Reads the array cookie associated with the given pointer,
+  /// if it has one.
+  ///
+  /// \param Ptr - a pointer to the first element in the array
+  /// \param ElementType - the base element type of elements of the array
+  /// \param NumElements - an out parameter which will be initialized
+  ///   with the number of elements allocated, or zero if there is no
+  ///   cookie
+  /// \param AllocPtr - an out parameter which will be initialized
+  ///   with a char* pointing to the address returned by the allocation
+  ///   function
+  /// \param CookieSize - an out parameter which will be initialized
+  ///   with the size of the cookie, or zero if there is no cookie
+  virtual void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                               QualType ElementType, llvm::Value *&NumElements,
+                               llvm::Value *&AllocPtr, CharUnits &CookieSize);
+
 };
 
 /// Creates an instance of a C++ ABI class.
-CXXABI *CreateItaniumCXXABI(CodeGenModule &CGM);
-CXXABI *CreateMicrosoftCXXABI(CodeGenModule &CGM);
+CGCXXABI *CreateARMCXXABI(CodeGenModule &CGM);
+CGCXXABI *CreateItaniumCXXABI(CodeGenModule &CGM);
+CGCXXABI *CreateMicrosoftCXXABI(CodeGenModule &CGM);
+
 }
 }
 
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 3d1e143dca2b..475dfa5c102a 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CGCall.h"
+#include "CGCXXABI.h"
 #include "ABIInfo.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
@@ -35,6 +36,7 @@ static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
   case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
   case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
   case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
+  // TODO: add support for CC_X86Pascal to llvm
   }
 }
 
@@ -99,6 +101,9 @@ static CallingConv getCallingConventionForDecl(const Decl *D) {
   if (D->hasAttr<ThisCallAttr>())
     return CC_X86ThisCall;
 
+  if (D->hasAttr<PascalAttr>())
+    return CC_X86Pascal;
+
   return CC_C;
 }
 
@@ -116,6 +121,9 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
   llvm::SmallVector<CanQualType, 16> ArgTys;
 
+  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for contructors!");
+  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
+
   // Add the 'this' pointer unless this is a static method.
   if (MD->isInstance())
     ArgTys.push_back(GetThisType(Context, MD->getParent()));
@@ -126,29 +134,32 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D, 
                                                     CXXCtorType Type) {
   llvm::SmallVector<CanQualType, 16> ArgTys;
-
-  // Add the 'this' pointer.
   ArgTys.push_back(GetThisType(Context, D->getParent()));
+  CanQualType ResTy = Context.VoidTy;
 
-  // Check if we need to add a VTT parameter (which has type void **).
-  if (Type == Ctor_Base && D->getParent()->getNumVBases() != 0)
-    ArgTys.push_back(Context.getPointerType(Context.VoidPtrTy));
+  TheCXXABI.BuildConstructorSignature(D, Type, ResTy, ArgTys);
 
-  return ::getFunctionInfo(*this, ArgTys, GetFormalType(D));
+  CanQual<FunctionProtoType> FTP = GetFormalType(D);
+
+  // Add the formal parameters.
+  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
+    ArgTys.push_back(FTP->getArgType(i));
+
+  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
                                                     CXXDtorType Type) {
-  llvm::SmallVector<CanQualType, 16> ArgTys;
-  
-  // Add the 'this' pointer.
+  llvm::SmallVector<CanQualType, 2> ArgTys;
   ArgTys.push_back(GetThisType(Context, D->getParent()));
-  
-  // Check if we need to add a VTT parameter (which has type void **).
-  if (Type == Dtor_Base && D->getParent()->getNumVBases() != 0)
-    ArgTys.push_back(Context.getPointerType(Context.VoidPtrTy));
+  CanQualType ResTy = Context.VoidTy;
+
+  TheCXXABI.BuildDestructorSignature(D, Type, ResTy, ArgTys);
 
-  return ::getFunctionInfo(*this, ArgTys, GetFormalType(D));
+  CanQual<FunctionProtoType> FTP = GetFormalType(D);
+  assert(FTP->getNumArgs() == 0 && "dtor with formal parameters");
+
+  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
 }
 
 const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
@@ -243,34 +254,26 @@ const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
                           ArgTys.data(), ArgTys.size());
   FunctionInfos.InsertNode(FI, InsertPos);
 
-  // ABI lowering wants to know what our preferred type for the argument is in
-  // various situations, pass it in.
-  llvm::SmallVector<const llvm::Type *, 8> PreferredArgTypes;
-  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
-       I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I) {
-    // If this is being called from the guts of the ConvertType loop, make sure
-    // to call ConvertTypeRecursive so we don't get into issues with cyclic
-    // pointer type structures.
-    PreferredArgTypes.push_back(ConvertTypeRecursive(*I));
-  }
-  
   // Compute ABI information.
-  getABIInfo().computeInfo(*FI, getContext(), TheModule.getContext(),
-                           PreferredArgTypes.data(), PreferredArgTypes.size());
+  getABIInfo().computeInfo(*FI);
+  
+  // Loop over all of the computed argument and return value info.  If any of
+  // them are direct or extend without a specified coerce type, specify the
+  // default now.
+  ABIArgInfo &RetInfo = FI->getReturnInfo();
+  if (RetInfo.canHaveCoerceToType() && RetInfo.getCoerceToType() == 0)
+    RetInfo.setCoerceToType(ConvertTypeRecursive(FI->getReturnType()));
+  
+  for (CGFunctionInfo::arg_iterator I = FI->arg_begin(), E = FI->arg_end();
+       I != E; ++I)
+    if (I->info.canHaveCoerceToType() && I->info.getCoerceToType() == 0)
+      I->info.setCoerceToType(ConvertTypeRecursive(I->type));
 
   // If this is a top-level call and ConvertTypeRecursive hit unresolved pointer
   // types, resolve them now.  These pointers may point to this function, which
   // we *just* filled in the FunctionInfo for.
-  if (!IsRecursive && !PointersToResolve.empty()) {
-    // Use PATypeHolder's so that our preferred types don't dangle under
-    // refinement.
-    llvm::SmallVector<llvm::PATypeHolder, 8> Handles(PreferredArgTypes.begin(),
-                                                     PreferredArgTypes.end());
+  if (!IsRecursive && !PointersToResolve.empty())
     HandleLateResolvedPointers();
-    PreferredArgTypes.clear();
-    PreferredArgTypes.append(Handles.begin(), Handles.end());
-  }
-  
   
   return *FI;
 }
@@ -311,11 +314,10 @@ void CodeGenTypes::GetExpandedTypes(QualType Ty,
            "Cannot expand structure with bit-field members.");
 
     QualType FT = FD->getType();
-    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
+    if (CodeGenFunction::hasAggregateLLVMType(FT))
       GetExpandedTypes(FT, ArgTys, IsRecursive);
-    } else {
+    else
       ArgTys.push_back(ConvertType(FT, IsRecursive));
-    }
   }
 }
 
@@ -613,7 +615,7 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
 
   case ABIArgInfo::Extend:
   case ABIArgInfo::Direct:
-    ResultType = ConvertType(RetTy, IsRecursive);
+    ResultType = RetAI.getCoerceToType();
     break;
 
   case ABIArgInfo::Indirect: {
@@ -627,10 +629,6 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
   case ABIArgInfo::Ignore:
     ResultType = llvm::Type::getVoidTy(getLLVMContext());
     break;
-
-  case ABIArgInfo::Coerce:
-    ResultType = RetAI.getCoerceToType();
-    break;
   }
 
   for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
@@ -641,7 +639,15 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
     case ABIArgInfo::Ignore:
       break;
 
-    case ABIArgInfo::Coerce: {
+    case ABIArgInfo::Indirect: {
+      // indirect arguments are always on the stack, which is addr space #0.
+      const llvm::Type *LTy = ConvertTypeForMem(it->type, IsRecursive);
+      ArgTys.push_back(llvm::PointerType::getUnqual(LTy));
+      break;
+    }
+
+    case ABIArgInfo::Extend:
+    case ABIArgInfo::Direct: {
       // If the coerce-to type is a first class aggregate, flatten it.  Either
       // way is semantically identical, but fast-isel and the optimizer
       // generally likes scalar values better than FCAs.
@@ -655,18 +661,6 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
       break;
     }
 
-    case ABIArgInfo::Indirect: {
-      // indirect arguments are always on the stack, which is addr space #0.
-      const llvm::Type *LTy = ConvertTypeForMem(it->type, IsRecursive);
-      ArgTys.push_back(llvm::PointerType::getUnqual(LTy));
-      break;
-    }
-
-    case ABIArgInfo::Extend:
-    case ABIArgInfo::Direct:
-      ArgTys.push_back(ConvertType(it->type, IsRecursive));
-      break;
-
     case ABIArgInfo::Expand:
       GetExpandedTypes(it->type, ArgTys, IsRecursive);
       break;
@@ -676,12 +670,18 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic,
   return llvm::FunctionType::get(ResultType, ArgTys, IsVariadic);
 }
 
-const llvm::Type *
-CodeGenTypes::GetFunctionTypeForVTable(const CXXMethodDecl *MD) {
+const llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
   const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
   
-  if (!VerifyFuncTypeComplete(FPT))
-    return GetFunctionType(getFunctionInfo(MD), FPT->isVariadic(), false);
+  if (!VerifyFuncTypeComplete(FPT)) {
+    const CGFunctionInfo *Info;
+    if (isa<CXXDestructorDecl>(MD))
+      Info = &getFunctionInfo(cast<CXXDestructorDecl>(MD), GD.getDtorType());
+    else
+      Info = &getFunctionInfo(MD);
+    return GetFunctionType(*Info, FPT->isVariadic(), false);
+  }
 
   return llvm::OpaqueType::get(getLLVMContext());
 }
@@ -730,13 +730,13 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
   const ABIArgInfo &RetAI = FI.getReturnInfo();
   switch (RetAI.getKind()) {
   case ABIArgInfo::Extend:
-   if (RetTy->isSignedIntegerType()) {
+   if (RetTy->hasSignedIntegerRepresentation())
      RetAttrs |= llvm::Attribute::SExt;
-   } else if (RetTy->isUnsignedIntegerType()) {
+   else if (RetTy->hasUnsignedIntegerRepresentation())
      RetAttrs |= llvm::Attribute::ZExt;
-   }
-   // FALLTHROUGH
+    break;
   case ABIArgInfo::Direct:
+  case ABIArgInfo::Ignore:
     break;
 
   case ABIArgInfo::Indirect:
@@ -748,10 +748,6 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
                    llvm::Attribute::ReadNone);
     break;
 
-  case ABIArgInfo::Ignore:
-  case ABIArgInfo::Coerce:
-    break;
-
   case ABIArgInfo::Expand:
     assert(0 && "Invalid ABI kind for return argument");
   }
@@ -759,7 +755,7 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
   if (RetAttrs)
     PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
 
-  // FIXME: we need to honour command line settings also...
+  // FIXME: we need to honor command line settings also.
   // FIXME: RegParm should be reduced in case of nested functions and/or global
   // register variable.
   signed RegParm = FI.getRegParm();
@@ -774,15 +770,27 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
     // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
     // have the corresponding parameter variable.  It doesn't make
     // sense to do it here because parameters are so fucked up.
-
     switch (AI.getKind()) {
-    case ABIArgInfo::Coerce:
+    case ABIArgInfo::Extend:
+      if (ParamType->isSignedIntegerType())
+        Attributes |= llvm::Attribute::SExt;
+      else if (ParamType->isUnsignedIntegerType())
+        Attributes |= llvm::Attribute::ZExt;
+      // FALL THROUGH
+    case ABIArgInfo::Direct:
+      if (RegParm > 0 &&
+          (ParamType->isIntegerType() || ParamType->isPointerType())) {
+        RegParm -=
+        (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
+        if (RegParm >= 0)
+          Attributes |= llvm::Attribute::InReg;
+      }
+      // FIXME: handle sseregparm someday...
+        
       if (const llvm::StructType *STy =
-          dyn_cast<llvm::StructType>(AI.getCoerceToType()))
-        Index += STy->getNumElements();
-      else
-        ++Index;
-      continue;  // Skip index increment.
+            dyn_cast<llvm::StructType>(AI.getCoerceToType()))
+        Index += STy->getNumElements()-1;  // 1 will be added below.
+      break;
 
     case ABIArgInfo::Indirect:
       if (AI.getIndirectByVal())
@@ -795,24 +803,6 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
                      llvm::Attribute::ReadNone);
       break;
 
-    case ABIArgInfo::Extend:
-     if (ParamType->isSignedIntegerType()) {
-       Attributes |= llvm::Attribute::SExt;
-     } else if (ParamType->isUnsignedIntegerType()) {
-       Attributes |= llvm::Attribute::ZExt;
-     }
-     // FALLS THROUGH
-    case ABIArgInfo::Direct:
-      if (RegParm > 0 &&
-          (ParamType->isIntegerType() || ParamType->isPointerType())) {
-        RegParm -=
-          (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
-        if (RegParm >= 0)
-          Attributes |= llvm::Attribute::InReg;
-      }
-      // FIXME: handle sseregparm someday...
-      break;
-
     case ABIArgInfo::Ignore:
       // Skip increment, no matching LLVM parameter.
       continue;
@@ -881,7 +871,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         // reference.
       } else {
         // Load scalar value from indirect argument.
-        V = EmitLoadOfScalar(V, false, Ty);
+        unsigned Alignment = getContext().getTypeAlignInChars(Ty).getQuantity();
+        V = EmitLoadOfScalar(V, false, Alignment, Ty);
         if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
           // This must be a promotion, for something like
           // "void a(x) short x; {..."
@@ -894,15 +885,13 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
 
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
-      assert(AI != Fn->arg_end() && "Argument mismatch!");
-      llvm::Value *V = AI;
-      if (hasAggregateLLVMType(Ty)) {
-        // Create a temporary alloca to hold the argument; the rest of
-        // codegen expects to access aggregates & complex values by
-        // reference.
-        V = CreateMemTemp(Ty);
-        Builder.CreateStore(AI, V);
-      } else {
+      // If we have the trivial case, handle it with no muss and fuss.
+      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
+          ArgI.getCoerceToType() == ConvertType(Ty) &&
+          ArgI.getDirectOffset() == 0) {
+        assert(AI != Fn->arg_end() && "Argument mismatch!");
+        llvm::Value *V = AI;
+        
         if (Arg->getType().isRestrictQualified())
           AI->addAttr(llvm::Attribute::NoAlias);
 
@@ -911,53 +900,36 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
           // "void a(x) short x; {..."
           V = EmitScalarConversion(V, Ty, Arg->getType());
         }
+        EmitParmDecl(*Arg, V);
+        break;
       }
-      EmitParmDecl(*Arg, V);
-      break;
-    }
-
-    case ABIArgInfo::Expand: {
-      // If this structure was expanded into multiple arguments then
-      // we need to create a temporary and reconstruct it from the
-      // arguments.
-      llvm::Value *Temp = CreateMemTemp(Ty, Arg->getName() + ".addr");
-      // FIXME: What are the right qualifiers here?
-      llvm::Function::arg_iterator End =
-        ExpandTypeFromArgs(Ty, LValue::MakeAddr(Temp, Qualifiers()), AI);
-      EmitParmDecl(*Arg, Temp);
-
-      // Name the arguments used in expansion and increment AI.
-      unsigned Index = 0;
-      for (; AI != End; ++AI, ++Index)
-        AI->setName(Arg->getName() + "." + llvm::Twine(Index));
-      continue;
-    }
-
-    case ABIArgInfo::Ignore:
-      // Initialize the local variable appropriately.
-      if (hasAggregateLLVMType(Ty)) {
-        EmitParmDecl(*Arg, CreateMemTemp(Ty));
-      } else {
-        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())));
-      }
-
-      // Skip increment, no matching LLVM parameter.
-      continue;
 
-    case ABIArgInfo::Coerce: {
-      // FIXME: This is very wasteful; EmitParmDecl is just going to drop the
-      // result in a new alloca anyway, so we could just store into that
-      // directly if we broke the abstraction down more.
       llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
-      Alloca->setAlignment(getContext().getDeclAlign(Arg).getQuantity());
+      
+      // The alignment we need to use is the max of the requested alignment for
+      // the argument plus the alignment required by our access code below.
+      unsigned AlignmentToUse = 
+        CGF.CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
+      AlignmentToUse = std::max(AlignmentToUse,
+                        (unsigned)getContext().getDeclAlign(Arg).getQuantity());
+      
+      Alloca->setAlignment(AlignmentToUse);
       llvm::Value *V = Alloca;
+      llvm::Value *Ptr = V;    // Pointer to store into.
+      
+      // If the value is offset in memory, apply the offset now.
+      if (unsigned Offs = ArgI.getDirectOffset()) {
+        Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
+        Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
+        Ptr = Builder.CreateBitCast(Ptr, 
+                          llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
+      }
       
       // If the coerce-to type is a first class aggregate, we flatten it and
       // pass the elements. Either way is semantically identical, but fast-isel
       // and the optimizer generally likes scalar values better than FCAs.
       if (const llvm::StructType *STy =
             dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
-        llvm::Value *Ptr = V;
         Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
         
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
@@ -970,13 +942,13 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         // Simple case, just do a coerced store of the argument into the alloca.
         assert(AI != Fn->arg_end() && "Argument mismatch!");
         AI->setName(Arg->getName() + ".coerce");
-        CreateCoercedStore(AI++, V, /*DestIsVolatile=*/false, *this);
+        CreateCoercedStore(AI++, Ptr, /*DestIsVolatile=*/false, *this);
       }
       
       
       // Match to what EmitParmDecl is expecting for this type.
       if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
-        V = EmitLoadOfScalar(V, false, Ty);
+        V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty);
         if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
           // This must be a promotion, for something like
           // "void a(x) short x; {..."
@@ -986,6 +958,32 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       EmitParmDecl(*Arg, V);
       continue;  // Skip ++AI increment, already done.
     }
+
+    case ABIArgInfo::Expand: {
+      // If this structure was expanded into multiple arguments then
+      // we need to create a temporary and reconstruct it from the
+      // arguments.
+      llvm::Value *Temp = CreateMemTemp(Ty, Arg->getName() + ".addr");
+      llvm::Function::arg_iterator End =
+        ExpandTypeFromArgs(Ty, MakeAddrLValue(Temp, Ty), AI);
+      EmitParmDecl(*Arg, Temp);
+
+      // Name the arguments used in expansion and increment AI.
+      unsigned Index = 0;
+      for (; AI != End; ++AI, ++Index)
+        AI->setName(Arg->getName() + "." + llvm::Twine(Index));
+      continue;
+    }
+
+    case ABIArgInfo::Ignore:
+      // Initialize the local variable appropriately.
+      if (hasAggregateLLVMType(Ty))
+        EmitParmDecl(*Arg, CreateMemTemp(Ty));
+      else
+        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())));
+
+      // Skip increment, no matching LLVM parameter.
+      continue;
     }
 
     ++AI;
@@ -1000,13 +998,14 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
     return;
   }
 
-  llvm::MDNode *RetDbgInfo = 0;
+  llvm::DebugLoc RetDbgLoc;
   llvm::Value *RV = 0;
   QualType RetTy = FI.getReturnType();
   const ABIArgInfo &RetAI = FI.getReturnInfo();
 
   switch (RetAI.getKind()) {
-  case ABIArgInfo::Indirect:
+  case ABIArgInfo::Indirect: {
+    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
     if (RetTy->isAnyComplexType()) {
       ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
       StoreComplexToAddr(RT, CurFn->arg_begin(), false);
@@ -1014,43 +1013,54 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
       // Do nothing; aggregrates get evaluated directly into the destination.
     } else {
       EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
-                        false, RetTy);
+                        false, Alignment, RetTy);
     }
     break;
+  }
 
   case ABIArgInfo::Extend:
-  case ABIArgInfo::Direct: {
-    // The internal return value temp always will have pointer-to-return-type
-    // type, just do a load.
-      
-    // If the instruction right before the insertion point is a store to the
-    // return value, we can elide the load, zap the store, and usually zap the
-    // alloca.
-    llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
-    llvm::StoreInst *SI = 0;
-    if (InsertBB->empty() || 
-        !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
-        SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
-      RV = Builder.CreateLoad(ReturnValue);
+  case ABIArgInfo::Direct:
+    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
+        RetAI.getDirectOffset() == 0) {
+      // The internal return value temp always will have pointer-to-return-type
+      // type, just do a load.
+        
+      // If the instruction right before the insertion point is a store to the
+      // return value, we can elide the load, zap the store, and usually zap the
+      // alloca.
+      llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
+      llvm::StoreInst *SI = 0;
+      if (InsertBB->empty() || 
+          !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
+          SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
+        RV = Builder.CreateLoad(ReturnValue);
+      } else {
+        // Get the stored value and nuke the now-dead store.
+        RetDbgLoc = SI->getDebugLoc();
+        RV = SI->getValueOperand();
+        SI->eraseFromParent();
+        
+        // If that was the only use of the return value, nuke it as well now.
+        if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
+          cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
+          ReturnValue = 0;
+        }
+      }
     } else {
-      // Get the stored value and nuke the now-dead store.
-      RetDbgInfo = SI->getDbgMetadata();
-      RV = SI->getValueOperand();
-      SI->eraseFromParent();
-      
-      // If that was the only use of the return value, nuke it as well now.
-      if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
-        cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
-        ReturnValue = 0;
+      llvm::Value *V = ReturnValue;
+      // If the value is offset in memory, apply the offset now.
+      if (unsigned Offs = RetAI.getDirectOffset()) {
+        V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
+        V = Builder.CreateConstGEP1_32(V, Offs);
+        V = Builder.CreateBitCast(V, 
+                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
       }
+      
+      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
     }
     break;
-  }
-  case ABIArgInfo::Ignore:
-    break;
 
-  case ABIArgInfo::Coerce:
-    RV = CreateCoercedLoad(ReturnValue, RetAI.getCoerceToType(), *this);
+  case ABIArgInfo::Ignore:
     break;
 
   case ABIArgInfo::Expand:
@@ -1058,8 +1068,8 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
   }
 
   llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
-  if (RetDbgInfo)
-    Ret->setDbgMetadata(RetDbgInfo);
+  if (!RetDbgLoc.isUnknown())
+    Ret->setDebugLoc(RetDbgLoc);
 }
 
 RValue CodeGenFunction::EmitDelegateCallArg(const VarDecl *Param) {
@@ -1089,7 +1099,8 @@ RValue CodeGenFunction::EmitDelegateCallArg(const VarDecl *Param) {
   if (hasAggregateLLVMType(ArgType))
     return RValue::getAggregate(Local);
 
-  return RValue::get(EmitLoadOfScalar(Local, false, ArgType));
+  unsigned Alignment = getContext().getDeclAlign(Param).getQuantity();
+  return RValue::get(EmitLoadOfScalar(Local, false, Alignment, ArgType));
 }
 
 RValue CodeGenFunction::EmitCallArg(const Expr *E, QualType ArgType) {
@@ -1149,49 +1160,60 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     const ABIArgInfo &ArgInfo = info_it->info;
     RValue RV = I->first;
 
+    unsigned Alignment =
+      getContext().getTypeAlignInChars(I->second).getQuantity();
     switch (ArgInfo.getKind()) {
-    case ABIArgInfo::Indirect:
+    case ABIArgInfo::Indirect: {
       if (RV.isScalar() || RV.isComplex()) {
         // Make a temporary alloca to pass the argument.
         Args.push_back(CreateMemTemp(I->second));
         if (RV.isScalar())
-          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false, I->second);
+          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false,
+                            Alignment, I->second);
         else
           StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
       } else {
         Args.push_back(RV.getAggregateAddr());
       }
       break;
-
-    case ABIArgInfo::Extend:
-    case ABIArgInfo::Direct:
-      if (RV.isScalar()) {
-        Args.push_back(RV.getScalarVal());
-      } else if (RV.isComplex()) {
-        llvm::Value *Tmp = llvm::UndefValue::get(ConvertType(I->second));
-        Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().first, 0);
-        Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().second, 1);
-        Args.push_back(Tmp);
-      } else {
-        Args.push_back(Builder.CreateLoad(RV.getAggregateAddr()));
-      }
-      break;
+    }
 
     case ABIArgInfo::Ignore:
       break;
+        
+    case ABIArgInfo::Extend:
+    case ABIArgInfo::Direct: {
+      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
+          ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
+          ArgInfo.getDirectOffset() == 0) {
+        if (RV.isScalar())
+          Args.push_back(RV.getScalarVal());
+        else
+          Args.push_back(Builder.CreateLoad(RV.getAggregateAddr()));
+        break;
+      }
 
-    case ABIArgInfo::Coerce: {
       // FIXME: Avoid the conversion through memory if possible.
       llvm::Value *SrcPtr;
       if (RV.isScalar()) {
         SrcPtr = CreateMemTemp(I->second, "coerce");
-        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, I->second);
+        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, Alignment,
+                          I->second);
       } else if (RV.isComplex()) {
         SrcPtr = CreateMemTemp(I->second, "coerce");
         StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
       } else
         SrcPtr = RV.getAggregateAddr();
       
+      // If the value is offset in memory, apply the offset now.
+      if (unsigned Offs = ArgInfo.getDirectOffset()) {
+        SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
+        SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
+        SrcPtr = Builder.CreateBitCast(SrcPtr, 
+                       llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
+
+      }
+      
       // If the coerce-to type is a first class aggregate, we flatten it and
       // pass the elements. Either way is semantically identical, but fast-isel
       // and the optimizer generally likes scalar values better than FCAs.
@@ -1201,7 +1223,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                        llvm::PointerType::getUnqual(STy));
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
-          Args.push_back(Builder.CreateLoad(EltPtr));
+          llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
+          // We don't know what we're loading from.
+          LI->setAlignment(1);
+          Args.push_back(LI);
         }
       } else {
         // In the simple case, just pass the coerced loaded value.
@@ -1294,39 +1319,43 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     CI->setName("call");
 
   switch (RetAI.getKind()) {
-  case ABIArgInfo::Indirect:
+  case ABIArgInfo::Indirect: {
+    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
     if (RetTy->isAnyComplexType())
       return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
     if (CodeGenFunction::hasAggregateLLVMType(RetTy))
       return RValue::getAggregate(Args[0]);
-    return RValue::get(EmitLoadOfScalar(Args[0], false, RetTy));
-
-  case ABIArgInfo::Extend:
-  case ABIArgInfo::Direct:
-    if (RetTy->isAnyComplexType()) {
-      llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
-      llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
-      return RValue::getComplex(std::make_pair(Real, Imag));
-    }
-    if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
-      llvm::Value *DestPtr = ReturnValue.getValue();
-      bool DestIsVolatile = ReturnValue.isVolatile();
-
-      if (!DestPtr) {
-        DestPtr = CreateMemTemp(RetTy, "agg.tmp");
-        DestIsVolatile = false;
-      }
-      Builder.CreateStore(CI, DestPtr, DestIsVolatile);
-      return RValue::getAggregate(DestPtr);
-    }
-    return RValue::get(CI);
+    return RValue::get(EmitLoadOfScalar(Args[0], false, Alignment, RetTy));
+  }
 
   case ABIArgInfo::Ignore:
     // If we are ignoring an argument that had a result, make sure to
     // construct the appropriate return value for our caller.
     return GetUndefRValue(RetTy);
+    
+  case ABIArgInfo::Extend:
+  case ABIArgInfo::Direct: {
+    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
+        RetAI.getDirectOffset() == 0) {
+      if (RetTy->isAnyComplexType()) {
+        llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
+        llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
+        return RValue::getComplex(std::make_pair(Real, Imag));
+      }
+      if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+        llvm::Value *DestPtr = ReturnValue.getValue();
+        bool DestIsVolatile = ReturnValue.isVolatile();
 
-  case ABIArgInfo::Coerce: {
+        if (!DestPtr) {
+          DestPtr = CreateMemTemp(RetTy, "agg.tmp");
+          DestIsVolatile = false;
+        }
+        Builder.CreateStore(CI, DestPtr, DestIsVolatile);
+        return RValue::getAggregate(DestPtr);
+      }
+      return RValue::get(CI);
+    }
+      
     llvm::Value *DestPtr = ReturnValue.getValue();
     bool DestIsVolatile = ReturnValue.isVolatile();
     
@@ -1335,12 +1364,22 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       DestIsVolatile = false;
     }
     
-    CreateCoercedStore(CI, DestPtr, DestIsVolatile, *this);
+    // If the value is offset in memory, apply the offset now.
+    llvm::Value *StorePtr = DestPtr;
+    if (unsigned Offs = RetAI.getDirectOffset()) {
+      StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
+      StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
+      StorePtr = Builder.CreateBitCast(StorePtr, 
+                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
+    }
+    CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
+    
+    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
     if (RetTy->isAnyComplexType())
       return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
     if (CodeGenFunction::hasAggregateLLVMType(RetTy))
       return RValue::getAggregate(DestPtr);
-    return RValue::get(EmitLoadOfScalar(DestPtr, false, RetTy));
+    return RValue::get(EmitLoadOfScalar(DestPtr, false, Alignment, RetTy));
   }
 
   case ABIArgInfo::Expand:
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index c50fe90f8a81..bf26799b83f3 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/RecordLayout.h"
@@ -22,13 +23,13 @@ using namespace CodeGen;
 static uint64_t 
 ComputeNonVirtualBaseClassOffset(ASTContext &Context, 
                                  const CXXRecordDecl *DerivedClass,
-                                 CXXBaseSpecifierArray::iterator Start,
-                                 CXXBaseSpecifierArray::iterator End) {
+                                 CastExpr::path_const_iterator Start,
+                                 CastExpr::path_const_iterator End) {
   uint64_t Offset = 0;
   
   const CXXRecordDecl *RD = DerivedClass;
   
-  for (CXXBaseSpecifierArray::iterator I = Start; I != End; ++I) {
+  for (CastExpr::path_const_iterator I = Start; I != End; ++I) {
     const CXXBaseSpecifier *Base = *I;
     assert(!Base->isVirtual() && "Should not see virtual bases here!");
 
@@ -50,12 +51,13 @@ ComputeNonVirtualBaseClassOffset(ASTContext &Context,
 
 llvm::Constant *
 CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
-                                        const CXXBaseSpecifierArray &BasePath) {
-  assert(!BasePath.empty() && "Base path should not be empty!");
+                                   CastExpr::path_const_iterator PathBegin,
+                                   CastExpr::path_const_iterator PathEnd) {
+  assert(PathBegin != PathEnd && "Base path should not be empty!");
 
   uint64_t Offset = 
-    ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl, 
-                                     BasePath.begin(), BasePath.end());
+    ComputeNonVirtualBaseClassOffset(getContext(), ClassDecl,
+                                     PathBegin, PathEnd);
   if (!Offset)
     return 0;
   
@@ -131,11 +133,12 @@ ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, llvm::Value *ThisPtr,
 llvm::Value *
 CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value, 
                                        const CXXRecordDecl *Derived,
-                                       const CXXBaseSpecifierArray &BasePath, 
+                                       CastExpr::path_const_iterator PathBegin,
+                                       CastExpr::path_const_iterator PathEnd,
                                        bool NullCheckValue) {
-  assert(!BasePath.empty() && "Base path should not be empty!");
+  assert(PathBegin != PathEnd && "Base path should not be empty!");
 
-  CXXBaseSpecifierArray::iterator Start = BasePath.begin();
+  CastExpr::path_const_iterator Start = PathBegin;
   const CXXRecordDecl *VBase = 0;
   
   // Get the virtual base.
@@ -147,11 +150,11 @@ CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
   
   uint64_t NonVirtualOffset = 
     ComputeNonVirtualBaseClassOffset(getContext(), VBase ? VBase : Derived,
-                                     Start, BasePath.end());
+                                     Start, PathEnd);
 
   // Get the base pointer type.
   const llvm::Type *BasePtrTy = 
-    ConvertType((BasePath.end()[-1])->getType())->getPointerTo();
+    ConvertType((PathEnd[-1])->getType())->getPointerTo();
   
   if (!NonVirtualOffset && !VBase) {
     // Just cast back.
@@ -206,16 +209,17 @@ CodeGenFunction::GetAddressOfBaseClass(llvm::Value *Value,
 llvm::Value *
 CodeGenFunction::GetAddressOfDerivedClass(llvm::Value *Value,
                                           const CXXRecordDecl *Derived,
-                                          const CXXBaseSpecifierArray &BasePath,
+                                        CastExpr::path_const_iterator PathBegin,
+                                          CastExpr::path_const_iterator PathEnd,
                                           bool NullCheckValue) {
-  assert(!BasePath.empty() && "Base path should not be empty!");
+  assert(PathBegin != PathEnd && "Base path should not be empty!");
 
   QualType DerivedTy =
     getContext().getCanonicalType(getContext().getTagDeclType(Derived));
   const llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo();
   
   llvm::Value *NonVirtualOffset =
-    CGM.GetNonVirtualBaseClassOffset(Derived, BasePath);
+    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
   
   if (!NonVirtualOffset) {
     // No offset, we can just cast back.
@@ -310,6 +314,28 @@ static llvm::Value *GetVTTParameter(CodeGenFunction &CGF, GlobalDecl GD,
   return VTT;
 }
 
+namespace {
+  /// Call the destructor for a direct base class.
+  struct CallBaseDtor : EHScopeStack::Cleanup {
+    const CXXRecordDecl *BaseClass;
+    bool BaseIsVirtual;
+    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
+      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      const CXXRecordDecl *DerivedClass =
+        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
+
+      const CXXDestructorDecl *D = BaseClass->getDestructor();
+      llvm::Value *Addr = 
+        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThis(),
+                                                  DerivedClass, BaseClass,
+                                                  BaseIsVirtual);
+      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual, Addr);
+    }
+  };
+}
+
 static void EmitBaseInitializer(CodeGenFunction &CGF, 
                                 const CXXRecordDecl *ClassDecl,
                                 CXXBaseOrMemberInitializer *BaseInit,
@@ -333,18 +359,14 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
   // virtual bases, and we only do virtual bases for complete ctors.
   llvm::Value *V = 
     CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
-                                              BaseClassDecl, 
-                                              BaseInit->isBaseVirtual());
+                                              BaseClassDecl,
+                                              isBaseVirtual);
 
   CGF.EmitAggExpr(BaseInit->getInit(), V, false, false, true);
   
-  if (CGF.Exceptions && !BaseClassDecl->hasTrivialDestructor()) {
-    // FIXME: Is this OK for C++0x delegating constructors?
-    CodeGenFunction::CleanupBlock Cleanup(CGF, EHCleanup);
-
-    CXXDestructorDecl *DD = BaseClassDecl->getDestructor();
-    CGF.EmitCXXDestructorCall(DD, Dtor_Base, isBaseVirtual, V);
-  }
+  if (CGF.Exceptions && !BaseClassDecl->hasTrivialDestructor())
+    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
+                                          isBaseVirtual);
 }
 
 static void EmitAggMemberInitializer(CodeGenFunction &CGF,
@@ -432,6 +454,25 @@ static void EmitAggMemberInitializer(CodeGenFunction &CGF,
   // Emit the fall-through block.
   CGF.EmitBlock(AfterFor, true);
 }
+
+namespace {
+  struct CallMemberDtor : EHScopeStack::Cleanup {
+    FieldDecl *Field;
+    CXXDestructorDecl *Dtor;
+
+    CallMemberDtor(FieldDecl *Field, CXXDestructorDecl *Dtor)
+      : Field(Field), Dtor(Dtor) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      // FIXME: Is this OK for C++0x delegating constructors?
+      llvm::Value *ThisPtr = CGF.LoadCXXThis();
+      LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 0);
+
+      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
+                                LHS.getAddress());
+    }
+  };
+}
   
 static void EmitMemberInitializer(CodeGenFunction &CGF,
                                   const CXXRecordDecl *ClassDecl,
@@ -487,7 +528,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
       BasePtr = llvm::PointerType::getUnqual(BasePtr);
       llvm::Value *BaseAddrPtr = CGF.Builder.CreateBitCast(LHS.getAddress(), 
                                                            BasePtr);
-      LHS = LValue::MakeAddr(BaseAddrPtr, CGF.MakeQualifiers(BaseElementTy));
+      LHS = CGF.MakeAddrLValue(BaseAddrPtr, BaseElementTy);
       
       // Create an array index that will be used to walk over all of the
       // objects we're constructing.
@@ -532,17 +573,9 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
       return;
     
     CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-    if (!RD->hasTrivialDestructor()) {
-      // FIXME: Is this OK for C++0x delegating constructors?
-      CodeGenFunction::CleanupBlock Cleanup(CGF, EHCleanup);
-      
-      llvm::Value *ThisPtr = CGF.LoadCXXThis();
-      LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 0);
-
-      CXXDestructorDecl *DD = RD->getDestructor();
-      CGF.EmitCXXDestructorCall(DD, Dtor_Complete, /*ForVirtualBase=*/false,
-                                LHS.getAddress());
-    }
+    if (!RD->hasTrivialDestructor())
+      CGF.EHStack.pushCleanup<CallMemberDtor>(EHCleanup, Field,
+                                              RD->getDestructor());
   }
 }
 
@@ -598,6 +631,8 @@ void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
   // Before we go any further, try the complete->base constructor
   // delegation optimization.
   if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor)) {
+    if (CGDebugInfo *DI = getDebugInfo()) 
+      DI->EmitStopPoint(Builder);
     EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args);
     return;
   }
@@ -663,113 +698,158 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
   const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
   CXXDtorType DtorType = CurGD.getDtorType();
 
+  // The call to operator delete in a deleting destructor happens
+  // outside of the function-try-block, which means it's always
+  // possible to delegate the destructor body to the complete
+  // destructor.  Do so.
+  if (DtorType == Dtor_Deleting) {
+    EnterDtorCleanups(Dtor, Dtor_Deleting);
+    EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
+                          LoadCXXThis());
+    PopCleanupBlock();
+    return;
+  }
+
   Stmt *Body = Dtor->getBody();
 
   // If the body is a function-try-block, enter the try before
-  // anything else --- unless we're in a deleting destructor, in which
-  // case we're just going to call the complete destructor and then
-  // call operator delete() on the way out.
-  bool isTryBody = (DtorType != Dtor_Deleting &&
-                    Body && isa<CXXTryStmt>(Body));
+  // anything else.
+  bool isTryBody = (Body && isa<CXXTryStmt>(Body));
   if (isTryBody)
     EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
 
-  // Emit the destructor epilogue now.  If this is a complete
-  // destructor with a function-try-block, perform the base epilogue
-  // as well.
-  //
-  // FIXME: This isn't really right, because an exception in the
-  // non-EH epilogue should jump to the appropriate place in the
-  // EH epilogue.
-  {
-    CleanupBlock Cleanup(*this, NormalCleanup);
-
-    if (isTryBody && DtorType == Dtor_Complete)
-      EmitDtorEpilogue(Dtor, Dtor_Base);
-    EmitDtorEpilogue(Dtor, DtorType);
-
-    if (Exceptions) {
-      Cleanup.beginEHCleanup();
-
-      if (isTryBody && DtorType == Dtor_Complete)
-        EmitDtorEpilogue(Dtor, Dtor_Base);
-      EmitDtorEpilogue(Dtor, DtorType);
-    }
-  }
-
-  bool SkipBody = false; // should get jump-threaded
-
-  // If this is the deleting variant, just invoke the complete
-  // variant, then call the appropriate operator delete() on the way
-  // out.
-  if (DtorType == Dtor_Deleting) {
-    EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
-                          LoadCXXThis());
-    SkipBody = true;
-
+  // Enter the epilogue cleanups.
+  RunCleanupsScope DtorEpilogue(*this);
+  
   // If this is the complete variant, just invoke the base variant;
   // the epilogue will destruct the virtual bases.  But we can't do
   // this optimization if the body is a function-try-block, because
   // we'd introduce *two* handler blocks.
-  } else if (!isTryBody && DtorType == Dtor_Complete) {
-    EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
-                          LoadCXXThis());
-    SkipBody = true;
+  switch (DtorType) {
+  case Dtor_Deleting: llvm_unreachable("already handled deleting case");
+
+  case Dtor_Complete:
+    // Enter the cleanup scopes for virtual bases.
+    EnterDtorCleanups(Dtor, Dtor_Complete);
+
+    if (!isTryBody) {
+      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
+                            LoadCXXThis());
+      break;
+    }
+    // Fallthrough: act like we're in the base variant.
       
-  // Otherwise, we're in the base variant, so we need to ensure the
-  // vtable ptrs are right before emitting the body.
-  } else {
+  case Dtor_Base:
+    // Enter the cleanup scopes for fields and non-virtual bases.
+    EnterDtorCleanups(Dtor, Dtor_Base);
+
+    // Initialize the vtable pointers before entering the body.
     InitializeVTablePointers(Dtor->getParent());
-  }
 
-  // Emit the body of the statement.
-  if (SkipBody)
-    (void) 0;
-  else if (isTryBody)
-    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
-  else if (Body)
-    EmitStmt(Body);
-  else {
-    assert(Dtor->isImplicit() && "bodyless dtor not implicit");
-    // nothing to do besides what's in the epilogue
+    if (isTryBody)
+      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
+    else if (Body)
+      EmitStmt(Body);
+    else {
+      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
+      // nothing to do besides what's in the epilogue
+    }
+    break;
   }
 
-  // We're done with the epilogue cleanup.
-  PopCleanupBlock();
+  // Jump out through the epilogue cleanups.
+  DtorEpilogue.ForceCleanup();
 
   // Exit the try if applicable.
   if (isTryBody)
     ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
 }
 
+namespace {
+  /// Call the operator delete associated with the current destructor.
+  struct CallDtorDelete : EHScopeStack::Cleanup {
+    CallDtorDelete() {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
+      const CXXRecordDecl *ClassDecl = Dtor->getParent();
+      CGF.EmitDeleteCall(Dtor->getOperatorDelete(), CGF.LoadCXXThis(),
+                         CGF.getContext().getTagDeclType(ClassDecl));
+    }
+  };
+
+  struct CallArrayFieldDtor : EHScopeStack::Cleanup {
+    const FieldDecl *Field;
+    CallArrayFieldDtor(const FieldDecl *Field) : Field(Field) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      QualType FieldType = Field->getType();
+      const ConstantArrayType *Array =
+        CGF.getContext().getAsConstantArrayType(FieldType);
+      
+      QualType BaseType =
+        CGF.getContext().getBaseElementType(Array->getElementType());
+      const CXXRecordDecl *FieldClassDecl = BaseType->getAsCXXRecordDecl();
+
+      llvm::Value *ThisPtr = CGF.LoadCXXThis();
+      LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 
+                                          // FIXME: Qualifiers?
+                                          /*CVRQualifiers=*/0);
+
+      const llvm::Type *BasePtr = CGF.ConvertType(BaseType)->getPointerTo();
+      llvm::Value *BaseAddrPtr =
+        CGF.Builder.CreateBitCast(LHS.getAddress(), BasePtr);
+      CGF.EmitCXXAggrDestructorCall(FieldClassDecl->getDestructor(),
+                                    Array, BaseAddrPtr);
+    }
+  };
+
+  struct CallFieldDtor : EHScopeStack::Cleanup {
+    const FieldDecl *Field;
+    CallFieldDtor(const FieldDecl *Field) : Field(Field) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      const CXXRecordDecl *FieldClassDecl =
+        Field->getType()->getAsCXXRecordDecl();
+
+      llvm::Value *ThisPtr = CGF.LoadCXXThis();
+      LValue LHS = CGF.EmitLValueForField(ThisPtr, Field, 
+                                          // FIXME: Qualifiers?
+                                          /*CVRQualifiers=*/0);
+
+      CGF.EmitCXXDestructorCall(FieldClassDecl->getDestructor(),
+                                Dtor_Complete, /*ForVirtualBase=*/false,
+                                LHS.getAddress());
+    }
+  };
+}
+
 /// EmitDtorEpilogue - Emit all code that comes at the end of class's
 /// destructor. This is to call destructors on members and base classes
 /// in reverse order of their construction.
-void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
-                                       CXXDtorType DtorType) {
+void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
+                                        CXXDtorType DtorType) {
   assert(!DD->isTrivial() &&
          "Should not emit dtor epilogue for trivial dtor!");
 
-  const CXXRecordDecl *ClassDecl = DD->getParent();
-
-  // In a deleting destructor, we've already called the complete
-  // destructor as a subroutine, so we just have to delete the
-  // appropriate value.
+  // The deleting-destructor phase just needs to call the appropriate
+  // operator delete that Sema picked up.
   if (DtorType == Dtor_Deleting) {
     assert(DD->getOperatorDelete() && 
            "operator delete missing - EmitDtorEpilogue");
-    EmitDeleteCall(DD->getOperatorDelete(), LoadCXXThis(),
-                   getContext().getTagDeclType(ClassDecl));
+    EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
     return;
   }
 
-  // For complete destructors, we've already called the base
-  // destructor (in GenerateBody), so we just need to destruct all the
-  // virtual bases.
+  const CXXRecordDecl *ClassDecl = DD->getParent();
+
+  // The complete-destructor phase just destructs all the virtual bases.
   if (DtorType == Dtor_Complete) {
-    // Handle virtual bases.
-    for (CXXRecordDecl::reverse_base_class_const_iterator I = 
-           ClassDecl->vbases_rbegin(), E = ClassDecl->vbases_rend();
+
+    // We push them in the forward order so that they'll be popped in
+    // the reverse order.
+    for (CXXRecordDecl::base_class_const_iterator I = 
+           ClassDecl->vbases_begin(), E = ClassDecl->vbases_end();
               I != E; ++I) {
       const CXXBaseSpecifier &Base = *I;
       CXXRecordDecl *BaseClassDecl
@@ -778,26 +858,48 @@ void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
       // Ignore trivial destructors.
       if (BaseClassDecl->hasTrivialDestructor())
         continue;
-      const CXXDestructorDecl *D = BaseClassDecl->getDestructor();
-      llvm::Value *V = 
-        GetAddressOfDirectBaseInCompleteClass(LoadCXXThis(),
-                                              ClassDecl, BaseClassDecl,
-                                              /*BaseIsVirtual=*/true);
-      EmitCXXDestructorCall(D, Dtor_Base, /*ForVirtualBase=*/true, V);
+
+      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
+                                        BaseClassDecl,
+                                        /*BaseIsVirtual*/ true);
     }
+
     return;
   }
 
   assert(DtorType == Dtor_Base);
+  
+  // Destroy non-virtual bases.
+  for (CXXRecordDecl::base_class_const_iterator I = 
+        ClassDecl->bases_begin(), E = ClassDecl->bases_end(); I != E; ++I) {
+    const CXXBaseSpecifier &Base = *I;
+    
+    // Ignore virtual bases.
+    if (Base.isVirtual())
+      continue;
+    
+    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
+    
+    // Ignore trivial destructors.
+    if (BaseClassDecl->hasTrivialDestructor())
+      continue;
+
+    EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
+                                      BaseClassDecl,
+                                      /*BaseIsVirtual*/ false);
+  }
 
-  // Collect the fields.
+  // Destroy direct fields.
   llvm::SmallVector<const FieldDecl *, 16> FieldDecls;
   for (CXXRecordDecl::field_iterator I = ClassDecl->field_begin(),
        E = ClassDecl->field_end(); I != E; ++I) {
     const FieldDecl *Field = *I;
     
     QualType FieldType = getContext().getCanonicalType(Field->getType());
-    FieldType = getContext().getBaseElementType(FieldType);
+    const ConstantArrayType *Array = 
+      getContext().getAsConstantArrayType(FieldType);
+    if (Array)
+      FieldType = getContext().getBaseElementType(Array->getElementType());
     
     const RecordType *RT = FieldType->getAs<RecordType>();
     if (!RT)
@@ -806,64 +908,11 @@ void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
     CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
     if (FieldClassDecl->hasTrivialDestructor())
         continue;
-    
-    FieldDecls.push_back(Field);
-  }
-  
-  // Now destroy the fields.
-  for (size_t i = FieldDecls.size(); i > 0; --i) {
-    const FieldDecl *Field = FieldDecls[i - 1];
-    
-    QualType FieldType = Field->getType();
-    const ConstantArrayType *Array = 
-      getContext().getAsConstantArrayType(FieldType);
-    if (Array)
-      FieldType = getContext().getBaseElementType(FieldType);
-    
-    const RecordType *RT = FieldType->getAs<RecordType>();
-    CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
 
-    llvm::Value *ThisPtr = LoadCXXThis();
-
-    LValue LHS = EmitLValueForField(ThisPtr, Field, 
-                                    // FIXME: Qualifiers?
-                                    /*CVRQualifiers=*/0);
-    if (Array) {
-      const llvm::Type *BasePtr = ConvertType(FieldType);
-      BasePtr = llvm::PointerType::getUnqual(BasePtr);
-      llvm::Value *BaseAddrPtr =
-        Builder.CreateBitCast(LHS.getAddress(), BasePtr);
-      EmitCXXAggrDestructorCall(FieldClassDecl->getDestructor(),
-                                Array, BaseAddrPtr);
-    } else
-      EmitCXXDestructorCall(FieldClassDecl->getDestructor(),
-                            Dtor_Complete, /*ForVirtualBase=*/false,
-                            LHS.getAddress());
-  }
-
-  // Destroy non-virtual bases.
-  for (CXXRecordDecl::reverse_base_class_const_iterator I = 
-        ClassDecl->bases_rbegin(), E = ClassDecl->bases_rend(); I != E; ++I) {
-    const CXXBaseSpecifier &Base = *I;
-    
-    // Ignore virtual bases.
-    if (Base.isVirtual())
-      continue;
-    
-    CXXRecordDecl *BaseClassDecl
-      = cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
-    
-    // Ignore trivial destructors.
-    if (BaseClassDecl->hasTrivialDestructor())
-      continue;
-
-    const CXXDestructorDecl *D = BaseClassDecl->getDestructor();    
-    llvm::Value *V = 
-      GetAddressOfDirectBaseInCompleteClass(LoadCXXThis(), ClassDecl, 
-                                            BaseClassDecl, 
-                                            /*BaseIsVirtual=*/false);
-
-    EmitCXXDestructorCall(D, Dtor_Base, /*ForVirtualBase=*/false, V);
+    if (Array)
+      EHStack.pushCleanup<CallArrayFieldDtor>(NormalAndEHCleanup, Field);
+    else
+      EHStack.pushCleanup<CallFieldDtor>(NormalAndEHCleanup, Field);
   }
 }
 
@@ -873,19 +922,24 @@ void CodeGenFunction::EmitDtorEpilogue(const CXXDestructorDecl *DD,
 /// 'D' is the default constructor for elements of the array, 'ArrayTy' is the
 /// array type and 'ArrayPtr' points to the beginning fo the array.
 /// It is assumed that all relevant checks have been made by the caller.
+///
+/// \param ZeroInitialization True if each element should be zero-initialized
+/// before it is constructed.
 void
 CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
-                                          const ConstantArrayType *ArrayTy,
-                                          llvm::Value *ArrayPtr,
-                                          CallExpr::const_arg_iterator ArgBeg,
-                                          CallExpr::const_arg_iterator ArgEnd) {
+                                            const ConstantArrayType *ArrayTy,
+                                            llvm::Value *ArrayPtr,
+                                            CallExpr::const_arg_iterator ArgBeg,
+                                            CallExpr::const_arg_iterator ArgEnd,
+                                            bool ZeroInitialization) {
 
   const llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
   llvm::Value * NumElements =
     llvm::ConstantInt::get(SizeTy, 
                            getContext().getConstantArrayElementCount(ArrayTy));
 
-  EmitCXXAggrConstructorCall(D, NumElements, ArrayPtr, ArgBeg, ArgEnd);
+  EmitCXXAggrConstructorCall(D, NumElements, ArrayPtr, ArgBeg, ArgEnd,
+                             ZeroInitialization);
 }
 
 void
@@ -893,7 +947,8 @@ CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
                                           llvm::Value *NumElements,
                                           llvm::Value *ArrayPtr,
                                           CallExpr::const_arg_iterator ArgBeg,
-                                          CallExpr::const_arg_iterator ArgEnd) {
+                                          CallExpr::const_arg_iterator ArgEnd,
+                                            bool ZeroInitialization) {
   const llvm::Type *SizeTy = ConvertType(getContext().getSizeType());
 
   // Create a temporary for the loop index and initialize it with 0.
@@ -924,6 +979,11 @@ CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
   llvm::Value *Address = Builder.CreateInBoundsGEP(ArrayPtr, Counter, 
                                                    "arrayidx");
 
+  // Zero initialize the storage, if requested.
+  if (ZeroInitialization)
+    EmitNullInitialization(Address, 
+                           getContext().getTypeDeclType(D->getParent()));
+  
   // C++ [class.temporary]p4: 
   // There are two contexts in which temporaries are destroyed at a different
   // point than the end of the full-expression. The first context is when a
@@ -1109,21 +1169,33 @@ void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
   EmitCXXMemberCall(DD, Callee, ReturnValueSlot(), This, VTT, 0, 0);
 }
 
+namespace {
+  struct CallLocalDtor : EHScopeStack::Cleanup {
+    const CXXDestructorDecl *Dtor;
+    llvm::Value *Addr;
+
+    CallLocalDtor(const CXXDestructorDecl *D, llvm::Value *Addr)
+      : Dtor(D), Addr(Addr) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
+                                /*ForVirtualBase=*/false, Addr);
+    }
+  };
+}
+
+void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
+                                            llvm::Value *Addr) {
+  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr);
+}
+
 void CodeGenFunction::PushDestructorCleanup(QualType T, llvm::Value *Addr) {
   CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
   if (!ClassDecl) return;
   if (ClassDecl->hasTrivialDestructor()) return;
 
   const CXXDestructorDecl *D = ClassDecl->getDestructor();
-
-  CleanupBlock Scope(*this, NormalCleanup);
-
-  EmitCXXDestructorCall(D, Dtor_Complete, /*ForVirtualBase=*/false, Addr);
-
-  if (Exceptions) {
-    Scope.beginEHCleanup();
-    EmitCXXDestructorCall(D, Dtor_Complete, /*ForVirtualBase=*/false, Addr);
-  }
+  PushDestructorCleanup(D, Addr);
 }
 
 llvm::Value *
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 4e158955f894..406db886eeed 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -15,7 +15,9 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/Basic/SourceManager.h"
@@ -37,7 +39,7 @@ using namespace clang::CodeGen;
 
 CGDebugInfo::CGDebugInfo(CodeGenModule &CGM)
   : CGM(CGM), DebugFactory(CGM.getModule()),
-    FwdDeclCount(0), BlockLiteralGenericSet(false) {
+    BlockLiteralGenericSet(false) {
   CreateCompileUnit();
 }
 
@@ -93,6 +95,58 @@ llvm::StringRef CGDebugInfo::getFunctionName(const FunctionDecl *FD) {
   return llvm::StringRef(StrPtr, NS.length());
 }
 
+llvm::StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) {
+  llvm::SmallString<256> MethodName;
+  llvm::raw_svector_ostream OS(MethodName);
+  OS << (OMD->isInstanceMethod() ? '-' : '+') << '[';
+  const DeclContext *DC = OMD->getDeclContext();
+  if (const ObjCImplementationDecl *OID = dyn_cast<const ObjCImplementationDecl>(DC)) {
+     OS << OID->getName();
+  } else if (const ObjCCategoryImplDecl *OCD = dyn_cast<const ObjCCategoryImplDecl>(DC)){
+      OS << ((NamedDecl *)OCD)->getIdentifier()->getNameStart() << '(' <<
+          OCD->getIdentifier()->getNameStart() << ')';
+  }
+  OS << ' ' << OMD->getSelector().getAsString() << ']';
+
+  char *StrPtr = DebugInfoNames.Allocate<char>(OS.tell());
+  memcpy(StrPtr, MethodName.begin(), OS.tell());
+  return llvm::StringRef(StrPtr, OS.tell());
+}
+
+/// getClassName - Get class name including template argument list.
+llvm::StringRef 
+CGDebugInfo::getClassName(RecordDecl *RD) {
+  ClassTemplateSpecializationDecl *Spec
+    = dyn_cast<ClassTemplateSpecializationDecl>(RD);
+  if (!Spec)
+    return RD->getName();
+
+  const TemplateArgument *Args;
+  unsigned NumArgs;
+  std::string Buffer;
+  if (TypeSourceInfo *TAW = Spec->getTypeAsWritten()) {
+    const TemplateSpecializationType *TST =
+      cast<TemplateSpecializationType>(TAW->getType());
+    Args = TST->getArgs();
+    NumArgs = TST->getNumArgs();
+  } else {
+    const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
+    Args = TemplateArgs.getFlatArgumentList();
+    NumArgs = TemplateArgs.flat_size();
+  }
+  Buffer = RD->getIdentifier()->getNameStart();
+  PrintingPolicy Policy(CGM.getLangOptions());
+  Buffer += TemplateSpecializationType::PrintTemplateArgumentList(Args,
+                                                                  NumArgs,
+                                                                  Policy);
+
+  // Copy this name on the side and use its reference.
+  char *StrPtr = DebugInfoNames.Allocate<char>(Buffer.length());
+  memcpy(StrPtr, Buffer.data(), Buffer.length());
+  return llvm::StringRef(StrPtr, Buffer.length());
+
+}
+
 /// getOrCreateFile - Get the file debug info descriptor for the input location.
 llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
   if (!Loc.isValid())
@@ -113,13 +167,8 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
       return llvm::DIFile(cast<llvm::MDNode>(it->second));
   }
 
-  // FIXME: We shouldn't even need to call 'makeAbsolute()' in the cases
-  // where we can consult the FileEntry.
-  llvm::sys::Path AbsFileName(PLoc.getFilename());
-  AbsFileName.makeAbsolute();
-
-  llvm::DIFile F = DebugFactory.CreateFile(AbsFileName.getLast(),
-                                           AbsFileName.getDirname(), TheCU);
+  llvm::DIFile F = DebugFactory.CreateFile(PLoc.getFilename(),
+                                           getCurrentDirname(), TheCU);
 
   DIFileCache[fname] = F;
   return F;
@@ -144,6 +193,16 @@ unsigned CGDebugInfo::getColumnNumber(SourceLocation Loc) {
   return PLoc.getColumn();
 }
 
+llvm::StringRef CGDebugInfo::getCurrentDirname() {
+  if (!CWDName.empty())
+    return CWDName;
+  char *CompDirnamePtr = NULL;
+  llvm::sys::Path CWD = llvm::sys::Path::GetCurrentDirectory();
+  CompDirnamePtr = DebugInfoNames.Allocate<char>(CWD.size());
+  memcpy(CompDirnamePtr, CWD.c_str(), CWD.size());
+  return CWDName = llvm::StringRef(CompDirnamePtr, CWD.size());
+}
+
 /// CreateCompileUnit - Create new compile unit.
 void CGDebugInfo::CreateCompileUnit() {
 
@@ -153,19 +212,22 @@ void CGDebugInfo::CreateCompileUnit() {
   if (MainFileName.empty())
     MainFileName = "<unknown>";
 
-  llvm::sys::Path AbsFileName(MainFileName);
-  AbsFileName.makeAbsolute();
-
   // The main file name provided via the "-main-file-name" option contains just
   // the file name itself with no path information. This file name may have had
   // a relative path, so we look into the actual file entry for the main
   // file to determine the real absolute path for the file.
   std::string MainFileDir;
-  if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID()))
+  if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
     MainFileDir = MainFile->getDir()->getName();
-  else
-    MainFileDir = AbsFileName.getDirname();
+    if (MainFileDir != ".")
+      MainFileName = MainFileDir + "/" + MainFileName;
+  }
 
+  // Save filename string.
+  char *FilenamePtr = DebugInfoNames.Allocate<char>(MainFileName.length());
+  memcpy(FilenamePtr, MainFileName.c_str(), MainFileName.length());
+  llvm::StringRef Filename(FilenamePtr, MainFileName.length());
+  
   unsigned LangTag;
   const LangOptions &LO = CGM.getLangOptions();
   if (LO.CPlusPlus) {
@@ -181,11 +243,7 @@ void CGDebugInfo::CreateCompileUnit() {
     LangTag = llvm::dwarf::DW_LANG_C89;
   }
 
-  const char *Producer =
-#ifdef CLANG_VENDOR
-    CLANG_VENDOR
-#endif
-    "clang " CLANG_VERSION_STRING;
+  std::string Producer = getClangFullVersion();
 
   // Figure out which version of the ObjC runtime we have.
   unsigned RuntimeVers = 0;
@@ -194,7 +252,8 @@ void CGDebugInfo::CreateCompileUnit() {
 
   // Create new compile unit.
   TheCU = DebugFactory.CreateCompileUnit(
-    LangTag, AbsFileName.getLast(), MainFileDir, Producer, true,
+    LangTag, Filename, getCurrentDirname(),
+    Producer, true,
     LO.Optimize, CGM.getCodeGenOpts().DwarfDebugFlags, RuntimeVers);
 }
 
@@ -203,10 +262,49 @@ void CGDebugInfo::CreateCompileUnit() {
 llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
                                      llvm::DIFile Unit) {
   unsigned Encoding = 0;
+  const char *BTName = NULL;
   switch (BT->getKind()) {
   default:
   case BuiltinType::Void:
     return llvm::DIType();
+  case BuiltinType::ObjCClass:
+    return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type,
+                                            Unit, "objc_class", Unit, 0, 0, 0, 0,
+                                            llvm::DIType::FlagFwdDecl, 
+                                            llvm::DIType(), llvm::DIArray());
+  case BuiltinType::ObjCId: {
+    // typedef struct objc_class *Class;
+    // typedef struct objc_object {
+    //  Class isa;
+    // } *id;
+
+    llvm::DIType OCTy = 
+      DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type,
+                                       Unit, "objc_class", Unit, 0, 0, 0, 0,
+                                       llvm::DIType::FlagFwdDecl, 
+                                       llvm::DIType(), llvm::DIArray());
+    unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
+    
+    llvm::DIType ISATy = 
+      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_pointer_type,
+                                     Unit, "", Unit,
+                                     0, Size, 0, 0, 0, OCTy);
+
+    llvm::SmallVector<llvm::DIDescriptor, 16> EltTys;
+
+    llvm::DIType FieldTy = 
+      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_member, Unit,
+                                     "isa", Unit,
+                                     0,Size, 0, 0, 0, ISATy);
+    EltTys.push_back(FieldTy);
+    llvm::DIArray Elements =
+      DebugFactory.GetOrCreateArray(EltTys.data(), EltTys.size());
+    
+    return DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_structure_type,
+                                            Unit, "objc_object", Unit, 0, 0, 0, 0,
+                                            0,
+                                            llvm::DIType(), Elements);
+  }
   case BuiltinType::UChar:
   case BuiltinType::Char_U: Encoding = llvm::dwarf::DW_ATE_unsigned_char; break;
   case BuiltinType::Char_S:
@@ -224,14 +322,23 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT,
   case BuiltinType::LongDouble:
   case BuiltinType::Double:    Encoding = llvm::dwarf::DW_ATE_float; break;
   }
+
+  switch (BT->getKind()) {
+  case BuiltinType::Long:      BTName = "long int"; break;
+  case BuiltinType::LongLong:  BTName = "long long int"; break;
+  case BuiltinType::ULong:     BTName = "long unsigned int"; break;
+  case BuiltinType::ULongLong: BTName = "long long unsigned int"; break;
+  default:
+    BTName = BT->getName(CGM.getContext().getLangOptions());
+    break;
+  }
   // Bit size, align and offset of the type.
   uint64_t Size = CGM.getContext().getTypeSize(BT);
   uint64_t Align = CGM.getContext().getTypeAlign(BT);
   uint64_t Offset = 0;
-
+  
   llvm::DIType DbgTy = 
-    DebugFactory.CreateBasicType(Unit,
-                                 BT->getName(CGM.getContext().getLangOptions()),
+    DebugFactory.CreateBasicType(Unit, BTName,
                                  Unit, 0, Size, Align,
                                  Offset, /*flags*/ 0, Encoding);
   return DbgTy;
@@ -461,7 +568,6 @@ CollectRecordFields(const RecordDecl *RD, llvm::DIFile Unit,
        I != E; ++I, ++FieldNo) {
     FieldDecl *Field = *I;
     llvm::DIType FieldTy = getOrCreateType(Field->getType(), Unit);
-
     llvm::StringRef FieldName = Field->getName();
 
     // Ignore unnamed fields. Do not ignore unnamed records.
@@ -481,7 +587,6 @@ CollectRecordFields(const RecordDecl *RD, llvm::DIFile Unit,
       Expr *BitWidth = Field->getBitWidth();
       if (BitWidth)
         FieldSize = BitWidth->EvaluateAsInt(CGM.getContext()).getZExtValue();
-
       FieldAlign =  CGM.getContext().getTypeAlign(FType);
     }
 
@@ -516,9 +621,12 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
                                0),
                       Unit);
   
-  // Static methods do not need "this" pointer argument.
-  if (Method->isStatic())
-    return FnTy;
+  unsigned BFlags=0;
+  AccessSpecifier Access = Method->getAccess();
+  if (Access == clang::AS_private)
+    BFlags |= llvm::DIType::FlagPrivate;
+  else if (Access == clang::AS_protected)
+    BFlags |= llvm::DIType::FlagProtected;
 
   // Add "this" pointer.
 
@@ -530,27 +638,18 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
   // First element is always return type. For 'void' functions it is NULL.
   Elts.push_back(Args.getElement(0));
 
-  // "this" pointer is always first argument.
-  ASTContext &Context = CGM.getContext();
-  QualType ThisPtr = 
-    Context.getPointerType(Context.getTagDeclType(Method->getParent()));
-  llvm::DIType ThisPtrType = 
-    DebugFactory.CreateArtificialType(getOrCreateType(ThisPtr, Unit));
-
-  unsigned Quals = Method->getTypeQualifiers();
-  if (Quals & Qualifiers::Const)
-    ThisPtrType = 
-      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_const_type, 
-                                     Unit, "", Unit,
-                                     0, 0, 0, 0, 0, ThisPtrType);
-  if (Quals & Qualifiers::Volatile)
-    ThisPtrType = 
-      DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_volatile_type, 
-                                     Unit, "", Unit,
-                                     0, 0, 0, 0, 0, ThisPtrType);
-
-  TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;  
-  Elts.push_back(ThisPtrType);
+  if (!Method->isStatic())
+  {
+        // "this" pointer is always first argument.
+        ASTContext &Context = CGM.getContext();
+        QualType ThisPtr =
+          Context.getPointerType(Context.getTagDeclType(Method->getParent()));
+        llvm::DIType ThisPtrType =
+          DebugFactory.CreateArtificialType(getOrCreateType(ThisPtr, Unit));
+
+        TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;
+        Elts.push_back(ThisPtrType);
+    }
 
   // Copy rest of the arguments.
   for (unsigned i = 1, e = Args.getNumElements(); i != e; ++i)
@@ -571,7 +670,7 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
 llvm::DISubprogram
 CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
                                      llvm::DIFile Unit,
-                                     llvm::DICompositeType &RecordTy) {
+                                     llvm::DIType RecordTy) {
   bool IsCtorOrDtor = 
     isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method);
   
@@ -612,7 +711,9 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
                                   MethodDefUnit, MethodLine,
                                   MethodTy, /*isLocalToUnit=*/false, 
                                   /* isDefintion=*/ false,
-                                  Virtuality, VIndex, ContainingType);
+                                  Virtuality, VIndex, ContainingType,
+                                  Method->isImplicit(),
+                                  CGM.getLangOptions().Optimize);
   
   // Don't cache ctors or dtors since we have to emit multiple functions for
   // a single ctor or dtor.
@@ -628,7 +729,7 @@ CGDebugInfo::CreateCXXMemberFunction(const CXXMethodDecl *Method,
 void CGDebugInfo::
 CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
                           llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
-                          llvm::DICompositeType &RecordTy) {
+                          llvm::DIType RecordTy) {
   for(CXXRecordDecl::method_iterator I = RD->method_begin(),
         E = RD->method_end(); I != E; ++I) {
     const CXXMethodDecl *Method = *I;
@@ -640,13 +741,41 @@ CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
   }
 }                                 
 
+/// CollectCXXFriends - A helper function to collect debug info for
+/// C++ base classes. This is used while creating debug info entry for
+/// a Record.
+void CGDebugInfo::
+CollectCXXFriends(const CXXRecordDecl *RD, llvm::DIFile Unit,
+                llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                llvm::DIType RecordTy) {
+
+  for (CXXRecordDecl::friend_iterator BI =  RD->friend_begin(),
+         BE = RD->friend_end(); BI != BE; ++BI) {
+
+    TypeSourceInfo *TInfo = (*BI)->getFriendType();
+    if(TInfo)
+    {
+        llvm::DIType Ty = getOrCreateType(TInfo->getType(), Unit);
+
+            llvm::DIType DTy =
+          DebugFactory.CreateDerivedType(llvm::dwarf::DW_TAG_friend,
+                                         RecordTy, llvm::StringRef(),
+                                         Unit, 0, 0, 0,
+                                         0, 0, Ty);
+
+        EltTys.push_back(DTy);
+    }
+
+  }
+}
+
 /// CollectCXXBases - A helper function to collect debug info for
 /// C++ base classes. This is used while creating debug info entry for 
 /// a Record.
 void CGDebugInfo::
 CollectCXXBases(const CXXRecordDecl *RD, llvm::DIFile Unit,
                 llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
-                llvm::DICompositeType &RecordTy) {
+                llvm::DIType RecordTy) {
 
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
   for (CXXRecordDecl::base_class_const_iterator BI = RD->bases_begin(),
@@ -786,14 +915,7 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
       return FwdDecl;
   }
 
-  // A RD->getName() is not unique. However, the debug info descriptors 
-  // are uniqued so use type name to ensure uniquness.
-  llvm::SmallString<128> FwdDeclName;
-  llvm::raw_svector_ostream(FwdDeclName) << "fwd.type." << FwdDeclCount++;
-  llvm::DICompositeType FwdDecl =
-    DebugFactory.CreateCompositeType(Tag, FDContext, FwdDeclName,
-                                     DefUnit, Line, 0, 0, 0, 0,
-                                     llvm::DIType(), llvm::DIArray());
+  llvm::DIType FwdDecl = DebugFactory.CreateTemporaryType();
 
   llvm::MDNode *MN = FwdDecl;
   llvm::TrackingVH<llvm::MDNode> FwdDeclNode = MN;
@@ -812,10 +934,36 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
     CollectCXXBases(CXXDecl, Unit, EltTys, FwdDecl);
     CollectVTableInfo(CXXDecl, Unit, EltTys);
   }
+  
+  // Collect static variables with initializers.
+  for (RecordDecl::decl_iterator I = RD->decls_begin(), E = RD->decls_end();
+       I != E; ++I)
+    if (const VarDecl *V = dyn_cast<VarDecl>(*I)) {
+      if (const Expr *Init = V->getInit()) {
+        Expr::EvalResult Result;
+        if (Init->Evaluate(Result, CGM.getContext()) && Result.Val.isInt()) {
+          llvm::ConstantInt *CI 
+            = llvm::ConstantInt::get(CGM.getLLVMContext(), Result.Val.getInt());
+          
+          // Create the descriptor for static variable.
+          llvm::DIFile VUnit = getOrCreateFile(V->getLocation());
+          llvm::StringRef VName = V->getName();
+          llvm::DIType VTy = getOrCreateType(V->getType(), VUnit);
+          // Do not use DIGlobalVariable for enums.
+          if (VTy.getTag() != llvm::dwarf::DW_TAG_enumeration_type) {
+            DebugFactory.CreateGlobalVariable(FwdDecl, VName, VName, VName, VUnit,
+                                              getLineNumber(V->getLocation()),
+                                              VTy, true, true, CI);
+          }
+        }
+      }
+    }
+
   CollectRecordFields(RD, Unit, EltTys);
   llvm::MDNode *ContainingType = NULL;
   if (CXXDecl) {
     CollectCXXMemberFunctions(CXXDecl, Unit, EltTys, FwdDecl);
+    CollectCXXFriends(CXXDecl, Unit, EltTys, FwdDecl);
 
     // A class's primary base or the class itself contains the vtable.
     const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
@@ -841,16 +989,22 @@ llvm::DIType CGDebugInfo::CreateType(const RecordType *Ty,
 
   llvm::DIDescriptor RDContext =  
     getContextDescriptor(dyn_cast<Decl>(RD->getDeclContext()), Unit);
+
+  llvm::StringRef RDName = RD->getName();
+  // If this is a class, include the template arguments also.
+  if (Tag == llvm::dwarf::DW_TAG_class_type) 
+    RDName = getClassName(RD);
+  
   llvm::DICompositeType RealDecl =
     DebugFactory.CreateCompositeType(Tag, RDContext,
-                                     RD->getName(),
+                                     RDName,
                                      DefUnit, Line, Size, Align, 0, 0, 
                                      llvm::DIType(), Elements, 
                                      0, ContainingType);
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
-  llvm::DIDerivedType(FwdDeclNode).replaceAllUsesWith(RealDecl);
+  llvm::DIType(FwdDeclNode).replaceAllUsesWith(RealDecl);
   RegionMap[RD] = llvm::WeakVH(RealDecl);
   return RealDecl;
 }
@@ -873,21 +1027,24 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
   unsigned Line = getLineNumber(ID->getLocation());
   unsigned RuntimeLang = TheCU.getLanguage();
 
+  // If this is just a forward declaration, return a special forward-declaration
+  // debug type.
+  if (ID->isForwardDecl()) {
+    llvm::DICompositeType FwdDecl =
+      DebugFactory.CreateCompositeType(Tag, Unit, ID->getName(),
+                                       DefUnit, Line, 0, 0, 0, 0,
+                                       llvm::DIType(), llvm::DIArray(),
+                                       RuntimeLang);
+    return FwdDecl;
+  }
+
   // To handle recursive interface, we
   // first generate a debug descriptor for the struct as a forward declaration.
   // Then (if it is a definition) we go through and get debug info for all of
   // its members.  Finally, we create a descriptor for the complete type (which
   // may refer to the forward decl if the struct is recursive) and replace all
   // uses of the forward declaration with the final definition.
-  llvm::DICompositeType FwdDecl =
-    DebugFactory.CreateCompositeType(Tag, Unit, ID->getName(),
-                                     DefUnit, Line, 0, 0, 0, 0,
-                                     llvm::DIType(), llvm::DIArray(),
-                                     RuntimeLang);
-
-  // If this is just a forward declaration, return it.
-  if (ID->isForwardDecl())
-    return FwdDecl;
+  llvm::DIType FwdDecl = DebugFactory.CreateTemporaryType();
 
   llvm::MDNode *MN = FwdDecl;
   llvm::TrackingVH<llvm::MDNode> FwdDeclNode = MN;
@@ -982,7 +1139,7 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
 
   // Now that we have a real decl for the struct, replace anything using the
   // old decl with the new one.  This will recursively update the debug info.
-  llvm::DIDerivedType(FwdDeclNode).replaceAllUsesWith(RealDecl);
+  llvm::DIType(FwdDeclNode).replaceAllUsesWith(RealDecl);
   RegionMap[ID] = llvm::WeakVH(RealDecl);
 
   return RealDecl;
@@ -990,39 +1147,8 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
 
 llvm::DIType CGDebugInfo::CreateType(const EnumType *Ty,
                                      llvm::DIFile Unit) {
-  EnumDecl *ED = Ty->getDecl();
-
-  llvm::SmallVector<llvm::DIDescriptor, 32> Enumerators;
-
-  // Create DIEnumerator elements for each enumerator.
-  for (EnumDecl::enumerator_iterator
-         Enum = ED->enumerator_begin(), EnumEnd = ED->enumerator_end();
-       Enum != EnumEnd; ++Enum) {
-    Enumerators.push_back(DebugFactory.CreateEnumerator(Enum->getName(),
-                                            Enum->getInitVal().getZExtValue()));
-  }
+  return CreateEnumType(Ty->getDecl(), Unit);
 
-  // Return a CompositeType for the enum itself.
-  llvm::DIArray EltArray =
-    DebugFactory.GetOrCreateArray(Enumerators.data(), Enumerators.size());
-
-  llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
-  unsigned Line = getLineNumber(ED->getLocation());
-
-  // Size and align of the type.
-  uint64_t Size = 0;
-  unsigned Align = 0;
-  if (!Ty->isIncompleteType()) {
-    Size = CGM.getContext().getTypeSize(Ty);
-    Align = CGM.getContext().getTypeAlign(Ty);
-  }
-
-  llvm::DIType DbgTy = 
-    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_enumeration_type,
-                                     Unit, ED->getName(), DefUnit, Line,
-                                     Size, Align, 0, 0,
-                                     llvm::DIType(), EltArray);
-  return DbgTy;
 }
 
 llvm::DIType CGDebugInfo::CreateType(const TagType *Ty,
@@ -1036,7 +1162,7 @@ llvm::DIType CGDebugInfo::CreateType(const TagType *Ty,
 }
 
 llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty,
-				     llvm::DIFile Unit) {
+                                     llvm::DIFile Unit) {
   llvm::DIType ElementTy = getOrCreateType(Ty->getElementType(), Unit);
   uint64_t NumElems = Ty->getNumElements();
   if (NumElems > 0)
@@ -1052,7 +1178,7 @@ llvm::DIType CGDebugInfo::CreateType(const VectorType *Ty,
     DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_vector_type,
                                      Unit, "", Unit,
                                      0, Size, Align, 0, 0,
-				     ElementTy,  SubscriptArray);
+                                     ElementTy,  SubscriptArray);
 }
 
 llvm::DIType CGDebugInfo::CreateType(const ArrayType *Ty,
@@ -1149,6 +1275,38 @@ llvm::DIType CGDebugInfo::CreateType(const MemberPointerType *Ty,
                                           0, 0, 0, llvm::DIType(), Elements);
 }
 
+/// CreateEnumType - get enumeration type.
+llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED, llvm::DIFile Unit){
+  llvm::SmallVector<llvm::DIDescriptor, 32> Enumerators;
+
+  // Create DIEnumerator elements for each enumerator.
+  for (EnumDecl::enumerator_iterator
+         Enum = ED->enumerator_begin(), EnumEnd = ED->enumerator_end();
+       Enum != EnumEnd; ++Enum) {
+    Enumerators.push_back(DebugFactory.CreateEnumerator(Enum->getName(),
+                                            Enum->getInitVal().getZExtValue()));
+  }
+
+  // Return a CompositeType for the enum itself.
+  llvm::DIArray EltArray =
+    DebugFactory.GetOrCreateArray(Enumerators.data(), Enumerators.size());
+
+  llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
+  unsigned Line = getLineNumber(ED->getLocation());
+  uint64_t Size = 0;
+  uint64_t Align = 0;
+  if (!ED->getTypeForDecl()->isIncompleteType()) {
+    Size = CGM.getContext().getTypeSize(ED->getTypeForDecl());
+    Align = CGM.getContext().getTypeAlign(ED->getTypeForDecl());
+  }
+  llvm::DIType DbgTy = 
+    DebugFactory.CreateCompositeType(llvm::dwarf::DW_TAG_enumeration_type,
+                                     Unit, ED->getName(), DefUnit, Line,
+                                     Size, Align, 0, 0,
+                                     llvm::DIType(), EltArray);
+  return DbgTy;
+}
+
 static QualType UnwrapTypeForDebugInfo(QualType T) {
   do {
     QualType LastT = T;
@@ -1312,6 +1470,8 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
   llvm::StringRef Name;
   llvm::StringRef LinkageName;
 
+  FnBeginRegionCount.push_back(RegionStack.size());
+
   const Decl *D = GD.getDecl();
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // If there is a DISubprogram for  this function available then use it.
@@ -1329,6 +1489,9 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
     Name = getFunctionName(FD);
     // Use mangled name as linkage name for c/c++ functions.
     LinkageName = CGM.getMangledName(GD);
+  } else if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D)) {
+    Name = getObjCMethodName(OMD);
+    LinkageName = Name;
   } else {
     // Use llvm function name as linkage name.
     Name = Fn->getName();
@@ -1346,16 +1509,22 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
   llvm::DISubprogram SP =
     DebugFactory.CreateSubprogram(Unit, Name, Name, LinkageName, Unit, LineNo,
                                   getOrCreateType(FnType, Unit),
-                                  Fn->hasInternalLinkage(), true/*definition*/);
+                                  Fn->hasInternalLinkage(), true/*definition*/,
+                                  0, 0, llvm::DIType(),
+                                  D->isImplicit(),
+                                  CGM.getLangOptions().Optimize, Fn);
 
   // Push function on region stack.
   llvm::MDNode *SPN = SP;
   RegionStack.push_back(SPN);
   RegionMap[D] = llvm::WeakVH(SP);
+
+  // Clear stack used to keep track of #line directives.
+  LineDirectiveFiles.clear();
 }
 
 
-void CGDebugInfo::EmitStopPoint(llvm::Function *Fn, CGBuilderTy &Builder) {
+void CGDebugInfo::EmitStopPoint(CGBuilderTy &Builder) {
   if (CurLoc.isInvalid() || CurLoc.isMacroID()) return;
 
   // Don't bother if things are the same as last time.
@@ -1377,13 +1546,63 @@ void CGDebugInfo::EmitStopPoint(llvm::Function *Fn, CGBuilderTy &Builder) {
                                                       Scope));
 }
 
+/// UpdateLineDirectiveRegion - Update region stack only if #line directive
+/// has introduced scope change.
+void CGDebugInfo::UpdateLineDirectiveRegion(CGBuilderTy &Builder) {
+  if (CurLoc.isInvalid() || CurLoc.isMacroID() ||
+      PrevLoc.isInvalid() || PrevLoc.isMacroID())
+    return;
+  SourceManager &SM = CGM.getContext().getSourceManager();
+  PresumedLoc PCLoc = SM.getPresumedLoc(CurLoc);
+  PresumedLoc PPLoc = SM.getPresumedLoc(PrevLoc);
+
+  if (!strcmp(PPLoc.getFilename(), PCLoc.getFilename()))
+    return;
+
+  // If #line directive stack is empty then we are entering a new scope.
+  if (LineDirectiveFiles.empty()) {
+    EmitRegionStart(Builder);
+    LineDirectiveFiles.push_back(PCLoc.getFilename());
+    return;
+  }
+
+  assert (RegionStack.size() >= LineDirectiveFiles.size()
+          && "error handling  #line regions!");
+
+  bool SeenThisFile = false;
+  for(std::vector<const char *>::iterator I = LineDirectiveFiles.begin(),
+        E = LineDirectiveFiles.end(); I != E; ++I)
+    if (!strcmp(PPLoc.getFilename(), *I)) {
+      SeenThisFile = true;
+      break;
+    }
+
+  // If #line for this file is seen earlier then pop out #line regions.
+  if (SeenThisFile) {
+    while (!LineDirectiveFiles.empty()) {
+      const char *LastFile = LineDirectiveFiles.back();
+      RegionStack.pop_back();
+      LineDirectiveFiles.pop_back();
+      if (!strcmp(PPLoc.getFilename(), LastFile))
+        break;
+    }
+    return;
+  } 
+
+  // .. otherwise insert new #line region.
+  EmitRegionStart(Builder);
+  LineDirectiveFiles.push_back(PCLoc.getFilename());
+
+  return;
+}
 /// EmitRegionStart- Constructs the debug code for entering a declarative
 /// region - "llvm.dbg.region.start.".
-void CGDebugInfo::EmitRegionStart(llvm::Function *Fn, CGBuilderTy &Builder) {
+void CGDebugInfo::EmitRegionStart(CGBuilderTy &Builder) {
   llvm::DIDescriptor D =
     DebugFactory.CreateLexicalBlock(RegionStack.empty() ? 
                                     llvm::DIDescriptor() : 
                                     llvm::DIDescriptor(RegionStack.back()),
+                                    getOrCreateFile(CurLoc),
                                     getLineNumber(CurLoc), 
                                     getColumnNumber(CurLoc));
   llvm::MDNode *DN = D;
@@ -1392,15 +1611,27 @@ void CGDebugInfo::EmitRegionStart(llvm::Function *Fn, CGBuilderTy &Builder) {
 
 /// EmitRegionEnd - Constructs the debug code for exiting a declarative
 /// region - "llvm.dbg.region.end."
-void CGDebugInfo::EmitRegionEnd(llvm::Function *Fn, CGBuilderTy &Builder) {
+void CGDebugInfo::EmitRegionEnd(CGBuilderTy &Builder) {
   assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
 
   // Provide an region stop point.
-  EmitStopPoint(Fn, Builder);
+  EmitStopPoint(Builder);
 
   RegionStack.pop_back();
 }
 
+/// EmitFunctionEnd - Constructs the debug code for exiting a function.
+void CGDebugInfo::EmitFunctionEnd(CGBuilderTy &Builder) {
+  assert(!RegionStack.empty() && "Region stack mismatch, stack empty!");
+  unsigned RCount = FnBeginRegionCount.back();
+  assert(RCount <= RegionStack.size() && "Region stack mismatch");
+
+  // Pop all regions for this function.
+  while (RegionStack.size() != RCount)
+    EmitRegionEnd(Builder);
+  FnBeginRegionCount.pop_back();
+}
+
 // EmitTypeForVarWithBlocksAttr - Build up structure info for the byref.  
 // See BuildByRefType.
 llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
@@ -1643,6 +1874,26 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
                                     true/*definition*/, Var);
 }
 
+/// EmitGlobalVariable - Emit global variable's debug info.
+void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, 
+                                     llvm::ConstantInt *Init,
+                                     CGBuilderTy &Builder) {
+  // Create the descriptor for the variable.
+  llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
+  llvm::StringRef Name = VD->getName();
+  llvm::DIType Ty = getOrCreateType(VD->getType(), Unit);
+  if (const EnumConstantDecl *ECD = dyn_cast<EnumConstantDecl>(VD)) {
+    if (const EnumDecl *ED = dyn_cast<EnumDecl>(ECD->getDeclContext()))
+      Ty = CreateEnumType(ED, Unit);
+  }
+  // Do not use DIGlobalVariable for enums.
+  if (Ty.getTag() == llvm::dwarf::DW_TAG_enumeration_type)
+    return;
+  DebugFactory.CreateGlobalVariable(Unit, Name, Name, Name, Unit,
+                                    getLineNumber(VD->getLocation()),
+                                    Ty, true, true, Init);
+}
+
 /// getOrCreateNamesSpace - Return namespace descriptor for the given
 /// namespace decl.
 llvm::DINameSpace 
@@ -1659,7 +1910,7 @@ CGDebugInfo::getOrCreateNameSpace(const NamespaceDecl *NSDecl,
     getContextDescriptor(dyn_cast<Decl>(NSDecl->getDeclContext()), Unit);
   llvm::DINameSpace NS =
     DebugFactory.CreateNameSpace(Context, NSDecl->getName(), 
-	llvm::DIFile(Unit), LineNo);
+                                 llvm::DIFile(Unit), LineNo);
   NameSpaceCache[NSDecl] = llvm::WeakVH(NS);
   return NS;
 }
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index 620a5f2f8480..a1ad012353e5 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -46,9 +46,6 @@ class CGDebugInfo {
   llvm::DICompileUnit TheCU;
   SourceLocation CurLoc, PrevLoc;
   llvm::DIType VTablePtrType;
-  /// FwdDeclCount - This counter is used to ensure unique names for forward
-  /// record decls.
-  unsigned FwdDeclCount;
   
   /// TypeCache - Cache of previously constructed Types.
   llvm::DenseMap<void *, llvm::WeakVH> TypeCache;
@@ -58,10 +55,19 @@ class CGDebugInfo {
 
   std::vector<llvm::TrackingVH<llvm::MDNode> > RegionStack;
   llvm::DenseMap<const Decl *, llvm::WeakVH> RegionMap;
+  // FnBeginRegionCount - Keep track of RegionStack counter at the beginning
+  // of a function. This is used to pop unbalanced regions at the end of a
+  // function.
+  std::vector<unsigned> FnBeginRegionCount;
+
+  /// LineDirectiveFiles - This stack is used to keep track of 
+  /// scopes introduced by #line directives.
+  std::vector<const char *> LineDirectiveFiles;
 
   /// DebugInfoNames - This is a storage for names that are
   /// constructed on demand. For example, C++ destructors, C++ operators etc..
   llvm::BumpPtrAllocator DebugInfoNames;
+  llvm::StringRef CWDName;
 
   llvm::DenseMap<const char *, llvm::WeakVH> DIFileCache;
   llvm::DenseMap<const FunctionDecl *, llvm::WeakVH> SPCache;
@@ -86,6 +92,7 @@ class CGDebugInfo {
   llvm::DIType CreateType(const ArrayType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const LValueReferenceType *Ty, llvm::DIFile F);
   llvm::DIType CreateType(const MemberPointerType *Ty, llvm::DIFile F);
+  llvm::DIType CreateEnumType(const EnumDecl *ED, llvm::DIFile Unit);
   llvm::DIType getOrCreateMethodType(const CXXMethodDecl *Method,
                                      llvm::DIFile F);
   llvm::DIType getOrCreateVTablePtrType(llvm::DIFile F);
@@ -98,16 +105,22 @@ class CGDebugInfo {
   
   llvm::DISubprogram CreateCXXMemberFunction(const CXXMethodDecl *Method,
                                              llvm::DIFile F,
-                                             llvm::DICompositeType &RecordTy);
+                                             llvm::DIType RecordTy);
   
   void CollectCXXMemberFunctions(const CXXRecordDecl *Decl,
                                  llvm::DIFile F,
                                  llvm::SmallVectorImpl<llvm::DIDescriptor> &E,
-                                 llvm::DICompositeType &T);
+                                 llvm::DIType T);
+
+  void CollectCXXFriends(const CXXRecordDecl *Decl,
+                       llvm::DIFile F,
+                       llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
+                       llvm::DIType RecordTy);
+
   void CollectCXXBases(const CXXRecordDecl *Decl,
                        llvm::DIFile F,
                        llvm::SmallVectorImpl<llvm::DIDescriptor> &EltTys,
-                       llvm::DICompositeType &RecordTy);
+                       llvm::DIType RecordTy);
 
 
   void CollectRecordFields(const RecordDecl *Decl, llvm::DIFile F,
@@ -127,20 +140,27 @@ public:
 
   /// EmitStopPoint - Emit a call to llvm.dbg.stoppoint to indicate a change of
   /// source line.
-  void EmitStopPoint(llvm::Function *Fn, CGBuilderTy &Builder);
+  void EmitStopPoint(CGBuilderTy &Builder);
 
   /// EmitFunctionStart - Emit a call to llvm.dbg.function.start to indicate
   /// start of a new function.
   void EmitFunctionStart(GlobalDecl GD, QualType FnType,
                          llvm::Function *Fn, CGBuilderTy &Builder);
 
+  /// EmitFunctionEnd - Constructs the debug code for exiting a function.
+  void EmitFunctionEnd(CGBuilderTy &Builder);
+
+  /// UpdateLineDirectiveRegion - Update region stack only if #line directive
+  /// has introduced scope change.
+  void UpdateLineDirectiveRegion(CGBuilderTy &Builder);
+
   /// EmitRegionStart - Emit a call to llvm.dbg.region.start to indicate start
   /// of a new block.
-  void EmitRegionStart(llvm::Function *Fn, CGBuilderTy &Builder);
+  void EmitRegionStart(CGBuilderTy &Builder);
 
   /// EmitRegionEnd - Emit call to llvm.dbg.region.end to indicate end of a
   /// block.
-  void EmitRegionEnd(llvm::Function *Fn, CGBuilderTy &Builder);
+  void EmitRegionEnd(CGBuilderTy &Builder);
 
   /// EmitDeclareOfAutoVariable - Emit call to llvm.dbg.declare for an automatic
   /// variable declaration.
@@ -165,6 +185,10 @@ public:
   /// EmitGlobalVariable - Emit information about an objective-c interface.
   void EmitGlobalVariable(llvm::GlobalVariable *GV, ObjCInterfaceDecl *Decl);
 
+  /// EmitGlobalVariable - Emit global variable's debug info.
+  void EmitGlobalVariable(const ValueDecl *VD, llvm::ConstantInt *Init, 
+                          CGBuilderTy &Builder);
+
 private:
   /// EmitDeclare - Emit call to llvm.dbg.declare for a variable declaration.
   void EmitDeclare(const VarDecl *decl, unsigned Tag, llvm::Value *AI,
@@ -183,6 +207,9 @@ private:
   llvm::DIDescriptor getContextDescriptor(const Decl *Decl,
                                           llvm::DIDescriptor &CU);
 
+  /// getCurrentDirname - Return current directory name.
+  llvm::StringRef getCurrentDirname();
+
   /// CreateCompileUnit - Create new compile unit.
   void CreateCompileUnit();
 
@@ -205,6 +232,12 @@ private:
   /// name is constructred on demand (e.g. C++ destructor) then the name
   /// is stored on the side.
   llvm::StringRef getFunctionName(const FunctionDecl *FD);
+  /// getObjCMethodName - Returns the unmangled name of an Objective-C method.
+  /// This is the display name for the debugging info.  
+  llvm::StringRef getObjCMethodName(const ObjCMethodDecl *FD);
+
+  /// getClassName - Get class name including template argument list.
+  llvm::StringRef getClassName(RecordDecl *RD);
 
   /// getVTableName - Get vtable name for the given Class.
   llvm::StringRef getVTableName(const CXXRecordDecl *Decl);
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index 1a62ea95555d..57e5236c67e5 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -107,11 +107,11 @@ void CodeGenFunction::EmitBlockVarDecl(const VarDecl &D) {
     CGM.ErrorUnsupported(&D, "__asm__");
 
   switch (D.getStorageClass()) {
-  case VarDecl::None:
-  case VarDecl::Auto:
-  case VarDecl::Register:
+  case SC_None:
+  case SC_Auto:
+  case SC_Register:
     return EmitLocalBlockVarDecl(D);
-  case VarDecl::Static: {
+  case SC_Static: {
     llvm::GlobalValue::LinkageTypes Linkage = 
       llvm::GlobalValue::InternalLinkage;
 
@@ -126,8 +126,8 @@ void CodeGenFunction::EmitBlockVarDecl(const VarDecl &D) {
     
     return EmitStaticBlockVarDecl(D, Linkage);
   }
-  case VarDecl::Extern:
-  case VarDecl::PrivateExtern:
+  case SC_Extern:
+  case SC_PrivateExtern:
     // Don't emit it now, allow it to be emitted lazily on its first use.
     return;
   }
@@ -183,7 +183,7 @@ llvm::GlobalVariable *
 CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
                                                     llvm::GlobalVariable *GV) {
   llvm::Constant *Init = CGM.EmitConstantExpr(D.getInit(), D.getType(), this);
-  
+
   // If constant emission failed, then this should be a C++ static
   // initializer.
   if (!Init) {
@@ -198,12 +198,12 @@ CodeGenFunction::AddInitializerToGlobalBlockVarDecl(const VarDecl &D,
     }
     return GV;
   }
-  
+
   // The initializer may differ in type from the global. Rewrite
   // the global to match the initializer.  (We have to do this
   // because some types, like unions, can't be completely represented
   // in the LLVM type system.)
-  if (GV->getType() != Init->getType()) {
+  if (GV->getType()->getElementType() != Init->getType()) {
     llvm::GlobalVariable *OldGV = GV;
     
     GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
@@ -373,7 +373,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
   }
 
   // T x;
-  Types.push_back(ConvertType(Ty));
+  Types.push_back(ConvertTypeForMem(Ty));
   
   const llvm::Type *T = llvm::StructType::get(VMContext, Types, Packed);
   
@@ -389,7 +389,7 @@ const llvm::Type *CodeGenFunction::BuildByRefType(const ValueDecl *D) {
 }
 
 namespace {
-  struct CallArrayDtor : EHScopeStack::LazyCleanup {
+  struct CallArrayDtor : EHScopeStack::Cleanup {
     CallArrayDtor(const CXXDestructorDecl *Dtor, 
                   const ConstantArrayType *Type,
                   llvm::Value *Loc)
@@ -408,7 +408,7 @@ namespace {
     }
   };
 
-  struct CallVarDtor : EHScopeStack::LazyCleanup {
+  struct CallVarDtor : EHScopeStack::Cleanup {
     CallVarDtor(const CXXDestructorDecl *Dtor,
                 llvm::Value *NRVOFlag,
                 llvm::Value *Loc)
@@ -440,12 +440,64 @@ namespace {
   };
 }
 
+namespace {
+  struct CallStackRestore : EHScopeStack::Cleanup {
+    llvm::Value *Stack;
+    CallStackRestore(llvm::Value *Stack) : Stack(Stack) {}
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      llvm::Value *V = CGF.Builder.CreateLoad(Stack, "tmp");
+      llvm::Value *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
+      CGF.Builder.CreateCall(F, V);
+    }
+  };
+
+  struct CallCleanupFunction : EHScopeStack::Cleanup {
+    llvm::Constant *CleanupFn;
+    const CGFunctionInfo &FnInfo;
+    llvm::Value *Addr;
+    const VarDecl &Var;
+    
+    CallCleanupFunction(llvm::Constant *CleanupFn, const CGFunctionInfo *Info,
+                        llvm::Value *Addr, const VarDecl *Var)
+      : CleanupFn(CleanupFn), FnInfo(*Info), Addr(Addr), Var(*Var) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      // In some cases, the type of the function argument will be different from
+      // the type of the pointer. An example of this is
+      // void f(void* arg);
+      // __attribute__((cleanup(f))) void *g;
+      //
+      // To fix this we insert a bitcast here.
+      QualType ArgTy = FnInfo.arg_begin()->type;
+      llvm::Value *Arg =
+        CGF.Builder.CreateBitCast(Addr, CGF.ConvertType(ArgTy));
+
+      CallArgList Args;
+      Args.push_back(std::make_pair(RValue::get(Arg),
+                            CGF.getContext().getPointerType(Var.getType())));
+      CGF.EmitCall(FnInfo, CleanupFn, ReturnValueSlot(), Args);
+    }
+  };
+
+  struct CallBlockRelease : EHScopeStack::Cleanup {
+    llvm::Value *Addr;
+    CallBlockRelease(llvm::Value *Addr) : Addr(Addr) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      llvm::Value *V = CGF.Builder.CreateStructGEP(Addr, 1, "forwarding");
+      V = CGF.Builder.CreateLoad(V);
+      CGF.BuildBlockRelease(V);
+    }
+  };
+}
+
 /// EmitLocalBlockVarDecl - Emit code and set up an entry in LocalDeclMap for a
 /// variable declaration with auto, register, or no storage class specifier.
 /// These turn into simple stack objects, or GlobalValues depending on target.
 void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
                                             SpecialInitFn *SpecialInit) {
   QualType Ty = D.getType();
+  unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
   bool isByRef = D.hasAttr<BlocksAttr>();
   bool needsDispose = false;
   CharUnits Align = CharUnits::Zero();
@@ -461,10 +513,10 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
       // If this value is an array or struct, is POD, and if the initializer is
       // a staticly determinable constant, try to optimize it (unless the NRVO
       // is already optimizing this).
-      if (D.getInit() && !isByRef &&
+      if (!NRVO && D.getInit() && !isByRef &&
           (Ty->isArrayType() || Ty->isRecordType()) &&
           Ty->isPODType() &&
-          D.getInit()->isConstantInitializer(getContext()) && !NRVO) {
+          D.getInit()->isConstantInitializer(getContext(), false)) {
         // If this variable is marked 'const', emit the value as a global.
         if (CGM.getCodeGenOpts().MergeAllConstants &&
             Ty.isConstant(getContext())) {
@@ -516,7 +568,7 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
     } else {
       // Targets that don't support recursion emit locals as globals.
       const char *Class =
-        D.getStorageClass() == VarDecl::Register ? ".reg." : ".auto.";
+        D.getStorageClass() == SC_Register ? ".reg." : ".auto.";
       DeclPtr = CreateStaticBlockVarDecl(D, Class,
                                          llvm::GlobalValue
                                          ::InternalLinkage);
@@ -540,20 +592,14 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
 
       DidCallStackSave = true;
 
-      {
-        // Push a cleanup block and restore the stack there.
-        CleanupBlock scope(*this, NormalCleanup);
-
-        V = Builder.CreateLoad(Stack, "tmp");
-        llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
-        Builder.CreateCall(F, V);
-      }
+      // Push a cleanup block and restore the stack there.
+      EHStack.pushCleanup<CallStackRestore>(NormalCleanup, Stack);
     }
 
     // Get the element type.
     const llvm::Type *LElemTy = ConvertTypeForMem(Ty);
     const llvm::Type *LElemPtrTy =
-      llvm::PointerType::get(LElemTy, D.getType().getAddressSpace());
+      llvm::PointerType::get(LElemTy, Ty.getAddressSpace());
 
     llvm::Value *VLASize = EmitVLASize(Ty);
 
@@ -658,13 +704,12 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
       Loc = Builder.CreateStructGEP(DeclPtr, getByRefValueLLVMField(&D), 
                                     D.getNameAsString());
     
-    bool isVolatile =
-    getContext().getCanonicalType(D.getType()).isVolatileQualified();
+    bool isVolatile = getContext().getCanonicalType(Ty).isVolatileQualified();
     
     // If the initializer was a simple constant initializer, we can optimize it
     // in various ways.
     if (IsSimpleConstantInitializer) {
-      llvm::Constant *Init = CGM.EmitConstantExpr(D.getInit(),D.getType(),this);
+      llvm::Constant *Init = CGM.EmitConstantExpr(D.getInit(), Ty,this);
       assert(Init != 0 && "Wasn't a simple constant init?");
       
       llvm::Value *AlignVal = 
@@ -708,10 +753,10 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
       }
     } else if (Ty->isReferenceType()) {
       RValue RV = EmitReferenceBindingToExpr(Init, &D);
-      EmitStoreOfScalar(RV.getScalarVal(), Loc, false, Ty);
+      EmitStoreOfScalar(RV.getScalarVal(), Loc, false, Alignment, Ty);
     } else if (!hasAggregateLLVMType(Init->getType())) {
       llvm::Value *V = EmitScalarExpr(Init);
-      EmitStoreOfScalar(V, Loc, isVolatile, D.getType());
+      EmitStoreOfScalar(V, Loc, isVolatile, Alignment, Ty);
     } else if (Init->getType()->isAnyComplexType()) {
       EmitComplexExprIntoAddr(Init, Loc, isVolatile);
     } else {
@@ -738,11 +783,11 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
         
         if (const ConstantArrayType *Array = 
               getContext().getAsConstantArrayType(Ty)) {
-          EHStack.pushLazyCleanup<CallArrayDtor>(NormalAndEHCleanup,
-                                                 D, Array, Loc);
+          EHStack.pushCleanup<CallArrayDtor>(NormalAndEHCleanup,
+                                             D, Array, Loc);
         } else {
-          EHStack.pushLazyCleanup<CallVarDtor>(NormalAndEHCleanup,
-                                               D, NRVOFlag, Loc);
+          EHStack.pushCleanup<CallVarDtor>(NormalAndEHCleanup,
+                                           D, NRVOFlag, Loc);
         }
       }
   }
@@ -755,52 +800,14 @@ void CodeGenFunction::EmitLocalBlockVarDecl(const VarDecl &D,
     assert(F && "Could not find function!");
 
     const CGFunctionInfo &Info = CGM.getTypes().getFunctionInfo(FD);
-
-    // In some cases, the type of the function argument will be different from
-    // the type of the pointer. An example of this is
-    // void f(void* arg);
-    // __attribute__((cleanup(f))) void *g;
-    //
-    // To fix this we insert a bitcast here.
-    QualType ArgTy = Info.arg_begin()->type;
-
-    CleanupBlock CleanupScope(*this, NormalCleanup);
-
-    // Normal cleanup.
-    CallArgList Args;
-    Args.push_back(std::make_pair(RValue::get(Builder.CreateBitCast(DeclPtr,
-                                                         ConvertType(ArgTy))),
-                                  getContext().getPointerType(D.getType())));
-    EmitCall(Info, F, ReturnValueSlot(), Args);
-
-    // EH cleanup.
-    if (Exceptions) {
-      CleanupScope.beginEHCleanup();
-
-      CallArgList Args;
-      Args.push_back(std::make_pair(RValue::get(Builder.CreateBitCast(DeclPtr,
-                                                           ConvertType(ArgTy))),
-                                    getContext().getPointerType(D.getType())));
-      EmitCall(Info, F, ReturnValueSlot(), Args);
-    }
+    EHStack.pushCleanup<CallCleanupFunction>(NormalAndEHCleanup,
+                                             F, &Info, DeclPtr, &D);
   }
 
-  if (needsDispose && CGM.getLangOptions().getGCMode() != LangOptions::GCOnly) {
-    CleanupBlock CleanupScope(*this, NormalCleanup);
-
-    llvm::Value *V = Builder.CreateStructGEP(DeclPtr, 1, "forwarding");
-    V = Builder.CreateLoad(V);
-    BuildBlockRelease(V);
-
-    // FIXME: Turn this on and audit the codegen
-    if (0 && Exceptions) {
-      CleanupScope.beginEHCleanup();
-
-      llvm::Value *V = Builder.CreateStructGEP(DeclPtr, 1, "forwarding");
-      V = Builder.CreateLoad(V);
-      BuildBlockRelease(V);
-    }
-  }
+  // If this is a block variable, clean it up.
+  // FIXME: this should be an EH cleanup as well.  rdar://problem/8224178
+  if (needsDispose && CGM.getLangOptions().getGCMode() != LangOptions::GCOnly)
+    EHStack.pushCleanup<CallBlockRelease>(NormalCleanup, DeclPtr);
 }
 
 /// Emit an alloca (or GlobalValue depending on target)
@@ -822,7 +829,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg) {
     DeclPtr = CreateMemTemp(Ty, D.getName() + ".addr");
 
     // Store the initial value into the alloca.
-    EmitStoreOfScalar(Arg, DeclPtr, CTy.isVolatileQualified(), Ty);
+    unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
+    EmitStoreOfScalar(Arg, DeclPtr, CTy.isVolatileQualified(), Alignment, Ty);
   }
   Arg->setName(D.getName());
 
diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp
index ec3f38667b7a..e2f197522b26 100644
--- a/lib/CodeGen/CGDeclCXX.cpp
+++ b/lib/CodeGen/CGDeclCXX.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenFunction.h"
+#include "CGCXXABI.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Intrinsics.h"
 
@@ -30,9 +31,10 @@ static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
   QualType T = D.getType();
   bool isVolatile = Context.getCanonicalType(T).isVolatileQualified();
 
+  unsigned Alignment = Context.getDeclAlign(&D).getQuantity();
   if (!CGF.hasAggregateLLVMType(T)) {
     llvm::Value *V = CGF.EmitScalarExpr(Init);
-    CGF.EmitStoreOfScalar(V, DeclPtr, isVolatile, T);
+    CGF.EmitStoreOfScalar(V, DeclPtr, isVolatile, Alignment, T);
   } else if (T->isAnyComplexType()) {
     CGF.EmitComplexExprIntoAddr(Init, DeclPtr, isVolatile);
   } else {
@@ -45,19 +47,15 @@ static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
   CodeGenModule &CGM = CGF.CGM;
   ASTContext &Context = CGF.getContext();
   
-  const Expr *Init = D.getInit();
   QualType T = D.getType();
-  if (!CGF.hasAggregateLLVMType(T) || T->isAnyComplexType())
-    return;
-                                
-  // Avoid generating destructor(s) for initialized objects. 
-  if (!isa<CXXConstructExpr>(Init))
-    return;
   
+  // Drill down past array types.
   const ConstantArrayType *Array = Context.getAsConstantArrayType(T);
   if (Array)
     T = Context.getBaseElementType(Array);
   
+  /// If that's not a record, we're done.
+  /// FIXME:  __attribute__((cleanup)) ?
   const RecordType *RT = T->getAs<RecordType>();
   if (!RT)
     return;
@@ -94,8 +92,9 @@ void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
     return;
   }
 
+  unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
   RValue RV = EmitReferenceBindingToExpr(Init, &D);
-  EmitStoreOfScalar(RV.getScalarVal(), DeclPtr, false, T);
+  EmitStoreOfScalar(RV.getScalarVal(), DeclPtr, false, Alignment, T);
 }
 
 void
@@ -174,13 +173,26 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D) {
     unsigned int order = D->getAttr<InitPriorityAttr>()->getPriority();
     OrderGlobalInits Key(order, PrioritizedCXXGlobalInits.size());
     PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
+    DelayedCXXInitPosition.erase(D);
+  }
+  else {
+    llvm::DenseMap<const Decl *, unsigned>::iterator I =
+      DelayedCXXInitPosition.find(D);
+    if (I == DelayedCXXInitPosition.end()) {
+      CXXGlobalInits.push_back(Fn);
+    } else {
+      assert(CXXGlobalInits[I->second] == 0);
+      CXXGlobalInits[I->second] = Fn;
+      DelayedCXXInitPosition.erase(I);
+    }
   }
-  else
-    CXXGlobalInits.push_back(Fn);
 }
 
 void
 CodeGenModule::EmitCXXGlobalInitFunc() {
+  while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
+    CXXGlobalInits.pop_back();
+
   if (CXXGlobalInits.empty() && PrioritizedCXXGlobalInits.empty())
     return;
 
@@ -200,8 +212,7 @@ CodeGenModule::EmitCXXGlobalInitFunc() {
       llvm::Function *Fn = PrioritizedCXXGlobalInits[i].second;
       LocalCXXGlobalInits.push_back(Fn);
     }
-    for (unsigned i = 0; i < CXXGlobalInits.size(); i++)
-      LocalCXXGlobalInits.push_back(CXXGlobalInits[i]);
+    LocalCXXGlobalInits.append(CXXGlobalInits.begin(), CXXGlobalInits.end());
     CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn,
                                                     &LocalCXXGlobalInits[0],
                                                     LocalCXXGlobalInits.size());
@@ -247,7 +258,8 @@ void CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
                 SourceLocation());
 
   for (unsigned i = 0; i != NumDecls; ++i)
-    Builder.CreateCall(Decls[i]);
+    if (Decls[i])
+      Builder.CreateCall(Decls[i]);
 
   FinishFunction();
 }
@@ -316,6 +328,20 @@ static llvm::Constant *getGuardAbortFn(CodeGenFunction &CGF) {
   return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort");
 }
 
+namespace {
+  struct CallGuardAbort : EHScopeStack::Cleanup {
+    llvm::GlobalVariable *Guard;
+    CallGuardAbort(llvm::GlobalVariable *Guard) : Guard(Guard) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      // It shouldn't be possible for this to throw, but if it can,
+      // this should allow for the possibility of an invoke.
+      CGF.Builder.CreateCall(getGuardAbortFn(CGF), Guard)
+        ->setDoesNotThrow();
+    }
+  };
+}
+
 void
 CodeGenFunction::EmitStaticCXXBlockVarDeclInit(const VarDecl &D,
                                                llvm::GlobalVariable *GV) {
@@ -325,10 +351,10 @@ CodeGenFunction::EmitStaticCXXBlockVarDeclInit(const VarDecl &D,
   bool ThreadsafeStatics = getContext().getLangOptions().ThreadsafeStatics;
   
   llvm::SmallString<256> GuardVName;
-  CGM.getMangleContext().mangleGuardVariable(&D, GuardVName);
+  CGM.getCXXABI().getMangleContext().mangleGuardVariable(&D, GuardVName);
 
   // Create the guard variable.
-  llvm::GlobalValue *GuardVariable =
+  llvm::GlobalVariable *GuardVariable =
     new llvm::GlobalVariable(CGM.getModule(), Int64Ty,
                              false, GV->getLinkage(),
                              llvm::Constant::getNullValue(Int64Ty),
@@ -360,23 +386,25 @@ CodeGenFunction::EmitStaticCXXBlockVarDeclInit(const VarDecl &D,
                          InitBlock, EndBlock);
   
     // Call __cxa_guard_abort along the exceptional edge.
-    if (Exceptions) {
-      CleanupBlock Cleanup(*this, EHCleanup);
-      Builder.CreateCall(getGuardAbortFn(*this), GuardVariable);
-    }
+    if (Exceptions)
+      EHStack.pushCleanup<CallGuardAbort>(EHCleanup, GuardVariable);
     
     EmitBlock(InitBlock);
   }
 
   if (D.getType()->isReferenceType()) {
+    unsigned Alignment = getContext().getDeclAlign(&D).getQuantity();
     QualType T = D.getType();
     RValue RV = EmitReferenceBindingToExpr(D.getInit(), &D);
-    EmitStoreOfScalar(RV.getScalarVal(), GV, /*Volatile=*/false, T);
-
+    EmitStoreOfScalar(RV.getScalarVal(), GV, /*Volatile=*/false, Alignment, T);
   } else
     EmitDeclInit(*this, D, GV);
 
   if (ThreadsafeStatics) {
+    // Pop the guard-abort cleanup if we pushed one.
+    if (Exceptions)
+      PopCleanupBlock();
+
     // Call __cxa_guard_release.  This cannot throw.
     Builder.CreateCall(getGuardReleaseFn(*this), GuardVariable);    
   } else {
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index 4980aad1b383..7fb616e5a150 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -16,8 +16,10 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/Support/CallSite.h"
 
+#include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "CGException.h"
+#include "TargetInfo.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -62,29 +64,26 @@ EHScopeStack::getEnclosingEHCleanup(iterator it) const {
         return stabilize(it);
       return cast<EHCleanupScope>(*it).getEnclosingEHCleanup();
     }
-    if (isa<EHLazyCleanupScope>(*it)) {
-      if (cast<EHLazyCleanupScope>(*it).isEHCleanup())
-        return stabilize(it);
-      return cast<EHLazyCleanupScope>(*it).getEnclosingEHCleanup();
-    }
     ++it;
   } while (it != end());
   return stable_end();
 }
 
 
-void *EHScopeStack::pushLazyCleanup(CleanupKind Kind, size_t Size) {
+void *EHScopeStack::pushCleanup(CleanupKind Kind, size_t Size) {
   assert(((Size % sizeof(void*)) == 0) && "cleanup type is misaligned");
-  char *Buffer = allocate(EHLazyCleanupScope::getSizeForCleanupSize(Size));
-  bool IsNormalCleanup = Kind != EHCleanup;
-  bool IsEHCleanup = Kind != NormalCleanup;
-  EHLazyCleanupScope *Scope =
-    new (Buffer) EHLazyCleanupScope(IsNormalCleanup,
-                                    IsEHCleanup,
-                                    Size,
-                                    BranchFixups.size(),
-                                    InnermostNormalCleanup,
-                                    InnermostEHCleanup);
+  char *Buffer = allocate(EHCleanupScope::getSizeForCleanupSize(Size));
+  bool IsNormalCleanup = Kind & NormalCleanup;
+  bool IsEHCleanup = Kind & EHCleanup;
+  bool IsActive = !(Kind & InactiveCleanup);
+  EHCleanupScope *Scope =
+    new (Buffer) EHCleanupScope(IsNormalCleanup,
+                                IsEHCleanup,
+                                IsActive,
+                                Size,
+                                BranchFixups.size(),
+                                InnermostNormalCleanup,
+                                InnermostEHCleanup);
   if (IsNormalCleanup)
     InnermostNormalCleanup = stable_begin();
   if (IsEHCleanup)
@@ -93,36 +92,19 @@ void *EHScopeStack::pushLazyCleanup(CleanupKind Kind, size_t Size) {
   return Scope->getCleanupBuffer();
 }
 
-void EHScopeStack::pushCleanup(llvm::BasicBlock *NormalEntry,
-                               llvm::BasicBlock *NormalExit,
-                               llvm::BasicBlock *EHEntry,
-                               llvm::BasicBlock *EHExit) {
-  char *Buffer = allocate(EHCleanupScope::getSize());
-  new (Buffer) EHCleanupScope(BranchFixups.size(),
-                              InnermostNormalCleanup,
-                              InnermostEHCleanup,
-                              NormalEntry, NormalExit, EHEntry, EHExit);
-  if (NormalEntry)
-    InnermostNormalCleanup = stable_begin();
-  if (EHEntry)
-    InnermostEHCleanup = stable_begin();
-}
-
 void EHScopeStack::popCleanup() {
   assert(!empty() && "popping exception stack when not empty");
 
-  if (isa<EHLazyCleanupScope>(*begin())) {
-    EHLazyCleanupScope &Cleanup = cast<EHLazyCleanupScope>(*begin());
-    InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
-    InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
-    StartOfData += Cleanup.getAllocatedSize();
-  } else {
-    assert(isa<EHCleanupScope>(*begin()));
-    EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
-    InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
-    InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
-    StartOfData += EHCleanupScope::getSize();
-  }
+  assert(isa<EHCleanupScope>(*begin()));
+  EHCleanupScope &Cleanup = cast<EHCleanupScope>(*begin());
+  InnermostNormalCleanup = Cleanup.getEnclosingNormalCleanup();
+  InnermostEHCleanup = Cleanup.getEnclosingEHCleanup();
+  StartOfData += Cleanup.getAllocatedSize();
+
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
+  // Destroy the cleanup.
+  Cleanup.~EHCleanupScope();
 
   // Check whether we can shrink the branch-fixups stack.
   if (!BranchFixups.empty()) {
@@ -149,6 +131,8 @@ void EHScopeStack::popFilter() {
   EHFilterScope &Filter = cast<EHFilterScope>(*begin());
   StartOfData += EHFilterScope::getSizeForNumFilters(Filter.getNumFilters());
 
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
   assert(CatchDepth > 0 && "mismatched filter push/pop");
   CatchDepth--;
 }
@@ -156,13 +140,16 @@ void EHScopeStack::popFilter() {
 EHCatchScope *EHScopeStack::pushCatch(unsigned NumHandlers) {
   char *Buffer = allocate(EHCatchScope::getSizeForNumHandlers(NumHandlers));
   CatchDepth++;
-  return new (Buffer) EHCatchScope(NumHandlers);
+  EHCatchScope *Scope = new (Buffer) EHCatchScope(NumHandlers);
+  for (unsigned I = 0; I != NumHandlers; ++I)
+    Scope->getHandlers()[I].Index = getNextEHDestIndex();
+  return Scope;
 }
 
 void EHScopeStack::pushTerminate() {
   char *Buffer = allocate(EHTerminateScope::getSize());
   CatchDepth++;
-  new (Buffer) EHTerminateScope();
+  new (Buffer) EHTerminateScope(getNextEHDestIndex());
 }
 
 /// Remove any 'null' fixups on the stack.  However, we can't pop more
@@ -176,11 +163,7 @@ void EHScopeStack::popNullFixups() {
   assert(hasNormalCleanups());
 
   EHScopeStack::iterator it = find(InnermostNormalCleanup);
-  unsigned MinSize;
-  if (isa<EHCleanupScope>(*it))
-    MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
-  else
-    MinSize = cast<EHLazyCleanupScope>(*it).getFixupDepth();
+  unsigned MinSize = cast<EHCleanupScope>(*it).getFixupDepth();
   assert(BranchFixups.size() >= MinSize && "fixup stack out of order");
 
   while (BranchFixups.size() > MinSize &&
@@ -188,20 +171,6 @@ void EHScopeStack::popNullFixups() {
     BranchFixups.pop_back();
 }
 
-void EHScopeStack::resolveBranchFixups(llvm::BasicBlock *Dest) {
-  assert(Dest && "null block passed to resolveBranchFixups");
-
-  if (BranchFixups.empty()) return;
-  assert(hasNormalCleanups() &&
-         "branch fixups exist with no normal cleanups on stack");
-
-  for (unsigned I = 0, E = BranchFixups.size(); I != E; ++I)
-    if (BranchFixups[I].Destination == Dest)
-      BranchFixups[I].Destination = 0;
-
-  popNullFixups();
-}
-
 static llvm::Constant *getAllocateExceptionFn(CodeGenFunction &CGF) {
   // void *__cxa_allocate_exception(size_t thrown_size);
   const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
@@ -303,7 +272,7 @@ llvm::Constant *CodeGenFunction::getUnwindResumeOrRethrowFn() {
                             false);
 
   if (CGM.getLangOptions().SjLjExceptions)
-    return CGM.CreateRuntimeFunction(FTy, "_Unwind_SjLj_Resume");
+    return CGM.CreateRuntimeFunction(FTy, "_Unwind_SjLj_Resume_or_Rethrow");
   return CGM.CreateRuntimeFunction(FTy, "_Unwind_Resume_or_Rethrow");
 }
 
@@ -317,74 +286,88 @@ static llvm::Constant *getTerminateFn(CodeGenFunction &CGF) {
       CGF.CGM.getLangOptions().CPlusPlus ? "_ZSt9terminatev" : "abort");
 }
 
-static const char *getCPersonalityFn(CodeGenFunction &CGF) {
-  return "__gcc_personality_v0";
+static llvm::Constant *getCatchallRethrowFn(CodeGenFunction &CGF,
+                                            const char *Name) {
+  const llvm::Type *Int8PtrTy =
+    llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  std::vector<const llvm::Type*> Args(1, Int8PtrTy);
+
+  const llvm::Type *VoidTy = llvm::Type::getVoidTy(CGF.getLLVMContext());
+  const llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, Args, false);
+
+  return CGF.CGM.CreateRuntimeFunction(FTy, Name);
+}
+
+const EHPersonality EHPersonality::GNU_C("__gcc_personality_v0");
+const EHPersonality EHPersonality::NeXT_ObjC("__objc_personality_v0");
+const EHPersonality EHPersonality::GNU_CPlusPlus("__gxx_personality_v0");
+const EHPersonality EHPersonality::GNU_CPlusPlus_SJLJ("__gxx_personality_sj0");
+const EHPersonality EHPersonality::GNU_ObjC("__gnu_objc_personality_v0",
+                                            "objc_exception_throw");
+
+static const EHPersonality &getCPersonality(const LangOptions &L) {
+  return EHPersonality::GNU_C;
 }
 
-static const char *getObjCPersonalityFn(CodeGenFunction &CGF) {
-  if (CGF.CGM.getLangOptions().NeXTRuntime) {
-    if (CGF.CGM.getLangOptions().ObjCNonFragileABI)
-      return "__objc_personality_v0";
-    else
-      return getCPersonalityFn(CGF);
+static const EHPersonality &getObjCPersonality(const LangOptions &L) {
+  if (L.NeXTRuntime) {
+    if (L.ObjCNonFragileABI) return EHPersonality::NeXT_ObjC;
+    else return getCPersonality(L);
   } else {
-    return "__gnu_objc_personality_v0";
+    return EHPersonality::GNU_ObjC;
   }
 }
 
-static const char *getCXXPersonalityFn(CodeGenFunction &CGF) {
-  if (CGF.CGM.getLangOptions().SjLjExceptions)
-    return "__gxx_personality_sj0";
+static const EHPersonality &getCXXPersonality(const LangOptions &L) {
+  if (L.SjLjExceptions)
+    return EHPersonality::GNU_CPlusPlus_SJLJ;
   else
-    return "__gxx_personality_v0";
+    return EHPersonality::GNU_CPlusPlus;
 }
 
 /// Determines the personality function to use when both C++
 /// and Objective-C exceptions are being caught.
-static const char *getObjCXXPersonalityFn(CodeGenFunction &CGF) {
+static const EHPersonality &getObjCXXPersonality(const LangOptions &L) {
   // The ObjC personality defers to the C++ personality for non-ObjC
   // handlers.  Unlike the C++ case, we use the same personality
   // function on targets using (backend-driven) SJLJ EH.
-  if (CGF.CGM.getLangOptions().NeXTRuntime) {
-    if (CGF.CGM.getLangOptions().ObjCNonFragileABI)
-      return "__objc_personality_v0";
+  if (L.NeXTRuntime) {
+    if (L.ObjCNonFragileABI)
+      return EHPersonality::NeXT_ObjC;
 
     // In the fragile ABI, just use C++ exception handling and hope
     // they're not doing crazy exception mixing.
     else
-      return getCXXPersonalityFn(CGF);
+      return getCXXPersonality(L);
   }
 
-  // I'm pretty sure the GNU runtime doesn't support mixed EH.
-  // TODO: we don't necessarily need mixed EH here;  remember what
-  // kind of exceptions we actually try to catch in this function.
-  CGF.CGM.ErrorUnsupported(CGF.CurCodeDecl,
-                           "the GNU Objective C runtime does not support "
-                           "catching C++ and Objective C exceptions in the "
-                           "same function");
-  // Use the C++ personality just to avoid returning null.
-  return getCXXPersonalityFn(CGF);
+  // The GNU runtime's personality function inherently doesn't support
+  // mixed EH.  Use the C++ personality just to avoid returning null.
+  return getCXXPersonality(L);
 }
 
-static llvm::Constant *getPersonalityFn(CodeGenFunction &CGF) {
-  const char *Name;
-  const LangOptions &Opts = CGF.CGM.getLangOptions();
-  if (Opts.CPlusPlus && Opts.ObjC1)
-    Name = getObjCXXPersonalityFn(CGF);
-  else if (Opts.CPlusPlus)
-    Name = getCXXPersonalityFn(CGF);
-  else if (Opts.ObjC1)
-    Name = getObjCPersonalityFn(CGF);
+const EHPersonality &EHPersonality::get(const LangOptions &L) {
+  if (L.CPlusPlus && L.ObjC1)
+    return getObjCXXPersonality(L);
+  else if (L.CPlusPlus)
+    return getCXXPersonality(L);
+  else if (L.ObjC1)
+    return getObjCPersonality(L);
   else
-    Name = getCPersonalityFn(CGF);
+    return getCPersonality(L);
+}
+
+static llvm::Constant *getPersonalityFn(CodeGenFunction &CGF,
+                                        const EHPersonality &Personality) {
+  const char *Name = Personality.getPersonalityFnName();
 
-  llvm::Constant *Personality =
+  llvm::Constant *Fn =
     CGF.CGM.CreateRuntimeFunction(llvm::FunctionType::get(
                                     llvm::Type::getInt32Ty(
                                       CGF.CGM.getLLVMContext()),
                                     true),
                             Name);
-  return llvm::ConstantExpr::getBitCast(Personality, CGF.CGM.PtrToInt8Ty);
+  return llvm::ConstantExpr::getBitCast(Fn, CGF.CGM.PtrToInt8Ty);
 }
 
 /// Returns the value to inject into a selector to indicate the
@@ -403,7 +386,7 @@ static llvm::Constant *getCleanupValue(CodeGenFunction &CGF) {
 namespace {
   /// A cleanup to free the exception object if its initialization
   /// throws.
-  struct FreeExceptionCleanup : EHScopeStack::LazyCleanup {
+  struct FreeExceptionCleanup : EHScopeStack::Cleanup {
     FreeExceptionCleanup(llvm::Value *ShouldFreeVar,
                          llvm::Value *ExnLocVar)
       : ShouldFreeVar(ShouldFreeVar), ExnLocVar(ExnLocVar) {}
@@ -453,9 +436,9 @@ static void EmitAnyExprToExn(CodeGenFunction &CGF, const Expr *E,
   // exception during initialization.
   // FIXME: stmt expressions might require this to be a normal
   // cleanup, too.
-  CGF.EHStack.pushLazyCleanup<FreeExceptionCleanup>(EHCleanup,
-                                                    ShouldFreeVar,
-                                                    ExnLocVar);
+  CGF.EHStack.pushCleanup<FreeExceptionCleanup>(EHCleanup,
+                                                ShouldFreeVar,
+                                                ExnLocVar);
   EHScopeStack::stable_iterator Cleanup = CGF.EHStack.stable_begin();
 
   CGF.Builder.CreateStore(ExnLoc, ExnLocVar);
@@ -637,7 +620,12 @@ void CodeGenFunction::EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
       //   http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_active.html#388
       QualType CaughtType = C->getCaughtType();
       CaughtType = CaughtType.getNonReferenceType().getUnqualifiedType();
-      llvm::Value *TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, true);
+
+      llvm::Value *TypeInfo = 0;
+      if (CaughtType->isObjCObjectPointerType())
+        TypeInfo = CGM.getObjCRuntime().GetEHType(CaughtType);
+      else
+        TypeInfo = CGM.GetAddrOfRTTIDescriptor(CaughtType, true);
       CatchScope->setHandler(I, TypeInfo, Handler);
     } else {
       // No exception decl indicates '...', a catch-all.
@@ -653,8 +641,6 @@ static bool isNonEHScope(const EHScope &S) {
   switch (S.getKind()) {
   case EHScope::Cleanup:
     return !cast<EHCleanupScope>(S).isEHCleanup();
-  case EHScope::LazyCleanup:
-    return !cast<EHLazyCleanupScope>(S).isEHCleanup();
   case EHScope::Filter:
   case EHScope::Catch:
   case EHScope::Terminate:
@@ -753,6 +739,9 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   // Save the current IR generation state.
   CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
 
+  const EHPersonality &Personality =
+    EHPersonality::get(CGF.CGM.getLangOptions());
+
   // Create and configure the landing pad.
   llvm::BasicBlock *LP = createBasicBlock("lpad");
   EmitBlock(LP);
@@ -768,11 +757,11 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   // Build the selector arguments.
   llvm::SmallVector<llvm::Value*, 8> EHSelector;
   EHSelector.push_back(Exn);
-  EHSelector.push_back(getPersonalityFn(*this));
+  EHSelector.push_back(getPersonalityFn(*this, Personality));
 
   // Accumulate all the handlers in scope.
-  llvm::DenseMap<llvm::Value*, JumpDest> EHHandlers;
-  JumpDest CatchAll;
+  llvm::DenseMap<llvm::Value*, UnwindDest> EHHandlers;
+  UnwindDest CatchAll;
   bool HasEHCleanup = false;
   bool HasEHFilter = false;
   llvm::SmallVector<llvm::Value*, 8> EHFilters;
@@ -780,12 +769,6 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
          I != E; ++I) {
 
     switch (I->getKind()) {
-    case EHScope::LazyCleanup:
-      if (!HasEHCleanup)
-        HasEHCleanup = cast<EHLazyCleanupScope>(*I).isEHCleanup();
-      // We otherwise don't care about cleanups.
-      continue;
-
     case EHScope::Cleanup:
       if (!HasEHCleanup)
         HasEHCleanup = cast<EHCleanupScope>(*I).isEHCleanup();
@@ -794,7 +777,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
     case EHScope::Filter: {
       assert(I.next() == EHStack.end() && "EH filter is not end of EH stack");
-      assert(!CatchAll.Block && "EH filter reached after catch-all");
+      assert(!CatchAll.isValid() && "EH filter reached after catch-all");
 
       // Filter scopes get added to the selector in wierd ways.
       EHFilterScope &Filter = cast<EHFilterScope>(*I);
@@ -812,9 +795,10 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
     case EHScope::Terminate:
       // Terminate scopes are basically catch-alls.
-      assert(!CatchAll.Block);
-      CatchAll.Block = getTerminateHandler();
-      CatchAll.ScopeDepth = EHStack.getEnclosingEHCleanup(I);
+      assert(!CatchAll.isValid());
+      CatchAll = UnwindDest(getTerminateHandler(),
+                            EHStack.getEnclosingEHCleanup(I),
+                            cast<EHTerminateScope>(*I).getDestIndex());
       goto done;
 
     case EHScope::Catch:
@@ -827,30 +811,32 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
       // Catch-all.  We should only have one of these per catch.
       if (!Handler.Type) {
-        assert(!CatchAll.Block);
-        CatchAll.Block = Handler.Block;
-        CatchAll.ScopeDepth = EHStack.getEnclosingEHCleanup(I);
+        assert(!CatchAll.isValid());
+        CatchAll = UnwindDest(Handler.Block,
+                              EHStack.getEnclosingEHCleanup(I),
+                              Handler.Index);
         continue;
       }
 
       // Check whether we already have a handler for this type.
-      JumpDest &Dest = EHHandlers[Handler.Type];
-      if (Dest.Block) continue;
+      UnwindDest &Dest = EHHandlers[Handler.Type];
+      if (Dest.isValid()) continue;
 
       EHSelector.push_back(Handler.Type);
-      Dest.Block = Handler.Block;
-      Dest.ScopeDepth = EHStack.getEnclosingEHCleanup(I);
+      Dest = UnwindDest(Handler.Block,
+                        EHStack.getEnclosingEHCleanup(I),
+                        Handler.Index);
     }
 
     // Stop if we found a catch-all.
-    if (CatchAll.Block) break;
+    if (CatchAll.isValid()) break;
   }
 
  done:
   unsigned LastToEmitInLoop = EHSelector.size();
 
   // If we have a catch-all, add null to the selector.
-  if (CatchAll.Block) {
+  if (CatchAll.isValid()) {
     EHSelector.push_back(getCatchAllValue(CGF));
 
   // If we have an EH filter, we need to add those handlers in the
@@ -899,14 +885,15 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   // filter (possibly with a cleanup), a catch-all, or another catch).
   for (unsigned I = 2; I != LastToEmitInLoop; ++I) {
     llvm::Value *Type = EHSelector[I];
-    JumpDest Dest = EHHandlers[Type];
-    assert(Dest.Block && "no handler entry for value in selector?");
+    UnwindDest Dest = EHHandlers[Type];
+    assert(Dest.isValid() && "no handler entry for value in selector?");
 
     // Figure out where to branch on a match.  As a debug code-size
     // optimization, if the scope depth matches the innermost cleanup,
     // we branch directly to the catch handler.
-    llvm::BasicBlock *Match = Dest.Block;
-    bool MatchNeedsCleanup = Dest.ScopeDepth != EHStack.getInnermostEHCleanup();
+    llvm::BasicBlock *Match = Dest.getBlock();
+    bool MatchNeedsCleanup =
+      Dest.getScopeDepth() != EHStack.getInnermostEHCleanup();
     if (MatchNeedsCleanup)
       Match = createBasicBlock("eh.match");
 
@@ -932,7 +919,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
   // Emit the final case in the selector.
   // This might be a catch-all....
-  if (CatchAll.Block) {
+  if (CatchAll.isValid()) {
     assert(isa<llvm::ConstantPointerNull>(EHSelector.back()));
     EmitBranchThroughEHCleanup(CatchAll);
 
@@ -951,7 +938,8 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
       }
 
       llvm::BasicBlock *CleanupContBB = createBasicBlock("ehspec.cleanup.cont");
-      EmitBranchThroughEHCleanup(JumpDest(CleanupContBB, EHStack.stable_end()));
+      EmitBranchThroughEHCleanup(UnwindDest(CleanupContBB, EHStack.stable_end(),
+                                            EHStack.getNextEHDestIndex()));
       EmitBlock(CleanupContBB);
 
       if (HasEHCleanup)
@@ -996,22 +984,7 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
 
   // ...or a cleanup.
   } else {
-    // We emit a jump to a notional label at the outermost unwind state.
-    llvm::BasicBlock *Unwind = createBasicBlock("eh.resume");
-    JumpDest Dest(Unwind, EHStack.stable_end());
-    EmitBranchThroughEHCleanup(Dest);
-
-    // The unwind block.  We have to reload the exception here because
-    // we might have unwound through arbitrary blocks, so the landing
-    // pad might not dominate.
-    EmitBlock(Unwind);
-
-    // This can always be a call because we necessarily didn't find
-    // anything on the EH stack which needs our help.
-    Builder.CreateCall(getUnwindResumeOrRethrowFn(),
-                       Builder.CreateLoad(getExceptionSlot()))
-      ->setDoesNotReturn();
-    Builder.CreateUnreachable();
+    EmitBranchThroughEHCleanup(getRethrowDest());
   }
 
   // Restore the old IR generation state.
@@ -1033,7 +1006,7 @@ namespace {
   ///     of the caught type, so we have to assume the actual thrown
   ///     exception type might have a throwing destructor, even if the
   ///     caught type's destructor is trivial or nothrow.
-  struct CallEndCatch : EHScopeStack::LazyCleanup {
+  struct CallEndCatch : EHScopeStack::Cleanup {
     CallEndCatch(bool MightThrow) : MightThrow(MightThrow) {}
     bool MightThrow;
 
@@ -1058,7 +1031,7 @@ static llvm::Value *CallBeginCatch(CodeGenFunction &CGF,
   llvm::CallInst *Call = CGF.Builder.CreateCall(getBeginCatchFn(CGF), Exn);
   Call->setDoesNotThrow();
 
-  CGF.EHStack.pushLazyCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow);
+  CGF.EHStack.pushCleanup<CallEndCatch>(NormalAndEHCleanup, EndMightThrow);
 
   return Call;
 }
@@ -1078,11 +1051,57 @@ static void InitCatchParam(CodeGenFunction &CGF,
   // If we're catching by reference, we can just cast the object
   // pointer to the appropriate pointer.
   if (isa<ReferenceType>(CatchType)) {
-    bool EndCatchMightThrow = cast<ReferenceType>(CatchType)->getPointeeType()
-      ->isRecordType();
+    QualType CaughtType = cast<ReferenceType>(CatchType)->getPointeeType();
+    bool EndCatchMightThrow = CaughtType->isRecordType();
 
     // __cxa_begin_catch returns the adjusted object pointer.
     llvm::Value *AdjustedExn = CallBeginCatch(CGF, Exn, EndCatchMightThrow);
+
+    // We have no way to tell the personality function that we're
+    // catching by reference, so if we're catching a pointer,
+    // __cxa_begin_catch will actually return that pointer by value.
+    if (const PointerType *PT = dyn_cast<PointerType>(CaughtType)) {
+      QualType PointeeType = PT->getPointeeType();
+
+      // When catching by reference, generally we should just ignore
+      // this by-value pointer and use the exception object instead.
+      if (!PointeeType->isRecordType()) {
+
+        // Exn points to the struct _Unwind_Exception header, which
+        // we have to skip past in order to reach the exception data.
+        unsigned HeaderSize =
+          CGF.CGM.getTargetCodeGenInfo().getSizeOfUnwindException();
+        AdjustedExn = CGF.Builder.CreateConstGEP1_32(Exn, HeaderSize);
+
+      // However, if we're catching a pointer-to-record type that won't
+      // work, because the personality function might have adjusted
+      // the pointer.  There's actually no way for us to fully satisfy
+      // the language/ABI contract here:  we can't use Exn because it
+      // might have the wrong adjustment, but we can't use the by-value
+      // pointer because it's off by a level of abstraction.
+      //
+      // The current solution is to dump the adjusted pointer into an
+      // alloca, which breaks language semantics (because changing the
+      // pointer doesn't change the exception) but at least works.
+      // The better solution would be to filter out non-exact matches
+      // and rethrow them, but this is tricky because the rethrow
+      // really needs to be catchable by other sites at this landing
+      // pad.  The best solution is to fix the personality function.
+      } else {
+        // Pull the pointer for the reference type off.
+        const llvm::Type *PtrTy =
+          cast<llvm::PointerType>(LLVMCatchTy)->getElementType();
+
+        // Create the temporary and write the adjusted pointer into it.
+        llvm::Value *ExnPtrTmp = CGF.CreateTempAlloca(PtrTy, "exn.byref.tmp");
+        llvm::Value *Casted = CGF.Builder.CreateBitCast(AdjustedExn, PtrTy);
+        CGF.Builder.CreateStore(Casted, ExnPtrTmp);
+
+        // Bind the reference to the temporary.
+        AdjustedExn = ExnPtrTmp;
+      }
+    }
+
     llvm::Value *ExnCast =
       CGF.Builder.CreateBitCast(AdjustedExn, LLVMCatchTy, "exn.byref");
     CGF.Builder.CreateStore(ExnCast, ParamAddr);
@@ -1113,8 +1132,11 @@ static void InitCatchParam(CodeGenFunction &CGF,
       CGF.StoreComplexToAddr(CGF.LoadComplexFromAddr(Cast, /*volatile*/ false),
                              ParamAddr, /*volatile*/ false);
     } else {
+      unsigned Alignment =
+        CGF.getContext().getDeclAlign(&CatchParam).getQuantity();
       llvm::Value *ExnLoad = CGF.Builder.CreateLoad(Cast, "exn.scalar");
-      CGF.EmitStoreOfScalar(ExnLoad, ParamAddr, /*volatile*/ false, CatchType);
+      CGF.EmitStoreOfScalar(ExnLoad, ParamAddr, /*volatile*/ false, Alignment,
+                            CatchType);
     }
     return;
   }
@@ -1203,7 +1225,7 @@ static void BeginCatch(CodeGenFunction &CGF,
 }
 
 namespace {
-  struct CallRethrow : EHScopeStack::LazyCleanup {
+  struct CallRethrow : EHScopeStack::Cleanup {
     void Emit(CodeGenFunction &CGF, bool IsForEH) {
       CGF.EmitCallOrInvoke(getReThrowFn(CGF), 0, 0);
     }
@@ -1253,7 +1275,7 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
     // _cxa_rethrow.  This needs to happen before __cxa_end_catch is
     // called, and so it is pushed after BeginCatch.
     if (ImplicitRethrow)
-      EHStack.pushLazyCleanup<CallRethrow>(NormalCleanup);
+      EHStack.pushCleanup<CallRethrow>(NormalCleanup);
 
     // Perform the body of the catch.
     EmitStmt(C->getHandlerBlock());
@@ -1269,6 +1291,97 @@ void CodeGenFunction::ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock) {
   EmitBlock(ContBB);
 }
 
+namespace {
+  struct CallEndCatchForFinally : EHScopeStack::Cleanup {
+    llvm::Value *ForEHVar;
+    llvm::Value *EndCatchFn;
+    CallEndCatchForFinally(llvm::Value *ForEHVar, llvm::Value *EndCatchFn)
+      : ForEHVar(ForEHVar), EndCatchFn(EndCatchFn) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      llvm::BasicBlock *EndCatchBB = CGF.createBasicBlock("finally.endcatch");
+      llvm::BasicBlock *CleanupContBB =
+        CGF.createBasicBlock("finally.cleanup.cont");
+
+      llvm::Value *ShouldEndCatch =
+        CGF.Builder.CreateLoad(ForEHVar, "finally.endcatch");
+      CGF.Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB);
+      CGF.EmitBlock(EndCatchBB);
+      CGF.EmitCallOrInvoke(EndCatchFn, 0, 0); // catch-all, so might throw
+      CGF.EmitBlock(CleanupContBB);
+    }
+  };
+
+  struct PerformFinally : EHScopeStack::Cleanup {
+    const Stmt *Body;
+    llvm::Value *ForEHVar;
+    llvm::Value *EndCatchFn;
+    llvm::Value *RethrowFn;
+    llvm::Value *SavedExnVar;
+
+    PerformFinally(const Stmt *Body, llvm::Value *ForEHVar,
+                   llvm::Value *EndCatchFn,
+                   llvm::Value *RethrowFn, llvm::Value *SavedExnVar)
+      : Body(Body), ForEHVar(ForEHVar), EndCatchFn(EndCatchFn),
+        RethrowFn(RethrowFn), SavedExnVar(SavedExnVar) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      // Enter a cleanup to call the end-catch function if one was provided.
+      if (EndCatchFn)
+        CGF.EHStack.pushCleanup<CallEndCatchForFinally>(NormalAndEHCleanup,
+                                                        ForEHVar, EndCatchFn);
+
+      // Save the current cleanup destination in case there are
+      // cleanups in the finally block.
+      llvm::Value *SavedCleanupDest =
+        CGF.Builder.CreateLoad(CGF.getNormalCleanupDestSlot(),
+                               "cleanup.dest.saved");
+
+      // Emit the finally block.
+      CGF.EmitStmt(Body);
+
+      // If the end of the finally is reachable, check whether this was
+      // for EH.  If so, rethrow.
+      if (CGF.HaveInsertPoint()) {
+        llvm::BasicBlock *RethrowBB = CGF.createBasicBlock("finally.rethrow");
+        llvm::BasicBlock *ContBB = CGF.createBasicBlock("finally.cont");
+
+        llvm::Value *ShouldRethrow =
+          CGF.Builder.CreateLoad(ForEHVar, "finally.shouldthrow");
+        CGF.Builder.CreateCondBr(ShouldRethrow, RethrowBB, ContBB);
+
+        CGF.EmitBlock(RethrowBB);
+        if (SavedExnVar) {
+          llvm::Value *Args[] = { CGF.Builder.CreateLoad(SavedExnVar) };
+          CGF.EmitCallOrInvoke(RethrowFn, Args, Args+1);
+        } else {
+          CGF.EmitCallOrInvoke(RethrowFn, 0, 0);
+        }
+        CGF.Builder.CreateUnreachable();
+
+        CGF.EmitBlock(ContBB);
+
+        // Restore the cleanup destination.
+        CGF.Builder.CreateStore(SavedCleanupDest,
+                                CGF.getNormalCleanupDestSlot());
+      }
+
+      // Leave the end-catch cleanup.  As an optimization, pretend that
+      // the fallthrough path was inaccessible; we've dynamically proven
+      // that we're not in the EH case along that path.
+      if (EndCatchFn) {
+        CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
+        CGF.PopCleanupBlock();
+        CGF.Builder.restoreIP(SavedIP);
+      }
+    
+      // Now make sure we actually have an insertion point or the
+      // cleanup gods will hate us.
+      CGF.EnsureInsertPoint();
+    }
+  };
+}
+
 /// Enters a finally block for an implementation using zero-cost
 /// exceptions.  This is mostly general, but hard-codes some
 /// language/ABI-specific behavior in the catch-all sections.
@@ -1320,62 +1433,9 @@ CodeGenFunction::EnterFinallyBlock(const Stmt *Body,
   InitTempAlloca(ForEHVar, llvm::ConstantInt::getFalse(getLLVMContext()));
 
   // Enter a normal cleanup which will perform the @finally block.
-  {
-    CodeGenFunction::CleanupBlock Cleanup(*this, NormalCleanup);
-
-    // Enter a cleanup to call the end-catch function if one was provided.
-    if (EndCatchFn) {
-      CodeGenFunction::CleanupBlock FinallyExitCleanup(CGF, NormalAndEHCleanup);
-
-      llvm::BasicBlock *EndCatchBB = createBasicBlock("finally.endcatch");
-      llvm::BasicBlock *CleanupContBB = createBasicBlock("finally.cleanup.cont");
-
-      llvm::Value *ShouldEndCatch =
-        Builder.CreateLoad(ForEHVar, "finally.endcatch");
-      Builder.CreateCondBr(ShouldEndCatch, EndCatchBB, CleanupContBB);
-      EmitBlock(EndCatchBB);
-      EmitCallOrInvoke(EndCatchFn, 0, 0); // catch-all, so might throw
-      EmitBlock(CleanupContBB);
-    }
-
-    // Emit the finally block.
-    EmitStmt(Body);
-
-    // If the end of the finally is reachable, check whether this was
-    // for EH.  If so, rethrow.
-    if (HaveInsertPoint()) {
-      llvm::BasicBlock *RethrowBB = createBasicBlock("finally.rethrow");
-      llvm::BasicBlock *ContBB = createBasicBlock("finally.cont");
-
-      llvm::Value *ShouldRethrow =
-        Builder.CreateLoad(ForEHVar, "finally.shouldthrow");
-      Builder.CreateCondBr(ShouldRethrow, RethrowBB, ContBB);
-
-      EmitBlock(RethrowBB);
-      if (SavedExnVar) {
-        llvm::Value *Args[] = { Builder.CreateLoad(SavedExnVar) };
-        EmitCallOrInvoke(RethrowFn, Args, Args+1);
-      } else {
-        EmitCallOrInvoke(RethrowFn, 0, 0);
-      }
-      Builder.CreateUnreachable();
-
-      EmitBlock(ContBB);
-    }
-
-    // Leave the end-catch cleanup.  As an optimization, pretend that
-    // the fallthrough path was inaccessible; we've dynamically proven
-    // that we're not in the EH case along that path.
-    if (EndCatchFn) {
-      CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
-      PopCleanupBlock();
-      Builder.restoreIP(SavedIP);
-    }
-    
-    // Now make sure we actually have an insertion point or the
-    // cleanup gods will hate us.
-    EnsureInsertPoint();
-  }
+  EHStack.pushCleanup<PerformFinally>(NormalCleanup, Body,
+                                      ForEHVar, EndCatchFn,
+                                      RethrowFn, SavedExnVar);
 
   // Enter a catch-all scope.
   llvm::BasicBlock *CatchAllBB = createBasicBlock("finally.catchall");
@@ -1437,10 +1497,12 @@ llvm::BasicBlock *CodeGenFunction::getTerminateLandingPad() {
   llvm::CallInst *Exn =
     Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_exception), "exn");
   Exn->setDoesNotThrow();
+
+  const EHPersonality &Personality = EHPersonality::get(CGM.getLangOptions());
   
   // Tell the backend what the exception table should be:
   // nothing but a catch-all.
-  llvm::Value *Args[3] = { Exn, getPersonalityFn(*this),
+  llvm::Value *Args[3] = { Exn, getPersonalityFn(*this, Personality),
                            getCatchAllValue(*this) };
   Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::eh_selector),
                      Args, Args+3, "eh.selector")
@@ -1478,69 +1540,35 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() {
   return TerminateHandler;
 }
 
-CodeGenFunction::CleanupBlock::CleanupBlock(CodeGenFunction &CGF,
-                                            CleanupKind Kind)
-  : CGF(CGF), SavedIP(CGF.Builder.saveIP()), NormalCleanupExitBB(0) {
-  llvm::BasicBlock *EntryBB = CGF.createBasicBlock("cleanup");
-  CGF.Builder.SetInsertPoint(EntryBB);
-
-  switch (Kind) {
-  case NormalAndEHCleanup:
-    NormalCleanupEntryBB = EHCleanupEntryBB = EntryBB;
-    break;
-
-  case NormalCleanup:
-    NormalCleanupEntryBB = EntryBB;
-    EHCleanupEntryBB = 0;
-    break;        
-
-  case EHCleanup:
-    NormalCleanupEntryBB = 0;
-    EHCleanupEntryBB = EntryBB;
-    CGF.EHStack.pushTerminate();
-    break;
-  }
-}
+CodeGenFunction::UnwindDest CodeGenFunction::getRethrowDest() {
+  if (RethrowBlock.isValid()) return RethrowBlock;
 
-void CodeGenFunction::CleanupBlock::beginEHCleanup() {
-  assert(EHCleanupEntryBB == 0 && "already started an EH cleanup");
-  NormalCleanupExitBB = CGF.Builder.GetInsertBlock();
-  assert(NormalCleanupExitBB && "end of normal cleanup is unreachable");
-      
-  EHCleanupEntryBB = CGF.createBasicBlock("eh.cleanup");
-  CGF.Builder.SetInsertPoint(EHCleanupEntryBB);
-  CGF.EHStack.pushTerminate();
-}
+  CGBuilderTy::InsertPoint SavedIP = Builder.saveIP();
 
-CodeGenFunction::CleanupBlock::~CleanupBlock() {
-  llvm::BasicBlock *EHCleanupExitBB = 0;
+  // We emit a jump to a notional label at the outermost unwind state.
+  llvm::BasicBlock *Unwind = createBasicBlock("eh.resume");
+  Builder.SetInsertPoint(Unwind);
 
-  // If we're currently writing the EH cleanup...
-  if (EHCleanupEntryBB) {
-    // Set the EH cleanup exit block.
-    EHCleanupExitBB = CGF.Builder.GetInsertBlock();
-    assert(EHCleanupExitBB && "end of EH cleanup is unreachable");
+  const EHPersonality &Personality = EHPersonality::get(CGM.getLangOptions());
 
-    // If we're actually writing both at once, set the normal exit, too.
-    if (EHCleanupEntryBB == NormalCleanupEntryBB)
-      NormalCleanupExitBB = EHCleanupExitBB;
+  // This can always be a call because we necessarily didn't find
+  // anything on the EH stack which needs our help.
+  llvm::Constant *RethrowFn;
+  if (const char *RethrowName = Personality.getCatchallRethrowFnName())
+    RethrowFn = getCatchallRethrowFn(*this, RethrowName);
+  else
+    RethrowFn = getUnwindResumeOrRethrowFn();
 
-    // Otherwise, we must have pushed a terminate handler.
-    else
-      CGF.EHStack.popTerminate();
+  Builder.CreateCall(RethrowFn, Builder.CreateLoad(getExceptionSlot()))
+    ->setDoesNotReturn();
+  Builder.CreateUnreachable();
 
-  // Otherwise, just set the normal cleanup exit block.
-  } else {
-    NormalCleanupExitBB = CGF.Builder.GetInsertBlock();
-    assert(NormalCleanupExitBB && "end of normal cleanup is unreachable");
-  }
-  
-  CGF.EHStack.pushCleanup(NormalCleanupEntryBB, NormalCleanupExitBB,
-                          EHCleanupEntryBB, EHCleanupExitBB);
+  Builder.restoreIP(SavedIP);
 
-  CGF.Builder.restoreIP(SavedIP);
+  RethrowBlock = UnwindDest(Unwind, EHStack.stable_end(), 0);
+  return RethrowBlock;
 }
 
-EHScopeStack::LazyCleanup::~LazyCleanup() {
-  llvm_unreachable("LazyCleanup is indestructable");
+EHScopeStack::Cleanup::~Cleanup() {
+  llvm_unreachable("Cleanup is indestructable");
 }
diff --git a/lib/CodeGen/CGException.h b/lib/CodeGen/CGException.h
index 80739cd8d73e..f1294746a42d 100644
--- a/lib/CodeGen/CGException.h
+++ b/lib/CodeGen/CGException.h
@@ -27,17 +27,43 @@ namespace llvm {
 namespace clang {
 namespace CodeGen {
 
+/// The exceptions personality for a function.  When 
+class EHPersonality {
+  const char *PersonalityFn;
+
+  // If this is non-null, this personality requires a non-standard
+  // function for rethrowing an exception after a catchall cleanup.
+  // This function must have prototype void(void*).
+  const char *CatchallRethrowFn;
+
+  EHPersonality(const char *PersonalityFn,
+                const char *CatchallRethrowFn = 0)
+    : PersonalityFn(PersonalityFn),
+      CatchallRethrowFn(CatchallRethrowFn) {}
+
+public:
+  static const EHPersonality &get(const LangOptions &Lang);
+  static const EHPersonality GNU_C;
+  static const EHPersonality GNU_ObjC;
+  static const EHPersonality NeXT_ObjC;
+  static const EHPersonality GNU_CPlusPlus;
+  static const EHPersonality GNU_CPlusPlus_SJLJ;
+
+  const char *getPersonalityFnName() const { return PersonalityFn; }
+  const char *getCatchallRethrowFnName() const { return CatchallRethrowFn; }
+};
+
 /// A protected scope for zero-cost EH handling.
 class EHScope {
   llvm::BasicBlock *CachedLandingPad;
 
-  unsigned K : 3;
+  unsigned K : 2;
 
 protected:
-  enum { BitsRemaining = 29 };
+  enum { BitsRemaining = 30 };
 
 public:
-  enum Kind { Cleanup, LazyCleanup, Catch, Terminate, Filter };
+  enum Kind { Cleanup, Catch, Terminate, Filter };
 
   EHScope(Kind K) : CachedLandingPad(0), K(K) {}
 
@@ -74,15 +100,13 @@ public:
     /// The catch handler for this type.
     llvm::BasicBlock *Block;
 
-    static Handler make(llvm::Value *Type, llvm::BasicBlock *Block) {
-      Handler Temp;
-      Temp.Type = Type;
-      Temp.Block = Block;
-      return Temp;
-    }
+    /// The unwind destination index for this handler.
+    unsigned Index;
   };
 
 private:
+  friend class EHScopeStack;
+
   Handler *getHandlers() {
     return reinterpret_cast<Handler*>(this+1);
   }
@@ -110,7 +134,8 @@ public:
 
   void setHandler(unsigned I, llvm::Value *Type, llvm::BasicBlock *Block) {
     assert(I < getNumHandlers());
-    getHandlers()[I] = Handler::make(Type, Block);
+    getHandlers()[I].Type = Type;
+    getHandlers()[I].Block = Block;
   }
 
   const Handler &getHandler(unsigned I) const {
@@ -128,21 +153,27 @@ public:
 };
 
 /// A cleanup scope which generates the cleanup blocks lazily.
-class EHLazyCleanupScope : public EHScope {
+class EHCleanupScope : public EHScope {
   /// Whether this cleanup needs to be run along normal edges.
   bool IsNormalCleanup : 1;
 
   /// Whether this cleanup needs to be run along exception edges.
   bool IsEHCleanup : 1;
 
-  /// The amount of extra storage needed by the LazyCleanup.
+  /// Whether this cleanup was activated before all normal uses.
+  bool ActivatedBeforeNormalUse : 1;
+
+  /// Whether this cleanup was activated before all EH uses.
+  bool ActivatedBeforeEHUse : 1;
+
+  /// The amount of extra storage needed by the Cleanup.
   /// Always a multiple of the scope-stack alignment.
   unsigned CleanupSize : 12;
 
   /// The number of fixups required by enclosing scopes (not including
   /// this one).  If this is the top cleanup scope, all the fixups
   /// from this index onwards belong to this scope.
-  unsigned FixupDepth : BitsRemaining - 14;
+  unsigned FixupDepth : BitsRemaining - 16;
 
   /// The nearest normal cleanup scope enclosing this one.
   EHScopeStack::stable_iterator EnclosingNormal;
@@ -158,27 +189,78 @@ class EHLazyCleanupScope : public EHScope {
   /// created if needed before the cleanup is popped.
   llvm::BasicBlock *EHBlock;
 
+  /// An optional i1 variable indicating whether this cleanup has been
+  /// activated yet.  This has one of three states:
+  ///   - it is null if the cleanup is inactive
+  ///   - it is activeSentinel() if the cleanup is active and was not
+  ///     required before activation
+  ///   - it points to a valid variable
+  llvm::AllocaInst *ActiveVar;
+
+  /// Extra information required for cleanups that have resolved
+  /// branches through them.  This has to be allocated on the side
+  /// because everything on the cleanup stack has be trivially
+  /// movable.
+  struct ExtInfo {
+    /// The destinations of normal branch-afters and branch-throughs.
+    llvm::SmallPtrSet<llvm::BasicBlock*, 4> Branches;
+
+    /// Normal branch-afters.
+    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
+      BranchAfters;
+
+    /// The destinations of EH branch-afters and branch-throughs.
+    /// TODO: optimize for the extremely common case of a single
+    /// branch-through.
+    llvm::SmallPtrSet<llvm::BasicBlock*, 4> EHBranches;
+
+    /// EH branch-afters.
+    llvm::SmallVector<std::pair<llvm::BasicBlock*,llvm::ConstantInt*>, 4>
+    EHBranchAfters;
+  };
+  mutable struct ExtInfo *ExtInfo;
+
+  struct ExtInfo &getExtInfo() {
+    if (!ExtInfo) ExtInfo = new struct ExtInfo();
+    return *ExtInfo;
+  }
+
+  const struct ExtInfo &getExtInfo() const {
+    if (!ExtInfo) ExtInfo = new struct ExtInfo();
+    return *ExtInfo;
+  }
+
 public:
   /// Gets the size required for a lazy cleanup scope with the given
   /// cleanup-data requirements.
   static size_t getSizeForCleanupSize(size_t Size) {
-    return sizeof(EHLazyCleanupScope) + Size;
+    return sizeof(EHCleanupScope) + Size;
   }
 
   size_t getAllocatedSize() const {
-    return sizeof(EHLazyCleanupScope) + CleanupSize;
+    return sizeof(EHCleanupScope) + CleanupSize;
   }
 
-  EHLazyCleanupScope(bool IsNormal, bool IsEH, unsigned CleanupSize,
-                     unsigned FixupDepth,
-                     EHScopeStack::stable_iterator EnclosingNormal,
-                     EHScopeStack::stable_iterator EnclosingEH)
-    : EHScope(EHScope::LazyCleanup),
+  EHCleanupScope(bool IsNormal, bool IsEH, bool IsActive,
+                 unsigned CleanupSize, unsigned FixupDepth,
+                 EHScopeStack::stable_iterator EnclosingNormal,
+                 EHScopeStack::stable_iterator EnclosingEH)
+    : EHScope(EHScope::Cleanup),
       IsNormalCleanup(IsNormal), IsEHCleanup(IsEH),
+      ActivatedBeforeNormalUse(IsActive),
+      ActivatedBeforeEHUse(IsActive),
       CleanupSize(CleanupSize), FixupDepth(FixupDepth),
       EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
-      NormalBlock(0), EHBlock(0)
-  {}
+      NormalBlock(0), EHBlock(0),
+      ActiveVar(IsActive ? activeSentinel() : 0),
+      ExtInfo(0)
+  {
+    assert(this->CleanupSize == CleanupSize && "cleanup size overflow");
+  }
+
+  ~EHCleanupScope() {
+    delete ExtInfo;
+  }
 
   bool isNormalCleanup() const { return IsNormalCleanup; }
   llvm::BasicBlock *getNormalBlock() const { return NormalBlock; }
@@ -188,6 +270,20 @@ public:
   llvm::BasicBlock *getEHBlock() const { return EHBlock; }
   void setEHBlock(llvm::BasicBlock *BB) { EHBlock = BB; }
 
+  static llvm::AllocaInst *activeSentinel() {
+    return reinterpret_cast<llvm::AllocaInst*>(1);
+  }
+
+  bool isActive() const { return ActiveVar != 0; }
+  llvm::AllocaInst *getActiveVar() const { return ActiveVar; }
+  void setActiveVar(llvm::AllocaInst *Var) { ActiveVar = Var; }
+
+  bool wasActivatedBeforeNormalUse() const { return ActivatedBeforeNormalUse; }
+  void setActivatedBeforeNormalUse(bool B) { ActivatedBeforeNormalUse = B; }
+
+  bool wasActivatedBeforeEHUse() const { return ActivatedBeforeEHUse; }
+  void setActivatedBeforeEHUse(bool B) { ActivatedBeforeEHUse = B; }
+
   unsigned getFixupDepth() const { return FixupDepth; }
   EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
     return EnclosingNormal;
@@ -199,67 +295,108 @@ public:
   size_t getCleanupSize() const { return CleanupSize; }
   void *getCleanupBuffer() { return this + 1; }
 
-  EHScopeStack::LazyCleanup *getCleanup() {
-    return reinterpret_cast<EHScopeStack::LazyCleanup*>(getCleanupBuffer());
+  EHScopeStack::Cleanup *getCleanup() {
+    return reinterpret_cast<EHScopeStack::Cleanup*>(getCleanupBuffer());
   }
 
-  static bool classof(const EHScope *Scope) {
-    return (Scope->getKind() == LazyCleanup);
+  /// True if this cleanup scope has any branch-afters or branch-throughs.
+  bool hasBranches() const { return ExtInfo && !ExtInfo->Branches.empty(); }
+
+  /// Add a branch-after to this cleanup scope.  A branch-after is a
+  /// branch from a point protected by this (normal) cleanup to a
+  /// point in the normal cleanup scope immediately containing it.
+  /// For example,
+  ///   for (;;) { A a; break; }
+  /// contains a branch-after.
+  ///
+  /// Branch-afters each have their own destination out of the
+  /// cleanup, guaranteed distinct from anything else threaded through
+  /// it.  Therefore branch-afters usually force a switch after the
+  /// cleanup.
+  void addBranchAfter(llvm::ConstantInt *Index,
+                      llvm::BasicBlock *Block) {
+    struct ExtInfo &ExtInfo = getExtInfo();
+    if (ExtInfo.Branches.insert(Block))
+      ExtInfo.BranchAfters.push_back(std::make_pair(Block, Index));
   }
-};
 
-/// A scope which needs to execute some code if we try to unwind ---
-/// either normally, via the EH mechanism, or both --- through it.
-class EHCleanupScope : public EHScope {
-  /// The number of fixups required by enclosing scopes (not including
-  /// this one).  If this is the top cleanup scope, all the fixups
-  /// from this index onwards belong to this scope.
-  unsigned FixupDepth : BitsRemaining;
+  /// Return the number of unique branch-afters on this scope.
+  unsigned getNumBranchAfters() const {
+    return ExtInfo ? ExtInfo->BranchAfters.size() : 0;
+  }
 
-  /// The nearest normal cleanup scope enclosing this one.
-  EHScopeStack::stable_iterator EnclosingNormal;
+  llvm::BasicBlock *getBranchAfterBlock(unsigned I) const {
+    assert(I < getNumBranchAfters());
+    return ExtInfo->BranchAfters[I].first;
+  }
 
-  /// The nearest EH cleanup scope enclosing this one.
-  EHScopeStack::stable_iterator EnclosingEH;
+  llvm::ConstantInt *getBranchAfterIndex(unsigned I) const {
+    assert(I < getNumBranchAfters());
+    return ExtInfo->BranchAfters[I].second;
+  }
 
-  llvm::BasicBlock *NormalEntry;
-  llvm::BasicBlock *NormalExit;
-  llvm::BasicBlock *EHEntry;
-  llvm::BasicBlock *EHExit;
+  /// Add a branch-through to this cleanup scope.  A branch-through is
+  /// a branch from a scope protected by this (normal) cleanup to an
+  /// enclosing scope other than the immediately-enclosing normal
+  /// cleanup scope.
+  ///
+  /// In the following example, the branch through B's scope is a
+  /// branch-through, while the branch through A's scope is a
+  /// branch-after:
+  ///   for (;;) { A a; B b; break; }
+  ///
+  /// All branch-throughs have a common destination out of the
+  /// cleanup, one possibly shared with the fall-through.  Therefore
+  /// branch-throughs usually don't force a switch after the cleanup.
+  ///
+  /// \return true if the branch-through was new to this scope
+  bool addBranchThrough(llvm::BasicBlock *Block) {
+    return getExtInfo().Branches.insert(Block);
+  }
 
-public:
-  static size_t getSize() { return sizeof(EHCleanupScope); }
+  /// Determines if this cleanup scope has any branch throughs.
+  bool hasBranchThroughs() const {
+    if (!ExtInfo) return false;
+    return (ExtInfo->BranchAfters.size() != ExtInfo->Branches.size());
+  }
 
-  EHCleanupScope(unsigned FixupDepth,
-                 EHScopeStack::stable_iterator EnclosingNormal,
-                 EHScopeStack::stable_iterator EnclosingEH,
-                 llvm::BasicBlock *NormalEntry, llvm::BasicBlock *NormalExit,
-                 llvm::BasicBlock *EHEntry, llvm::BasicBlock *EHExit)
-    : EHScope(Cleanup), FixupDepth(FixupDepth),
-      EnclosingNormal(EnclosingNormal), EnclosingEH(EnclosingEH),
-      NormalEntry(NormalEntry), NormalExit(NormalExit),
-      EHEntry(EHEntry), EHExit(EHExit) {
-    assert((NormalEntry != 0) == (NormalExit != 0));
-    assert((EHEntry != 0) == (EHExit != 0));
+  // Same stuff, only for EH branches instead of normal branches.
+  // It's quite possible that we could find a better representation
+  // for this.
+
+  bool hasEHBranches() const { return ExtInfo && !ExtInfo->EHBranches.empty(); }
+  void addEHBranchAfter(llvm::ConstantInt *Index,
+                        llvm::BasicBlock *Block) {
+    struct ExtInfo &ExtInfo = getExtInfo();
+    if (ExtInfo.EHBranches.insert(Block))
+      ExtInfo.EHBranchAfters.push_back(std::make_pair(Block, Index));
   }
 
-  bool isNormalCleanup() const { return NormalEntry != 0; }
-  bool isEHCleanup() const { return EHEntry != 0; }
+  unsigned getNumEHBranchAfters() const {
+    return ExtInfo ? ExtInfo->EHBranchAfters.size() : 0;
+  }
 
-  llvm::BasicBlock *getNormalEntry() const { return NormalEntry; }
-  llvm::BasicBlock *getNormalExit() const { return NormalExit; }
-  llvm::BasicBlock *getEHEntry() const { return EHEntry; }
-  llvm::BasicBlock *getEHExit() const { return EHExit; }
-  unsigned getFixupDepth() const { return FixupDepth; }
-  EHScopeStack::stable_iterator getEnclosingNormalCleanup() const {
-    return EnclosingNormal;
+  llvm::BasicBlock *getEHBranchAfterBlock(unsigned I) const {
+    assert(I < getNumEHBranchAfters());
+    return ExtInfo->EHBranchAfters[I].first;
   }
-  EHScopeStack::stable_iterator getEnclosingEHCleanup() const {
-    return EnclosingEH;
+
+  llvm::ConstantInt *getEHBranchAfterIndex(unsigned I) const {
+    assert(I < getNumEHBranchAfters());
+    return ExtInfo->EHBranchAfters[I].second;
+  }
+
+  bool addEHBranchThrough(llvm::BasicBlock *Block) {
+    return getExtInfo().EHBranches.insert(Block);
+  }
+
+  bool hasEHBranchThroughs() const {
+    if (!ExtInfo) return false;
+    return (ExtInfo->EHBranchAfters.size() != ExtInfo->EHBranches.size());
   }
 
   static bool classof(const EHScope *Scope) {
-    return Scope->getKind() == Cleanup;
+    return (Scope->getKind() == Cleanup);
   }
 };
 
@@ -310,10 +447,13 @@ public:
 /// An exceptions scope which calls std::terminate if any exception
 /// reaches it.
 class EHTerminateScope : public EHScope {
+  unsigned DestIndex : BitsRemaining;
 public:
-  EHTerminateScope() : EHScope(Terminate) {}
+  EHTerminateScope(unsigned Index) : EHScope(Terminate), DestIndex(Index) {}
   static size_t getSize() { return sizeof(EHTerminateScope); }
 
+  unsigned getDestIndex() const { return DestIndex; }
+
   static bool classof(const EHScope *Scope) {
     return Scope->getKind() == Terminate;
   }
@@ -348,13 +488,9 @@ public:
           static_cast<const EHFilterScope*>(get())->getNumFilters());
       break;
 
-    case EHScope::LazyCleanup:
-      Ptr += static_cast<const EHLazyCleanupScope*>(get())
-        ->getAllocatedSize();
-      break;
-
     case EHScope::Cleanup:
-      Ptr += EHCleanupScope::getSize();
+      Ptr += static_cast<const EHCleanupScope*>(get())
+        ->getAllocatedSize();
       break;
 
     case EHScope::Terminate:
@@ -377,6 +513,9 @@ public:
     return copy;
   }
 
+  bool encloses(iterator other) const { return Ptr >= other.Ptr; }
+  bool strictlyEncloses(iterator other) const { return Ptr > other.Ptr; }
+
   bool operator==(iterator other) const { return Ptr == other.Ptr; }
   bool operator!=(iterator other) const { return Ptr != other.Ptr; }
 };
@@ -396,6 +535,8 @@ inline void EHScopeStack::popCatch() {
   StartOfData += EHCatchScope::getSizeForNumHandlers(
                           cast<EHCatchScope>(*begin()).getNumHandlers());
 
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
   assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
   CatchDepth--;
 }
@@ -406,6 +547,8 @@ inline void EHScopeStack::popTerminate() {
   assert(isa<EHTerminateScope>(*begin()));
   StartOfData += EHTerminateScope::getSize();
 
+  if (empty()) NextEHDestIndex = FirstEHDestIndex;
+
   assert(CatchDepth > 0 && "mismatched catch/terminate push/pop");
   CatchDepth--;
 }
@@ -422,6 +565,28 @@ EHScopeStack::stabilize(iterator ir) const {
   return stable_iterator(EndOfBuffer - ir.Ptr);
 }
 
+inline EHScopeStack::stable_iterator
+EHScopeStack::getInnermostActiveNormalCleanup() const {
+  for (EHScopeStack::stable_iterator
+         I = getInnermostNormalCleanup(), E = stable_end(); I != E; ) {
+    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
+    if (S.isActive()) return I;
+    I = S.getEnclosingNormalCleanup();
+  }
+  return stable_end();
+}
+
+inline EHScopeStack::stable_iterator
+EHScopeStack::getInnermostActiveEHCleanup() const {
+  for (EHScopeStack::stable_iterator
+         I = getInnermostEHCleanup(), E = stable_end(); I != E; ) {
+    EHCleanupScope &S = cast<EHCleanupScope>(*find(I));
+    if (S.isActive()) return I;
+    I = S.getEnclosingEHCleanup();
+  }
+  return stable_end();
+}
+
 }
 }
 
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 43bab9fece6e..3750ab80c3fc 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -14,6 +14,7 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "CGCall.h"
+#include "CGCXXABI.h"
 #include "CGRecordLayout.h"
 #include "CGObjCRuntime.h"
 #include "clang/AST/ASTContext.h"
@@ -65,22 +66,12 @@ llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
 /// expression and compare the result against zero, returning an Int1Ty value.
 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
-  QualType BoolTy = getContext().BoolTy;
-  if (E->getType()->isMemberFunctionPointerType()) {
-    LValue LV = EmitAggExprToLValue(E);
-
-    // Get the pointer.
-    llvm::Value *FuncPtr = Builder.CreateStructGEP(LV.getAddress(), 0,
-                                                   "src.ptr");
-    FuncPtr = Builder.CreateLoad(FuncPtr);
-
-    llvm::Value *IsNotNull = 
-      Builder.CreateICmpNE(FuncPtr,
-                            llvm::Constant::getNullValue(FuncPtr->getType()),
-                            "tobool");
-
-    return IsNotNull;
+  if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
+    llvm::Value *MemPtr = EmitScalarExpr(E);
+    return CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, MemPtr, MPT);
   }
+
+  QualType BoolTy = getContext().BoolTy;
   if (!E->getType()->isAnyComplexType())
     return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy);
 
@@ -130,7 +121,7 @@ void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
     EmitAggExpr(E, Location, IsLocationVolatile, /*Ignore*/ false, IsInit);
   else {
     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
-    LValue LV = LValue::MakeAddr(Location, MakeQualifiers(E->getType()));
+    LValue LV = MakeAddrLValue(Location, E->getType());
     EmitStoreThroughLValue(RV, LV, E->getType());
   }
 }
@@ -142,17 +133,14 @@ struct SubobjectAdjustment {
   
   union {
     struct {
-      const CXXBaseSpecifierArray *BasePath;
+      const CastExpr *BasePath;
       const CXXRecordDecl *DerivedClass;
     } DerivedToBase;
     
-    struct {
-      FieldDecl *Field;
-      unsigned CVRQualifiers;
-    } Field;
+    FieldDecl *Field;
   };
   
-  SubobjectAdjustment(const CXXBaseSpecifierArray *BasePath, 
+  SubobjectAdjustment(const CastExpr *BasePath, 
                       const CXXRecordDecl *DerivedClass)
     : Kind(DerivedToBaseAdjustment) 
   {
@@ -160,11 +148,10 @@ struct SubobjectAdjustment {
     DerivedToBase.DerivedClass = DerivedClass;
   }
   
-  SubobjectAdjustment(FieldDecl *Field, unsigned CVRQualifiers)
-    : Kind(FieldAdjustment) 
+  SubobjectAdjustment(FieldDecl *Field)
+    : Kind(FieldAdjustment)
   { 
-    this->Field.Field = Field;
-    this->Field.CVRQualifiers = CVRQualifiers;
+    this->Field = Field;
   }
 };
 
@@ -174,7 +161,7 @@ CreateReferenceTemporary(CodeGenFunction& CGF, QualType Type,
   if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
     if (VD->hasGlobalStorage()) {
       llvm::SmallString<256> Name;
-      CGF.CGM.getMangleContext().mangleReferenceTemporary(VD, Name);
+      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Name);
       
       const llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
   
@@ -230,18 +217,17 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
       } 
 
       if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
-        if ((CE->getCastKind() == CastExpr::CK_DerivedToBase ||
-             CE->getCastKind() == CastExpr::CK_UncheckedDerivedToBase) &&
+        if ((CE->getCastKind() == CK_DerivedToBase ||
+             CE->getCastKind() == CK_UncheckedDerivedToBase) &&
             E->getType()->isRecordType()) {
           E = CE->getSubExpr();
           CXXRecordDecl *Derived 
             = cast<CXXRecordDecl>(E->getType()->getAs<RecordType>()->getDecl());
-          Adjustments.push_back(SubobjectAdjustment(&CE->getBasePath(), 
-                                                    Derived));
+          Adjustments.push_back(SubobjectAdjustment(CE, Derived));
           continue;
         }
 
-        if (CE->getCastKind() == CastExpr::CK_NoOp) {
+        if (CE->getCastKind() == CK_NoOp) {
           E = CE->getSubExpr();
           continue;
         }
@@ -250,8 +236,7 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
             ME->getBase()->getType()->isRecordType()) {
           if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
             E = ME->getBase();
-            Adjustments.push_back(SubobjectAdjustment(Field,
-                                              E->getType().getCVRQualifiers()));
+            Adjustments.push_back(SubobjectAdjustment(Field));
             continue;
           }
         }
@@ -291,14 +276,14 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
           Object = 
               CGF.GetAddressOfBaseClass(Object, 
                                         Adjustment.DerivedToBase.DerivedClass, 
-                                        *Adjustment.DerivedToBase.BasePath, 
+                              Adjustment.DerivedToBase.BasePath->path_begin(),
+                              Adjustment.DerivedToBase.BasePath->path_end(),
                                         /*NullCheckValue=*/false);
           break;
             
         case SubobjectAdjustment::FieldAdjustment: {
-          unsigned CVR = Adjustment.Field.CVRQualifiers;
           LValue LV = 
-            CGF.EmitLValueForField(Object, Adjustment.Field.Field, CVR);
+            CGF.EmitLValueForField(Object, Adjustment.Field, 0);
           if (LV.isSimple()) {
             Object = LV.getAddress();
             break;
@@ -306,11 +291,11 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
           
           // For non-simple lvalues, we actually have to create a copy of
           // the object we're binding to.
-          QualType T = Adjustment.Field.Field->getType().getNonReferenceType()
-                                                        .getUnqualifiedType();
+          QualType T = Adjustment.Field->getType().getNonReferenceType()
+                                                  .getUnqualifiedType();
           Object = CreateReferenceTemporary(CGF, T, InitializedDecl);
-          LValue TempLV = LValue::MakeAddr(Object, 
-                                           Qualifiers::fromCVRMask(CVR));
+          LValue TempLV = CGF.MakeAddrLValue(Object,
+                                             Adjustment.Field->getType());
           CGF.EmitStoreThroughLValue(CGF.EmitLoadOfLValue(LV, T), TempLV, T);
           break;
         }
@@ -330,9 +315,12 @@ EmitExprForReferenceBinding(CodeGenFunction& CGF, const Expr* E,
   ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(), 
                                                 InitializedDecl);
 
+
+  unsigned Alignment =
+    CGF.getContext().getTypeAlignInChars(E->getType()).getQuantity();
   if (RV.isScalar())
     CGF.EmitStoreOfScalar(RV.getScalarVal(), ReferenceTemporary,
-                          /*Volatile=*/false, E->getType());
+                          /*Volatile=*/false, Alignment, E->getType());
   else
     CGF.StoreComplexToAddr(RV.getComplexVal(), ReferenceTemporary,
                            /*Volatile=*/false);
@@ -347,7 +335,6 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
   llvm::Value *Value = EmitExprForReferenceBinding(*this, E, ReferenceTemporary,
                                                    ReferenceTemporaryDtor,
                                                    InitializedDecl);
-
   if (!ReferenceTemporaryDtor)
     return RValue::get(Value);
   
@@ -362,16 +349,8 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr* E,
       return RValue::get(Value);
     }
   }
-  
-  CleanupBlock Cleanup(*this, NormalCleanup);
-  EmitCXXDestructorCall(ReferenceTemporaryDtor, Dtor_Complete, 
-                        /*ForVirtualBase=*/false, ReferenceTemporary);
-            
-  if (Exceptions) {
-    Cleanup.beginEHCleanup();
-    EmitCXXDestructorCall(ReferenceTemporaryDtor, Dtor_Complete, 
-                          /*ForVirtualBase=*/false, ReferenceTemporary);
-  }
+
+  PushDestructorCleanup(ReferenceTemporaryDtor, ReferenceTemporary);
 
   return RValue::get(Value);
 }
@@ -462,9 +441,12 @@ RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
     return RValue::getComplex(std::make_pair(U, U));
   }
   
+  // If this is a use of an undefined aggregate type, the aggregate must have an
+  // identifiable address.  Just because the contents of the value are undefined
+  // doesn't mean that the address can't be taken and compared.
   if (hasAggregateLLVMType(Ty)) {
-    const llvm::Type *LTy = llvm::PointerType::getUnqual(ConvertType(Ty));
-    return RValue::getAggregate(llvm::UndefValue::get(LTy));
+    llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
+    return RValue::getAggregate(DestPtr);
   }
   
   return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
@@ -480,8 +462,7 @@ LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
                                               const char *Name) {
   ErrorUnsupported(E, Name);
   llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
-  return LValue::MakeAddr(llvm::UndefValue::get(Ty),
-                          MakeQualifiers(E->getType()));
+  return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
 }
 
 LValue CodeGenFunction::EmitCheckedLValue(const Expr *E) {
@@ -590,10 +571,12 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
 }
 
 llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
-                                               QualType Ty) {
+                                              unsigned Alignment, QualType Ty) {
   llvm::LoadInst *Load = Builder.CreateLoad(Addr, "tmp");
   if (Volatile)
     Load->setVolatile(true);
+  if (Alignment)
+    Load->setAlignment(Alignment);
 
   // Bool can have different representation in memory than in registers.
   llvm::Value *V = Load;
@@ -605,14 +588,18 @@ llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
 }
 
 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
-                                        bool Volatile, QualType Ty) {
+                                        bool Volatile, unsigned Alignment,
+                                        QualType Ty) {
 
   if (Ty->isBooleanType()) {
     // Bool can have different representation in memory than in registers.
     const llvm::PointerType *DstPtr = cast<llvm::PointerType>(Addr->getType());
     Value = Builder.CreateIntCast(Value, DstPtr->getElementType(), false);
   }
-  Builder.CreateStore(Value, Addr, Volatile);
+
+  llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
+  if (Alignment)
+    Store->setAlignment(Alignment);
 }
 
 /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
@@ -628,18 +615,15 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, QualType ExprType) {
 
   if (LV.isSimple()) {
     llvm::Value *Ptr = LV.getAddress();
-    const llvm::Type *EltTy =
-      cast<llvm::PointerType>(Ptr->getType())->getElementType();
 
-    // Simple scalar l-value.
-    //
-    // FIXME: We shouldn't have to use isSingleValueType here.
-    if (EltTy->isSingleValueType())
-      return RValue::get(EmitLoadOfScalar(Ptr, LV.isVolatileQualified(),
-                                          ExprType));
+    // Functions are l-values that don't require loading.
+    if (ExprType->isFunctionType())
+      return RValue::get(Ptr);
+
+    // Everything needs a load.
+    return RValue::get(EmitLoadOfScalar(Ptr, LV.isVolatileQualified(),
+                                        LV.getAlignment(), ExprType));
 
-    assert(ExprType->isFunctionType() && "Unknown scalar value");
-    return RValue::get(Ptr);
   }
 
   if (LV.isVectorElt()) {
@@ -836,8 +820,10 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
       CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
                                               BytesBetween);
-    } else if (Dst.isGlobalObjCRef())
-      CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst);
+    } else if (Dst.isGlobalObjCRef()) {
+      CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
+                                                Dst.isThreadLocalRef());
+    }
     else
       CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
     return;
@@ -845,7 +831,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
 
   assert(Src.isScalar() && "Can't emit an agg store with this method");
   EmitStoreOfScalar(Src.getScalarVal(), Dst.getAddress(),
-                    Dst.isVolatileQualified(), Ty);
+                    Dst.isVolatileQualified(), Dst.getAlignment(), Ty);
 }
 
 void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
@@ -1045,20 +1031,22 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
     return;
   
   if (isa<ObjCIvarRefExpr>(E)) {
-    LV.SetObjCIvar(LV, true);
+    LV.setObjCIvar(true);
     ObjCIvarRefExpr *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr*>(E));
     LV.setBaseIvarExp(Exp->getBase());
-    LV.SetObjCArray(LV, E->getType()->isArrayType());
+    LV.setObjCArray(E->getType()->isArrayType());
     return;
   }
   
   if (const DeclRefExpr *Exp = dyn_cast<DeclRefExpr>(E)) {
     if (const VarDecl *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
       if ((VD->isBlockVarDecl() && !VD->hasLocalStorage()) ||
-          VD->isFileVarDecl())
-        LV.SetGlobalObjCRef(LV, true);
+          VD->isFileVarDecl()) {
+        LV.setGlobalObjCRef(true);
+        LV.setThreadLocalRef(VD->isThreadSpecified());
+      }
     }
-    LV.SetObjCArray(LV, E->getType()->isArrayType());
+    LV.setObjCArray(E->getType()->isArrayType());
     return;
   }
   
@@ -1076,7 +1064,7 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
       if (ExpTy->isPointerType())
         ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
       if (ExpTy->isRecordType())
-        LV.SetObjCIvar(LV, false); 
+        LV.setObjCIvar(false); 
     }
     return;
   }
@@ -1095,11 +1083,11 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
     if (LV.isObjCIvar() && !LV.isObjCArray()) 
       // Using array syntax to assigning to what an ivar points to is not 
       // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
-      LV.SetObjCIvar(LV, false); 
+      LV.setObjCIvar(false); 
     else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
       // Using array syntax to assigning to what global points to is not 
       // same as assigning to the global itself. {id *G;} G[i] = 0;
-      LV.SetGlobalObjCRef(LV, false);
+      LV.setGlobalObjCRef(false);
     return;
   }
   
@@ -1107,7 +1095,7 @@ static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
     setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
     // We don't know if member is an 'ivar', but this flag is looked at
     // only in the context of LV.isObjCIvar().
-    LV.SetObjCArray(LV, E->getType()->isArrayType());
+    LV.setObjCArray(E->getType()->isArrayType());
     return;
   }
 }
@@ -1120,7 +1108,8 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
   llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
   if (VD->getType()->isReferenceType())
     V = CGF.Builder.CreateLoad(V, "tmp");
-  LValue LV = LValue::MakeAddr(V, CGF.MakeQualifiers(E->getType()));
+  unsigned Alignment = CGF.getContext().getDeclAlign(VD).getQuantity();
+  LValue LV = CGF.MakeAddrLValue(V, E->getType(), Alignment);
   setObjCGCLValueClass(CGF.getContext(), E, LV);
   return LV;
 }
@@ -1140,20 +1129,18 @@ static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
       V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType), "tmp");
     }
   }
-  return LValue::MakeAddr(V, CGF.MakeQualifiers(E->getType()));
+  unsigned Alignment = CGF.getContext().getDeclAlign(FD).getQuantity();
+  return CGF.MakeAddrLValue(V, E->getType(), Alignment);
 }
 
 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
   const NamedDecl *ND = E->getDecl();
+  unsigned Alignment = CGF.getContext().getDeclAlign(ND).getQuantity();
 
   if (ND->hasAttr<WeakRefAttr>()) {
     const ValueDecl* VD = cast<ValueDecl>(ND);
     llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
-
-    Qualifiers Quals = MakeQualifiers(E->getType());
-    LValue LV = LValue::MakeAddr(Aliasee, Quals);
-
-    return LV;
+    return MakeAddrLValue(Aliasee, E->getType(), Alignment);
   }
 
   if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
@@ -1170,11 +1157,6 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
       V = CGM.getStaticLocalDeclAddress(VD);
     assert(V && "DeclRefExpr not entered in LocalDeclMap?");
 
-    Qualifiers Quals = MakeQualifiers(E->getType());
-    // local variables do not get their gc attribute set.
-    // local static?
-    if (NonGCable) Quals.removeObjCGCAttr();
-
     if (VD->hasAttr<BlocksAttr>()) {
       V = Builder.CreateStructGEP(V, 1, "forwarding");
       V = Builder.CreateLoad(V);
@@ -1183,21 +1165,31 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
     }
     if (VD->getType()->isReferenceType())
       V = Builder.CreateLoad(V, "tmp");
-    LValue LV = LValue::MakeAddr(V, Quals);
-    LValue::SetObjCNonGC(LV, NonGCable);
+
+    LValue LV = MakeAddrLValue(V, E->getType(), Alignment);
+    if (NonGCable) {
+      LV.getQuals().removeObjCGCAttr();
+      LV.setNonGC(true);
+    }
     setObjCGCLValueClass(getContext(), E, LV);
     return LV;
   }
   
+  // If we're emitting an instance method as an independent lvalue,
+  // we're actually emitting a member pointer.
+  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(ND))
+    if (MD->isInstance()) {
+      llvm::Value *V = CGM.getCXXABI().EmitMemberPointer(MD);
+      return MakeAddrLValue(V, MD->getType(), Alignment);
+    }
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
     return EmitFunctionDeclLValue(*this, E, FD);
   
-  // FIXME: the qualifier check does not seem sufficient here
-  if (E->getQualifier()) {
-    const FieldDecl *FD = cast<FieldDecl>(ND);
-    llvm::Value *V = CGM.EmitPointerToDataMember(FD);
-
-    return LValue::MakeAddr(V, MakeQualifiers(FD->getType()));
+  // If we're emitting a field as an independent lvalue, we're
+  // actually emitting a member pointer.
+  if (const FieldDecl *FD = dyn_cast<FieldDecl>(ND)) {
+    llvm::Value *V = CGM.getCXXABI().EmitMemberPointer(FD);
+    return MakeAddrLValue(V, FD->getType(), Alignment);
   }
   
   assert(false && "Unhandled DeclRefExpr");
@@ -1208,25 +1200,26 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
 }
 
 LValue CodeGenFunction::EmitBlockDeclRefLValue(const BlockDeclRefExpr *E) {
-  return LValue::MakeAddr(GetAddrOfBlockDecl(E), MakeQualifiers(E->getType()));
+  unsigned Alignment =
+    CGF.getContext().getDeclAlign(E->getDecl()).getQuantity();
+  return MakeAddrLValue(GetAddrOfBlockDecl(E), E->getType(), Alignment);
 }
 
 LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
   // __extension__ doesn't affect lvalue-ness.
-  if (E->getOpcode() == UnaryOperator::Extension)
+  if (E->getOpcode() == UO_Extension)
     return EmitLValue(E->getSubExpr());
 
   QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
   switch (E->getOpcode()) {
   default: assert(0 && "Unknown unary operator lvalue!");
-  case UnaryOperator::Deref: {
+  case UO_Deref: {
     QualType T = E->getSubExpr()->getType()->getPointeeType();
     assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
 
-    Qualifiers Quals = MakeQualifiers(T);
-    Quals.setAddressSpace(ExprTy.getAddressSpace());
+    LValue LV = MakeAddrLValue(EmitScalarExpr(E->getSubExpr()), T);
+    LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
 
-    LValue LV = LValue::MakeAddr(EmitScalarExpr(E->getSubExpr()), Quals);
     // We should not generate __weak write barrier on indirect reference
     // of a pointer to object; as in void foo (__weak id *param); *param = 0;
     // But, we continue to generate __strong write barrier on indirect write
@@ -1234,21 +1227,21 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
     if (getContext().getLangOptions().ObjC1 &&
         getContext().getLangOptions().getGCMode() != LangOptions::NonGC &&
         LV.isObjCWeak())
-      LValue::SetObjCNonGC(LV, !E->isOBJCGCCandidate(getContext()));
+      LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
     return LV;
   }
-  case UnaryOperator::Real:
-  case UnaryOperator::Imag: {
+  case UO_Real:
+  case UO_Imag: {
     LValue LV = EmitLValue(E->getSubExpr());
-    unsigned Idx = E->getOpcode() == UnaryOperator::Imag;
-    return LValue::MakeAddr(Builder.CreateStructGEP(LV.getAddress(),
+    unsigned Idx = E->getOpcode() == UO_Imag;
+    return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
                                                     Idx, "idx"),
-                            MakeQualifiers(ExprTy));
+                          ExprTy);
   }
-  case UnaryOperator::PreInc:
-  case UnaryOperator::PreDec: {
+  case UO_PreInc:
+  case UO_PreDec: {
     LValue LV = EmitLValue(E->getSubExpr());
-    bool isInc = E->getOpcode() == UnaryOperator::PreInc;
+    bool isInc = E->getOpcode() == UO_PreInc;
     
     if (E->getType()->isAnyComplexType())
       EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
@@ -1260,53 +1253,56 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
 }
 
 LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
-  return LValue::MakeAddr(CGM.GetAddrOfConstantStringFromLiteral(E),
-                          Qualifiers());
+  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
+                        E->getType());
 }
 
 LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
-  return LValue::MakeAddr(CGM.GetAddrOfConstantStringFromObjCEncode(E),
-                          Qualifiers());
+  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
+                        E->getType());
 }
 
 
-LValue CodeGenFunction::EmitPredefinedFunctionName(unsigned Type) {
-  std::string GlobalVarName;
+LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
+  switch (E->getIdentType()) {
+  default:
+    return EmitUnsupportedLValue(E, "predefined expression");
 
-  switch (Type) {
-  default: assert(0 && "Invalid type");
   case PredefinedExpr::Func:
-    GlobalVarName = "__func__.";
-    break;
   case PredefinedExpr::Function:
-    GlobalVarName = "__FUNCTION__.";
-    break;
-  case PredefinedExpr::PrettyFunction:
-    GlobalVarName = "__PRETTY_FUNCTION__.";
-    break;
-  }
+  case PredefinedExpr::PrettyFunction: {
+    unsigned Type = E->getIdentType();
+    std::string GlobalVarName;
+
+    switch (Type) {
+    default: assert(0 && "Invalid type");
+    case PredefinedExpr::Func:
+      GlobalVarName = "__func__.";
+      break;
+    case PredefinedExpr::Function:
+      GlobalVarName = "__FUNCTION__.";
+      break;
+    case PredefinedExpr::PrettyFunction:
+      GlobalVarName = "__PRETTY_FUNCTION__.";
+      break;
+    }
 
-  llvm::StringRef FnName = CurFn->getName();
-  if (FnName.startswith("\01"))
-    FnName = FnName.substr(1);
-  GlobalVarName += FnName;
+    llvm::StringRef FnName = CurFn->getName();
+    if (FnName.startswith("\01"))
+      FnName = FnName.substr(1);
+    GlobalVarName += FnName;
 
-  std::string FunctionName =
-    PredefinedExpr::ComputeName((PredefinedExpr::IdentType)Type, CurCodeDecl);
+    const Decl *CurDecl = CurCodeDecl;
+    if (CurDecl == 0)
+      CurDecl = getContext().getTranslationUnitDecl();
 
-  llvm::Constant *C =
-    CGM.GetAddrOfConstantCString(FunctionName, GlobalVarName.c_str());
-  return LValue::MakeAddr(C, Qualifiers());
-}
+    std::string FunctionName =
+      PredefinedExpr::ComputeName((PredefinedExpr::IdentType)Type, CurDecl);
 
-LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
-  switch (E->getIdentType()) {
-  default:
-    return EmitUnsupportedLValue(E, "predefined expression");
-  case PredefinedExpr::Func:
-  case PredefinedExpr::Function:
-  case PredefinedExpr::PrettyFunction:
-    return EmitPredefinedFunctionName(E->getIdentType());
+    llvm::Constant *C =
+      CGM.GetAddrOfConstantCString(FunctionName, GlobalVarName.c_str());
+    return MakeAddrLValue(C, E->getType());
+  }
   }
 }
 
@@ -1315,10 +1311,9 @@ llvm::BasicBlock *CodeGenFunction::getTrapBB() {
 
   // If we are not optimzing, don't collapse all calls to trap in the function
   // to the same call, that way, in the debugger they can see which operation
-  // did in fact fail.  If we are optimizing, we collpase all call to trap down
+  // did in fact fail.  If we are optimizing, we collapse all calls to trap down
   // to just one per function to save on codesize.
-  if (GCO.OptimizationLevel
-      && TrapBB)
+  if (GCO.OptimizationLevel && TrapBB)
     return TrapBB;
 
   llvm::BasicBlock *Cont = 0;
@@ -1345,7 +1340,7 @@ llvm::BasicBlock *CodeGenFunction::getTrapBB() {
 static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
   // If this isn't just an array->pointer decay, bail out.
   const CastExpr *CE = dyn_cast<CastExpr>(E);
-  if (CE == 0 || CE->getCastKind() != CastExpr::CK_ArrayToPointerDecay)
+  if (CE == 0 || CE->getCastKind() != CK_ArrayToPointerDecay)
     return 0;
   
   // If this is a decay from variable width array, bail out.
@@ -1382,7 +1377,7 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
   if (CatchUndefined) {
     if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E->getBase())){
       if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(ICE->getSubExpr())) {
-        if (ICE->getCastKind() == CastExpr::CK_ArrayToPointerDecay) {
+        if (ICE->getCastKind() == CK_ArrayToPointerDecay) {
           if (const ConstantArrayType *CAT
               = getContext().getAsConstantArrayType(DRE->getType())) {
             llvm::APInt Size = CAT->getSize();
@@ -1454,13 +1449,12 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
   assert(!T.isNull() &&
          "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type");
 
-  Qualifiers Quals = MakeQualifiers(T);
-  Quals.setAddressSpace(E->getBase()->getType().getAddressSpace());
+  LValue LV = MakeAddrLValue(Address, T);
+  LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
 
-  LValue LV = LValue::MakeAddr(Address, Quals);
   if (getContext().getLangOptions().ObjC1 &&
       getContext().getLangOptions().getGCMode() != LangOptions::NonGC) {
-    LValue::SetObjCNonGC(LV, !E->isOBJCGCCandidate(getContext()));
+    LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
     setObjCGCLValueClass(getContext(), E, LV);
   }
   return LV;
@@ -1489,9 +1483,8 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
     // it.
     llvm::Value *Ptr = EmitScalarExpr(E->getBase());
     const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
-    Qualifiers Quals = MakeQualifiers(PT->getPointeeType());
-    Quals.removeObjCGCAttr();
-    Base = LValue::MakeAddr(Ptr, Quals);
+    Base = MakeAddrLValue(Ptr, PT->getPointeeType());
+    Base.getQuals().removeObjCGCAttr();
   } else if (E->getBase()->isLvalue(getContext()) == Expr::LV_Valid) {
     // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
     // emit the base as an lvalue.
@@ -1506,7 +1499,7 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
     // Store the vector to memory (because LValue wants an address).
     llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
     Builder.CreateStore(Vec, VecMem);
-    Base = LValue::MakeAddr(VecMem, Qualifiers());
+    Base = MakeAddrLValue(VecMem, E->getBase()->getType());
   }
   
   // Encode the element access list into a vector of unsigned indices.
@@ -1566,7 +1559,7 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
   if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
     LValue LV = EmitLValueForField(BaseValue, Field, 
                                    BaseQuals.getCVRQualifiers());
-    LValue::SetObjCNonGC(LV, isNonGC);
+    LV.setNonGC(isNonGC);
     setObjCGCLValueClass(getContext(), E, LV);
     return LV;
   }
@@ -1645,13 +1638,15 @@ LValue CodeGenFunction::EmitLValueForField(llvm::Value* BaseValue,
   if (Field->getType()->isReferenceType())
     V = Builder.CreateLoad(V, "tmp");
 
-  Qualifiers Quals = MakeQualifiers(Field->getType());
-  Quals.addCVRQualifiers(CVRQualifiers);
+  unsigned Alignment = getContext().getDeclAlign(Field).getQuantity();
+  LValue LV = MakeAddrLValue(V, Field->getType(), Alignment);
+  LV.getQuals().addCVRQualifiers(CVRQualifiers);
+
   // __weak attribute on a field is ignored.
-  if (Quals.getObjCGCAttr() == Qualifiers::Weak)
-    Quals.removeObjCGCAttr();
+  if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
+    LV.getQuals().removeObjCGCAttr();
   
-  return LValue::MakeAddr(V, Quals);
+  return LV;
 }
 
 LValue 
@@ -1670,13 +1665,14 @@ CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value* BaseValue,
 
   assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
 
-  return LValue::MakeAddr(V, MakeQualifiers(FieldType));
+  unsigned Alignment = getContext().getDeclAlign(Field).getQuantity();
+  return MakeAddrLValue(V, FieldType, Alignment);
 }
 
 LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr* E){
   llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
   const Expr* InitExpr = E->getInitializer();
-  LValue Result = LValue::MakeAddr(DeclPtr, MakeQualifiers(E->getType()));
+  LValue Result = MakeAddrLValue(DeclPtr, E->getType());
 
   EmitAnyExprToMem(InitExpr, DeclPtr, /*Volatile*/ false);
 
@@ -1729,7 +1725,7 @@ CodeGenFunction::EmitConditionalOperatorLValue(const ConditionalOperator* E) {
     EmitBlock(ContBlock);
     
     Temp = Builder.CreateLoad(Temp, "lv");
-    return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+    return MakeAddrLValue(Temp, E->getType());
   }
   
   // ?: here should be an aggregate.
@@ -1749,35 +1745,65 @@ CodeGenFunction::EmitConditionalOperatorLValue(const ConditionalOperator* E) {
 /// cast from scalar to union.
 LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   switch (E->getCastKind()) {
-  default:
+  case CK_ToVoid:
     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
-
-  case CastExpr::CK_Dynamic: {
+   
+  case CK_NoOp:
+    if (E->getSubExpr()->Classify(getContext()).getKind() 
+                                          != Expr::Classification::CL_PRValue) {
+      LValue LV = EmitLValue(E->getSubExpr());
+      if (LV.isPropertyRef() || LV.isKVCRef()) {
+        QualType QT = E->getSubExpr()->getType();
+        RValue RV = 
+          LV.isPropertyRef() ? EmitLoadOfPropertyRefLValue(LV, QT) 
+                             : EmitLoadOfKVCRefLValue(LV, QT);
+        assert(!RV.isScalar() && "EmitCastLValue-scalar cast of property ref");
+        llvm::Value *V = RV.getAggregateAddr();
+        return MakeAddrLValue(V, QT);
+      }
+      return LV;
+    }
+    // Fall through to synthesize a temporary.
+      
+  case CK_Unknown:
+  case CK_BitCast:
+  case CK_ArrayToPointerDecay:
+  case CK_FunctionToPointerDecay:
+  case CK_NullToMemberPointer:
+  case CK_IntegralToPointer:
+  case CK_PointerToIntegral:
+  case CK_VectorSplat:
+  case CK_IntegralCast:
+  case CK_IntegralToFloating:
+  case CK_FloatingToIntegral:
+  case CK_FloatingCast:
+  case CK_DerivedToBaseMemberPointer:
+  case CK_BaseToDerivedMemberPointer:
+  case CK_MemberPointerToBoolean:
+  case CK_AnyPointerToBlockPointerCast: {
+    // These casts only produce lvalues when we're binding a reference to a 
+    // temporary realized from a (converted) pure rvalue. Emit the expression
+    // as a value, copy it into a temporary, and return an lvalue referring to
+    // that temporary.
+    llvm::Value *V = CreateMemTemp(E->getType(), "ref.temp");
+    EmitAnyExprToMem(E, V, false, false);
+    return MakeAddrLValue(V, E->getType());
+  }
+
+  case CK_Dynamic: {
     LValue LV = EmitLValue(E->getSubExpr());
     llvm::Value *V = LV.getAddress();
     const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(E);
-    return LValue::MakeAddr(EmitDynamicCast(V, DCE),
-                            MakeQualifiers(E->getType()));
+    return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType());
   }
 
-  case CastExpr::CK_NoOp: {
-    LValue LV = EmitLValue(E->getSubExpr());
-    if (LV.isPropertyRef()) {
-      QualType QT = E->getSubExpr()->getType();
-      RValue RV = EmitLoadOfPropertyRefLValue(LV, QT);
-      assert(!RV.isScalar() && "EmitCastLValue - scalar cast of property ref");
-      llvm::Value *V = RV.getAggregateAddr();
-      return LValue::MakeAddr(V, MakeQualifiers(QT));
-    }
-    return LV;
-  }
-  case CastExpr::CK_ConstructorConversion:
-  case CastExpr::CK_UserDefinedConversion:
-  case CastExpr::CK_AnyPointerToObjCPointerCast:
+  case CK_ConstructorConversion:
+  case CK_UserDefinedConversion:
+  case CK_AnyPointerToObjCPointerCast:
     return EmitLValue(E->getSubExpr());
   
-  case CastExpr::CK_UncheckedDerivedToBase:
-  case CastExpr::CK_DerivedToBase: {
+  case CK_UncheckedDerivedToBase:
+  case CK_DerivedToBase: {
     const RecordType *DerivedClassTy = 
       E->getSubExpr()->getType()->getAs<RecordType>();
     CXXRecordDecl *DerivedClassDecl = 
@@ -1785,8 +1811,11 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     
     LValue LV = EmitLValue(E->getSubExpr());
     llvm::Value *This;
-    if (LV.isPropertyRef()) {
-      RValue RV = EmitLoadOfPropertyRefLValue(LV, E->getSubExpr()->getType());
+    if (LV.isPropertyRef() || LV.isKVCRef()) {
+      QualType QT = E->getSubExpr()->getType();
+      RValue RV = 
+        LV.isPropertyRef() ? EmitLoadOfPropertyRefLValue(LV, QT)
+                           : EmitLoadOfKVCRefLValue(LV, QT);
       assert (!RV.isScalar() && "EmitCastLValue");
       This = RV.getAggregateAddr();
     }
@@ -1796,13 +1825,14 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     // Perform the derived-to-base conversion
     llvm::Value *Base = 
       GetAddressOfBaseClass(This, DerivedClassDecl, 
-                            E->getBasePath(), /*NullCheckValue=*/false);
+                            E->path_begin(), E->path_end(),
+                            /*NullCheckValue=*/false);
     
-    return LValue::MakeAddr(Base, MakeQualifiers(E->getType()));
+    return MakeAddrLValue(Base, E->getType());
   }
-  case CastExpr::CK_ToUnion:
+  case CK_ToUnion:
     return EmitAggExprToLValue(E);
-  case CastExpr::CK_BaseToDerived: {
+  case CK_BaseToDerived: {
     const RecordType *DerivedClassTy = E->getType()->getAs<RecordType>();
     CXXRecordDecl *DerivedClassDecl = 
       cast<CXXRecordDecl>(DerivedClassTy->getDecl());
@@ -1812,26 +1842,36 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     // Perform the base-to-derived conversion
     llvm::Value *Derived = 
       GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl, 
-                               E->getBasePath(),/*NullCheckValue=*/false);
+                               E->path_begin(), E->path_end(),
+                               /*NullCheckValue=*/false);
     
-    return LValue::MakeAddr(Derived, MakeQualifiers(E->getType()));
+    return MakeAddrLValue(Derived, E->getType());
   }
-  case CastExpr::CK_LValueBitCast: {
+  case CK_LValueBitCast: {
     // This must be a reinterpret_cast (or c-style equivalent).
     const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
     
     LValue LV = EmitLValue(E->getSubExpr());
     llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
                                            ConvertType(CE->getTypeAsWritten()));
-    return LValue::MakeAddr(V, MakeQualifiers(E->getType()));
+    return MakeAddrLValue(V, E->getType());
   }
+  case CK_ObjCObjectLValueCast: {
+    LValue LV = EmitLValue(E->getSubExpr());
+    QualType ToType = getContext().getLValueReferenceType(E->getType());
+    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(), 
+                                           ConvertType(ToType));
+    return MakeAddrLValue(V, E->getType());
   }
+  }
+  
+  llvm_unreachable("Unhandled lvalue cast kind?");
 }
 
 LValue CodeGenFunction::EmitNullInitializationLValue(
                                               const CXXScalarValueInitExpr *E) {
   QualType Ty = E->getType();
-  LValue LV = LValue::MakeAddr(CreateMemTemp(Ty), MakeQualifiers(Ty));
+  LValue LV = MakeAddrLValue(CreateMemTemp(Ty), Ty);
   EmitNullInitialization(LV.getAddress(), Ty);
   return LV;
 }
@@ -1881,28 +1921,26 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
 
 LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
   // Comma expressions just emit their LHS then their RHS as an l-value.
-  if (E->getOpcode() == BinaryOperator::Comma) {
+  if (E->getOpcode() == BO_Comma) {
     EmitAnyExpr(E->getLHS());
     EnsureInsertPoint();
     return EmitLValue(E->getRHS());
   }
 
-  if (E->getOpcode() == BinaryOperator::PtrMemD ||
-      E->getOpcode() == BinaryOperator::PtrMemI)
+  if (E->getOpcode() == BO_PtrMemD ||
+      E->getOpcode() == BO_PtrMemI)
     return EmitPointerToDataMemberBinaryExpr(E);
   
   // Can only get l-value for binary operator expressions which are a
   // simple assignment of aggregate type.
-  if (E->getOpcode() != BinaryOperator::Assign)
+  if (E->getOpcode() != BO_Assign)
     return EmitUnsupportedLValue(E, "binary l-value expression");
 
   if (!hasAggregateLLVMType(E->getType())) {
     // Emit the LHS as an l-value.
     LValue LV = EmitLValue(E->getLHS());
-    
-    llvm::Value *RHS = EmitScalarExpr(E->getRHS());
-    EmitStoreOfScalar(RHS, LV.getAddress(), LV.isVolatileQualified(), 
-                      E->getType());
+    // Store the value through the l-value.
+    EmitStoreThroughLValue(EmitAnyExpr(E->getRHS()), LV, E->getType());
     return LV;
   }
   
@@ -1913,13 +1951,13 @@ LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
   RValue RV = EmitCallExpr(E);
 
   if (!RV.isScalar())
-    return LValue::MakeAddr(RV.getAggregateAddr(),MakeQualifiers(E->getType()));
+    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
     
   assert(E->getCallReturnType()->isReferenceType() &&
          "Can't have a scalar return unless the return type is a "
          "reference type!");
 
-  return LValue::MakeAddr(RV.getScalarVal(), MakeQualifiers(E->getType()));
+  return MakeAddrLValue(RV.getScalarVal(), E->getType());
 }
 
 LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
@@ -1930,13 +1968,12 @@ LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
   llvm::Value *Temp = CreateMemTemp(E->getType(), "tmp");
   EmitCXXConstructExpr(Temp, E);
-  return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+  return MakeAddrLValue(Temp, E->getType());
 }
 
 LValue
 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
-  llvm::Value *Temp = EmitCXXTypeidExpr(E);
-  return LValue::MakeAddr(Temp, MakeQualifiers(E->getType()));
+  return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
 }
 
 LValue
@@ -1950,20 +1987,19 @@ LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
   RValue RV = EmitObjCMessageExpr(E);
   
   if (!RV.isScalar())
-    return LValue::MakeAddr(RV.getAggregateAddr(),
-                            MakeQualifiers(E->getType()));
+    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
   
   assert(E->getMethodDecl()->getResultType()->isReferenceType() &&
          "Can't have a scalar return unless the return type is a "
          "reference type!");
   
-  return LValue::MakeAddr(RV.getScalarVal(), MakeQualifiers(E->getType()));
+  return MakeAddrLValue(RV.getScalarVal(), E->getType());
 }
 
 LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
   llvm::Value *V = 
     CGM.getObjCRuntime().GetSelector(Builder, E->getSelector(), true);
-  return LValue::MakeAddr(V, MakeQualifiers(E->getType()));
+  return MakeAddrLValue(V, E->getType());
 }
 
 llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
@@ -2025,7 +2061,7 @@ LValue CodeGenFunction::EmitObjCSuperExprLValue(const ObjCSuperExpr *E) {
 LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
   // Can only get l-value for message expression returning aggregate type
   RValue RV = EmitAnyExprToTemp(E);
-  return LValue::MakeAddr(RV.getAggregateAddr(), MakeQualifiers(E->getType()));
+  return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
 }
 
 RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
@@ -2054,20 +2090,19 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
 LValue CodeGenFunction::
 EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
   llvm::Value *BaseV;
-  if (E->getOpcode() == BinaryOperator::PtrMemI)
+  if (E->getOpcode() == BO_PtrMemI)
     BaseV = EmitScalarExpr(E->getLHS());
   else
     BaseV = EmitLValue(E->getLHS()).getAddress();
-  const llvm::Type *i8Ty = llvm::Type::getInt8PtrTy(getLLVMContext());
-  BaseV = Builder.CreateBitCast(BaseV, i8Ty);
+
   llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
-  llvm::Value *AddV = Builder.CreateInBoundsGEP(BaseV, OffsetV, "add.ptr");
 
-  QualType Ty = E->getRHS()->getType();
-  Ty = Ty->getAs<MemberPointerType>()->getPointeeType();
-  
-  const llvm::Type *PType = ConvertType(getContext().getPointerType(Ty));
-  AddV = Builder.CreateBitCast(AddV, PType);
-  return LValue::MakeAddr(AddV, MakeQualifiers(Ty));
+  const MemberPointerType *MPT
+    = E->getRHS()->getType()->getAs<MemberPointerType>();
+
+  llvm::Value *AddV =
+    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, BaseV, OffsetV, MPT);
+
+  return MakeAddrLValue(AddV, MPT->getPointeeType());
 }
 
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 219a5f915329..28c8b3545b38 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -108,7 +108,6 @@ public:
   void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
   void VisitBinAssign(const BinaryOperator *E);
   void VisitBinComma(const BinaryOperator *E);
-  void VisitUnaryAddrOf(const UnaryOperator *E);
 
   void VisitObjCMessageExpr(ObjCMessageExpr *E);
   void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
@@ -193,10 +192,18 @@ void AggExprEmitter::EmitGCMove(const Expr *E, RValue Src) {
 void AggExprEmitter::EmitFinalDestCopy(const Expr *E, RValue Src, bool Ignore) {
   assert(Src.isAggregate() && "value must be aggregate value!");
 
-  // If the result is ignored, don't copy from the value.
+  // If DestPtr is null, then we're evaluating an aggregate expression
+  // in a context (like an expression statement) that doesn't care
+  // about the result.  C says that an lvalue-to-rvalue conversion is
+  // performed in these cases; C++ says that it is not.  In either
+  // case, we don't actually need to do anything unless the value is
+  // volatile.
   if (DestPtr == 0) {
-    if (!Src.isVolatileQualified() || (IgnoreResult && Ignore))
+    if (!Src.isVolatileQualified() ||
+        CGF.CGM.getLangOptions().CPlusPlus ||
+        (IgnoreResult && Ignore))
       return;
+
     // If the source is volatile, we must read from it; to do that, we need
     // some place to put it.
     DestPtr = CGF.CreateMemTemp(E->getType(), "agg.tmp");
@@ -235,7 +242,7 @@ void AggExprEmitter::EmitFinalDestCopy(const Expr *E, LValue Src, bool Ignore) {
 //===----------------------------------------------------------------------===//
 
 void AggExprEmitter::VisitCastExpr(CastExpr *E) {
-  if (!DestPtr && E->getCastKind() != CastExpr::CK_Dynamic) {
+  if (!DestPtr && E->getCastKind() != CK_Dynamic) {
     Visit(E->getSubExpr());
     return;
   }
@@ -243,7 +250,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   switch (E->getCastKind()) {
   default: assert(0 && "Unhandled cast kind!");
 
-  case CastExpr::CK_Dynamic: {
+  case CK_Dynamic: {
     assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
     LValue LV = CGF.EmitCheckedLValue(E->getSubExpr());
     // FIXME: Do we also need to handle property references here?
@@ -257,105 +264,39 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
     break;
   }
       
-  case CastExpr::CK_ToUnion: {
+  case CK_ToUnion: {
     // GCC union extension
-    QualType PtrTy =
-    CGF.getContext().getPointerType(E->getSubExpr()->getType());
+    QualType Ty = E->getSubExpr()->getType();
+    QualType PtrTy = CGF.getContext().getPointerType(Ty);
     llvm::Value *CastPtr = Builder.CreateBitCast(DestPtr,
                                                  CGF.ConvertType(PtrTy));
-    EmitInitializationToLValue(E->getSubExpr(),
-                               LValue::MakeAddr(CastPtr, Qualifiers()), 
-                               E->getSubExpr()->getType());
+    EmitInitializationToLValue(E->getSubExpr(), CGF.MakeAddrLValue(CastPtr, Ty),
+                               Ty);
     break;
   }
 
-  case CastExpr::CK_DerivedToBase:
-  case CastExpr::CK_BaseToDerived:
-  case CastExpr::CK_UncheckedDerivedToBase: {
+  case CK_DerivedToBase:
+  case CK_BaseToDerived:
+  case CK_UncheckedDerivedToBase: {
     assert(0 && "cannot perform hierarchy conversion in EmitAggExpr: "
                 "should have been unpacked before we got here");
     break;
   }
 
   // FIXME: Remove the CK_Unknown check here.
-  case CastExpr::CK_Unknown:
-  case CastExpr::CK_NoOp:
-  case CastExpr::CK_UserDefinedConversion:
-  case CastExpr::CK_ConstructorConversion:
+  case CK_Unknown:
+  case CK_NoOp:
+  case CK_UserDefinedConversion:
+  case CK_ConstructorConversion:
     assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
                                                    E->getType()) &&
            "Implicit cast types must be compatible");
     Visit(E->getSubExpr());
     break;
 
-  case CastExpr::CK_NullToMemberPointer: {
-    // If the subexpression's type is the C++0x nullptr_t, emit the
-    // subexpression, which may have side effects.
-    if (E->getSubExpr()->getType()->isNullPtrType())
-      Visit(E->getSubExpr());
-
-    const llvm::Type *PtrDiffTy = 
-      CGF.ConvertType(CGF.getContext().getPointerDiffType());
-
-    llvm::Value *NullValue = llvm::Constant::getNullValue(PtrDiffTy);
-    llvm::Value *Ptr = Builder.CreateStructGEP(DestPtr, 0, "ptr");
-    Builder.CreateStore(NullValue, Ptr, VolatileDest);
-    
-    llvm::Value *Adj = Builder.CreateStructGEP(DestPtr, 1, "adj");
-    Builder.CreateStore(NullValue, Adj, VolatileDest);
-
-    break;
-  }
-      
-  case CastExpr::CK_LValueBitCast:
+  case CK_LValueBitCast:
     llvm_unreachable("there are no lvalue bit-casts on aggregates");
     break;
-      
-  case CastExpr::CK_BitCast: {
-    // This must be a member function pointer cast.
-    Visit(E->getSubExpr());
-    break;
-  }
-
-  case CastExpr::CK_DerivedToBaseMemberPointer:
-  case CastExpr::CK_BaseToDerivedMemberPointer: {
-    QualType SrcType = E->getSubExpr()->getType();
-    
-    llvm::Value *Src = CGF.CreateMemTemp(SrcType, "tmp");
-    CGF.EmitAggExpr(E->getSubExpr(), Src, SrcType.isVolatileQualified());
-    
-    llvm::Value *SrcPtr = Builder.CreateStructGEP(Src, 0, "src.ptr");
-    SrcPtr = Builder.CreateLoad(SrcPtr);
-    
-    llvm::Value *SrcAdj = Builder.CreateStructGEP(Src, 1, "src.adj");
-    SrcAdj = Builder.CreateLoad(SrcAdj);
-    
-    llvm::Value *DstPtr = Builder.CreateStructGEP(DestPtr, 0, "dst.ptr");
-    Builder.CreateStore(SrcPtr, DstPtr, VolatileDest);
-    
-    llvm::Value *DstAdj = Builder.CreateStructGEP(DestPtr, 1, "dst.adj");
-    
-    // Now See if we need to update the adjustment.
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(SrcType->getAs<MemberPointerType>()->
-                          getClass()->getAs<RecordType>()->getDecl());
-    const CXXRecordDecl *DerivedDecl = 
-      cast<CXXRecordDecl>(E->getType()->getAs<MemberPointerType>()->
-                          getClass()->getAs<RecordType>()->getDecl());
-    if (E->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
-      std::swap(DerivedDecl, BaseDecl);
-
-    if (llvm::Constant *Adj = 
-          CGF.CGM.GetNonVirtualBaseClassOffset(DerivedDecl, E->getBasePath())) {
-      if (E->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
-        SrcAdj = Builder.CreateSub(SrcAdj, Adj, "adj");
-      else
-        SrcAdj = Builder.CreateAdd(SrcAdj, Adj, "adj");
-    }
-    
-    Builder.CreateStore(SrcAdj, DstAdj, VolatileDest);
-    break;
-  }
   }
 }
 
@@ -391,42 +332,12 @@ void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
                   /*IgnoreResult=*/false, IsInitializer);
 }
 
-void AggExprEmitter::VisitUnaryAddrOf(const UnaryOperator *E) {
-  // We have a member function pointer.
-  const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>();
-  (void) MPT;
-  assert(MPT->getPointeeType()->isFunctionProtoType() &&
-         "Unexpected member pointer type!");
-
-  // The creation of member function pointers has no side effects; if
-  // there is no destination pointer, we have nothing to do.
-  if (!DestPtr)
-    return;
-  
-  const DeclRefExpr *DRE = cast<DeclRefExpr>(E->getSubExpr());
-  const CXXMethodDecl *MD = 
-    cast<CXXMethodDecl>(DRE->getDecl())->getCanonicalDecl();
-
-  const llvm::Type *PtrDiffTy = 
-    CGF.ConvertType(CGF.getContext().getPointerDiffType());
-
-  llvm::Value *DstPtr = Builder.CreateStructGEP(DestPtr, 0, "dst.ptr");
-  llvm::Value *FuncPtr = CGF.CGM.GetCXXMemberFunctionPointerValue(MD);
-  Builder.CreateStore(FuncPtr, DstPtr, VolatileDest);
-
-  llvm::Value *AdjPtr = Builder.CreateStructGEP(DestPtr, 1, "dst.adj");
-  // The adjustment will always be 0.
-  Builder.CreateStore(llvm::ConstantInt::get(PtrDiffTy, 0), AdjPtr,
-                      VolatileDest);
-}
-
 void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
   CGF.EmitCompoundStmt(*E->getSubStmt(), true, DestPtr, VolatileDest);
 }
 
 void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
-  if (E->getOpcode() == BinaryOperator::PtrMemD ||
-      E->getOpcode() == BinaryOperator::PtrMemI)
+  if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
     VisitPointerToDataMemberBinaryOperator(E);
   else
     CGF.ErrorUnsupported(E, "aggregate binary expression");
@@ -519,7 +430,7 @@ void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
     return;
   }
 
-  EmitFinalDestCopy(VE, LValue::MakeAddr(ArgPtr, Qualifiers()));
+  EmitFinalDestCopy(VE, CGF.MakeAddrLValue(ArgPtr, VE->getType()));
 }
 
 void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
@@ -546,12 +457,6 @@ AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
   if (!Val) // Create a temporary variable.
     Val = CGF.CreateMemTemp(E->getType(), "tmp");
 
-  if (E->requiresZeroInitialization())
-    EmitNullInitializationToLValue(LValue::MakeAddr(Val, 
-                                                    // FIXME: Qualifiers()?
-                                                 E->getType().getQualifiers()),
-                                   E->getType());
-
   CGF.EmitCXXConstructExpr(Val, E);
 }
 
@@ -568,8 +473,8 @@ void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
     // Create a temporary variable.
     Val = CGF.CreateMemTemp(E->getType(), "tmp");
   }
-  LValue LV = LValue::MakeAddr(Val, Qualifiers());
-  EmitNullInitializationToLValue(LV, E->getType());
+  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Val, E->getType()),
+                                 E->getType());
 }
 
 void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
@@ -579,8 +484,8 @@ void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
     // Create a temporary variable.
     Val = CGF.CreateMemTemp(E->getType(), "tmp");
   }
-  LValue LV = LValue::MakeAddr(Val, Qualifiers());
-  EmitNullInitializationToLValue(LV, E->getType());
+  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Val, E->getType()),
+                                 E->getType());
 }
 
 void 
@@ -625,7 +530,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     llvm::GlobalVariable* GV =
     new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
                              llvm::GlobalValue::InternalLinkage, C, "");
-    EmitFinalDestCopy(E, LValue::MakeAddr(GV, Qualifiers()));
+    EmitFinalDestCopy(E, CGF.MakeAddrLValue(GV, E->getType()));
     return;
   }
 #endif
@@ -656,17 +561,15 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     ElementType = CGF.getContext().getAsArrayType(ElementType)->getElementType();
 
     // FIXME: were we intentionally ignoring address spaces and GC attributes?
-    Qualifiers Quals = CGF.MakeQualifiers(ElementType);
 
     for (uint64_t i = 0; i != NumArrayElements; ++i) {
       llvm::Value *NextVal = Builder.CreateStructGEP(DestPtr, i, ".array");
+      LValue LV = CGF.MakeAddrLValue(NextVal, ElementType);
       if (i < NumInitElements)
-        EmitInitializationToLValue(E->getInit(i),
-                                   LValue::MakeAddr(NextVal, Quals), 
-                                   ElementType);
+        EmitInitializationToLValue(E->getInit(i), LV, ElementType);
+
       else
-        EmitNullInitializationToLValue(LValue::MakeAddr(NextVal, Quals),
-                                       ElementType);
+        EmitNullInitializationToLValue(LV, ElementType);
     }
     return;
   }
@@ -679,8 +582,17 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
   // the optimizer, especially with bitfields.
   unsigned NumInitElements = E->getNumInits();
   RecordDecl *SD = E->getType()->getAs<RecordType>()->getDecl();
-  unsigned CurInitVal = 0;
-
+  
+  // If we're initializing the whole aggregate, just do it in place.
+  // FIXME: This is a hack around an AST bug (PR6537).
+  if (NumInitElements == 1 && E->getType() == E->getInit(0)->getType()) {
+    EmitInitializationToLValue(E->getInit(0),
+                               CGF.MakeAddrLValue(DestPtr, E->getType()),
+                               E->getType());
+    return;
+  }
+  
+  
   if (E->getType()->isUnionType()) {
     // Only initialize one field of a union. The field itself is
     // specified by the initializer list.
@@ -712,19 +624,10 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
 
     return;
   }
-  
-  // If we're initializing the whole aggregate, just do it in place.
-  // FIXME: This is a hack around an AST bug (PR6537).
-  if (NumInitElements == 1 && E->getType() == E->getInit(0)->getType()) {
-    EmitInitializationToLValue(E->getInit(0),
-                               LValue::MakeAddr(DestPtr, Qualifiers()),
-                               E->getType());
-    return;
-  }
-  
 
   // Here we iterate over the fields; this makes it simpler to both
   // default-initialize fields and skip over unnamed fields.
+  unsigned CurInitVal = 0;
   for (RecordDecl::field_iterator Field = SD->field_begin(),
                                FieldEnd = SD->field_end();
        Field != FieldEnd; ++Field) {
@@ -738,7 +641,7 @@ void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
     // FIXME: volatility
     LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestPtr, *Field, 0);
     // We never generate write-barries for initialized fields.
-    LValue::SetObjCNonGC(FieldLoc, true);
+    FieldLoc.setNonGC(true);
     if (CurInitVal < NumInitElements) {
       // Store the initializer into the field.
       EmitInitializationToLValue(E->getInit(CurInitVal++), FieldLoc,
@@ -776,10 +679,10 @@ void CodeGenFunction::EmitAggExpr(const Expr *E, llvm::Value *DestPtr,
 
 LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
   assert(hasAggregateLLVMType(E->getType()) && "Invalid argument!");
-  Qualifiers Q = MakeQualifiers(E->getType());
   llvm::Value *Temp = CreateMemTemp(E->getType());
-  EmitAggExpr(E, Temp, Q.hasVolatile());
-  return LValue::MakeAddr(Temp, Q);
+  LValue LV = MakeAddrLValue(Temp, E->getType());
+  EmitAggExpr(E, Temp, LV.isVolatileQualified());
+  return LV;
 }
 
 void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index 69e5f0ef4be8..9a98281771fa 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -12,7 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenFunction.h"
+#include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
+#include "llvm/Intrinsics.h"
 using namespace clang;
 using namespace CodeGen;
 
@@ -91,14 +93,9 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
     return EmitCall(getContext().getPointerType(MD->getType()), Callee,
                     ReturnValue, CE->arg_begin(), CE->arg_end());
   }
-  
-  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
 
-  const llvm::Type *Ty =
-    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
-                                   FPT->isVariadic());
+  // Compute the object pointer.
   llvm::Value *This;
-
   if (ME->isArrow())
     This = EmitScalarExpr(ME->getBase());
   else {
@@ -106,7 +103,10 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
     This = BaseLV.getAddress();
   }
 
-  if (MD->isCopyAssignment() && MD->isTrivial()) {
+  if (MD->isTrivial()) {
+    if (isa<CXXDestructorDecl>(MD)) return RValue::get(0);
+
+    assert(MD->isCopyAssignment() && "unknown trivial member function");
     // We don't like to generate the trivial copy assignment operator when
     // it isn't necessary; just produce the proper effect here.
     llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
@@ -114,25 +114,34 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
     return RValue::get(This);
   }
 
+  // Compute the function type we're calling.
+  const CGFunctionInfo &FInfo =
+    (isa<CXXDestructorDecl>(MD)
+     ? CGM.getTypes().getFunctionInfo(cast<CXXDestructorDecl>(MD),
+                                      Dtor_Complete)
+     : CGM.getTypes().getFunctionInfo(MD));
+
+  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+  const llvm::Type *Ty
+    = CGM.getTypes().GetFunctionType(FInfo, FPT->isVariadic());
+
   // C++ [class.virtual]p12:
   //   Explicit qualification with the scope operator (5.1) suppresses the
   //   virtual call mechanism.
   //
   // We also don't emit a virtual call if the base expression has a record type
   // because then we know what the type is.
+  bool UseVirtualCall = MD->isVirtual() && !ME->hasQualifier()
+                     && !canDevirtualizeMemberFunctionCalls(ME->getBase());
+
   llvm::Value *Callee;
-  if (const CXXDestructorDecl *Destructor
-             = dyn_cast<CXXDestructorDecl>(MD)) {
-    if (Destructor->isTrivial())
-      return RValue::get(0);
-    if (MD->isVirtual() && !ME->hasQualifier() && 
-        !canDevirtualizeMemberFunctionCalls(ME->getBase())) {
-      Callee = BuildVirtualCall(Destructor, Dtor_Complete, This, Ty); 
+  if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(MD)) {
+    if (UseVirtualCall) {
+      Callee = BuildVirtualCall(Dtor, Dtor_Complete, This, Ty);
     } else {
-      Callee = CGM.GetAddrOfFunction(GlobalDecl(Destructor, Dtor_Complete), Ty);
+      Callee = CGM.GetAddrOfFunction(GlobalDecl(Dtor, Dtor_Complete), Ty);
     }
-  } else if (MD->isVirtual() && !ME->hasQualifier() && 
-             !canDevirtualizeMemberFunctionCalls(ME->getBase())) {
+  } else if (UseVirtualCall) {
     Callee = BuildVirtualCall(MD, This, Ty); 
   } else {
     Callee = CGM.GetAddrOfFunction(MD, Ty);
@@ -152,89 +161,27 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   
   const MemberPointerType *MPT = 
     MemFnExpr->getType()->getAs<MemberPointerType>();
+
   const FunctionProtoType *FPT = 
     MPT->getPointeeType()->getAs<FunctionProtoType>();
   const CXXRecordDecl *RD = 
     cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
 
-  const llvm::FunctionType *FTy = 
-    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
-                                   FPT->isVariadic());
-
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-
   // Get the member function pointer.
-  llvm::Value *MemFnPtr = CreateMemTemp(MemFnExpr->getType(), "mem.fn");
-  EmitAggExpr(MemFnExpr, MemFnPtr, /*VolatileDest=*/false);
+  llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr);
 
   // Emit the 'this' pointer.
   llvm::Value *This;
   
-  if (BO->getOpcode() == BinaryOperator::PtrMemI)
+  if (BO->getOpcode() == BO_PtrMemI)
     This = EmitScalarExpr(BaseExpr);
   else 
     This = EmitLValue(BaseExpr).getAddress();
-  
-  // Adjust it.
-  llvm::Value *Adj = Builder.CreateStructGEP(MemFnPtr, 1);
-  Adj = Builder.CreateLoad(Adj, "mem.fn.adj");
-  
-  llvm::Value *Ptr = Builder.CreateBitCast(This, Int8PtrTy, "ptr");
-  Ptr = Builder.CreateGEP(Ptr, Adj, "adj");
-  
-  This = Builder.CreateBitCast(Ptr, This->getType(), "this");
-  
-  llvm::Value *FnPtr = Builder.CreateStructGEP(MemFnPtr, 0, "mem.fn.ptr");
-  
-  const llvm::Type *PtrDiffTy = ConvertType(getContext().getPointerDiffType());
 
-  llvm::Value *FnAsInt = Builder.CreateLoad(FnPtr, "fn");
-  
-  // If the LSB in the function pointer is 1, the function pointer points to
-  // a virtual function.
-  llvm::Value *IsVirtual 
-    = Builder.CreateAnd(FnAsInt, llvm::ConstantInt::get(PtrDiffTy, 1),
-                        "and");
-  
-  IsVirtual = Builder.CreateTrunc(IsVirtual,
-                                  llvm::Type::getInt1Ty(VMContext));
+  // Ask the ABI to load the callee.  Note that This is modified.
+  llvm::Value *Callee =
+    CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(CGF, This, MemFnPtr, MPT);
   
-  llvm::BasicBlock *FnVirtual = createBasicBlock("fn.virtual");
-  llvm::BasicBlock *FnNonVirtual = createBasicBlock("fn.nonvirtual");
-  llvm::BasicBlock *FnEnd = createBasicBlock("fn.end");
-  
-  Builder.CreateCondBr(IsVirtual, FnVirtual, FnNonVirtual);
-  EmitBlock(FnVirtual);
-  
-  const llvm::Type *VTableTy = 
-    FTy->getPointerTo()->getPointerTo();
-
-  llvm::Value *VTable = Builder.CreateBitCast(This, VTableTy->getPointerTo());
-  VTable = Builder.CreateLoad(VTable);
-  
-  VTable = Builder.CreateBitCast(VTable, Int8PtrTy);
-  llvm::Value *VTableOffset = 
-    Builder.CreateSub(FnAsInt, llvm::ConstantInt::get(PtrDiffTy, 1));
-  
-  VTable = Builder.CreateGEP(VTable, VTableOffset, "fn");
-  VTable = Builder.CreateBitCast(VTable, VTableTy);
-  
-  llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "virtualfn");
-  
-  EmitBranch(FnEnd);
-  EmitBlock(FnNonVirtual);
-  
-  // If the function is not virtual, just load the pointer.
-  llvm::Value *NonVirtualFn = Builder.CreateLoad(FnPtr, "fn");
-  NonVirtualFn = Builder.CreateIntToPtr(NonVirtualFn, FTy->getPointerTo());
-  
-  EmitBlock(FnEnd);
-
-  llvm::PHINode *Callee = Builder.CreatePHI(FTy->getPointerTo());
-  Callee->reserveOperandSpace(2);
-  Callee->addIncoming(VirtualFn, FnVirtual);
-  Callee->addIncoming(NonVirtualFn, FnNonVirtual);
-
   CallArgList Args;
 
   QualType ThisType = 
@@ -263,11 +210,17 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
              "EmitCXXOperatorMemberCallExpr - user declared copy assignment");
       LValue LV = EmitLValue(E->getArg(0));
       llvm::Value *This;
-      if (LV.isPropertyRef()) {
+      if (LV.isPropertyRef() || LV.isKVCRef()) {
         llvm::Value *AggLoc  = CreateMemTemp(E->getArg(1)->getType());
         EmitAggExpr(E->getArg(1), AggLoc, false /*VolatileDest*/);
-        EmitObjCPropertySet(LV.getPropertyRefExpr(),
-                            RValue::getAggregate(AggLoc, false /*VolatileDest*/));
+        if (LV.isPropertyRef())
+          EmitObjCPropertySet(LV.getPropertyRefExpr(),
+                              RValue::getAggregate(AggLoc, 
+                                                   false /*VolatileDest*/));
+        else
+          EmitObjCPropertySet(LV.getKVCRefExpr(),
+                              RValue::getAggregate(AggLoc, 
+                                                   false /*VolatileDest*/));
         return RValue::getAggregate(0, false);
       }
       else
@@ -286,8 +239,11 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
                                    FPT->isVariadic());
   LValue LV = EmitLValue(E->getArg(0));
   llvm::Value *This;
-  if (LV.isPropertyRef()) {
-    RValue RV = EmitLoadOfPropertyRefLValue(LV, E->getArg(0)->getType());
+  if (LV.isPropertyRef() || LV.isKVCRef()) {
+    QualType QT = E->getArg(0)->getType();
+    RValue RV = 
+      LV.isPropertyRef() ? EmitLoadOfPropertyRefLValue(LV, QT)
+                         : EmitLoadOfKVCRefLValue(LV, QT);
     assert (!RV.isScalar() && "EmitCXXOperatorMemberCallExpr");
     This = RV.getAggregateAddr();
   }
@@ -309,19 +265,18 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
                                       const CXXConstructExpr *E) {
   assert(Dest && "Must have a destination!");
   const CXXConstructorDecl *CD = E->getConstructor();
-  const ConstantArrayType *Array =
-  getContext().getAsConstantArrayType(E->getType());
-  // For a copy constructor, even if it is trivial, must fall thru so
-  // its argument is code-gen'ed.
-  if (!CD->isCopyConstructor()) {
-    QualType InitType = E->getType();
-    if (Array)
-      InitType = getContext().getBaseElementType(Array);
-    const CXXRecordDecl *RD =
-    cast<CXXRecordDecl>(InitType->getAs<RecordType>()->getDecl());
-    if (RD->hasTrivialConstructor())
-      return;
-  }
+  
+  // If we require zero initialization before (or instead of) calling the
+  // constructor, as can be the case with a non-user-provided default
+  // constructor, emit the zero initialization now.
+  if (E->requiresZeroInitialization())
+    EmitNullInitialization(Dest, E->getType());
+
+  
+  // If this is a call to a trivial default constructor, do nothing.
+  if (CD->isTrivial() && CD->isDefaultConstructor())
+    return;
+  
   // Code gen optimization to eliminate copy constructor and return
   // its first argument instead, if in fact that argument is a temporary 
   // object.
@@ -331,6 +286,9 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
       return;
     }
   }
+  
+  const ConstantArrayType *Array 
+    = getContext().getAsConstantArrayType(E->getType());
   if (Array) {
     QualType BaseElementTy = getContext().getBaseElementType(Array);
     const llvm::Type *BasePtr = ConvertType(BaseElementTy);
@@ -354,131 +312,205 @@ CodeGenFunction::EmitCXXConstructExpr(llvm::Value *Dest,
   }
 }
 
-static CharUnits CalculateCookiePadding(ASTContext &Ctx, QualType ElementType) {
-  const RecordType *RT = ElementType->getAs<RecordType>();
-  if (!RT)
-    return CharUnits::Zero();
-  
-  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
-  if (!RD)
-    return CharUnits::Zero();
-  
-  // Check if the class has a trivial destructor.
-  if (RD->hasTrivialDestructor()) {
-    // Check if the usual deallocation function takes two arguments.
-    const CXXMethodDecl *UsualDeallocationFunction = 0;
-    
-    DeclarationName OpName =
-      Ctx.DeclarationNames.getCXXOperatorName(OO_Array_Delete);
-    DeclContext::lookup_const_iterator Op, OpEnd;
-    for (llvm::tie(Op, OpEnd) = RD->lookup(OpName);
-         Op != OpEnd; ++Op) {
-      const CXXMethodDecl *Delete = cast<CXXMethodDecl>(*Op);
-
-      if (Delete->isUsualDeallocationFunction()) {
-        UsualDeallocationFunction = Delete;
-        break;
-      }
-    }
-    
-    // No usual deallocation function, we don't need a cookie.
-    if (!UsualDeallocationFunction)
-      return CharUnits::Zero();
-    
-    // The usual deallocation function doesn't take a size_t argument, so we
-    // don't need a cookie.
-    if (UsualDeallocationFunction->getNumParams() == 1)
-      return CharUnits::Zero();
-        
-    assert(UsualDeallocationFunction->getNumParams() == 2 && 
-           "Unexpected deallocation function type!");
-  }  
-  
-  // Padding is the maximum of sizeof(size_t) and alignof(ElementType)
-  return std::max(Ctx.getTypeSizeInChars(Ctx.getSizeType()),
-                  Ctx.getTypeAlignInChars(ElementType));
+/// Check whether the given operator new[] is the global placement
+/// operator new[].
+static bool IsPlacementOperatorNewArray(ASTContext &Ctx,
+                                        const FunctionDecl *Fn) {
+  // Must be in global scope.  Note that allocation functions can't be
+  // declared in namespaces.
+  if (!Fn->getDeclContext()->getRedeclContext()->isFileContext())
+    return false;
+
+  // Signature must be void *operator new[](size_t, void*).
+  // The size_t is common to all operator new[]s.
+  if (Fn->getNumParams() != 2)
+    return false;
+
+  CanQualType ParamType = Ctx.getCanonicalType(Fn->getParamDecl(1)->getType());
+  return (ParamType == Ctx.VoidPtrTy);
 }
 
-static CharUnits CalculateCookiePadding(ASTContext &Ctx, const CXXNewExpr *E) {
+static CharUnits CalculateCookiePadding(CodeGenFunction &CGF,
+                                        const CXXNewExpr *E) {
   if (!E->isArray())
     return CharUnits::Zero();
 
   // No cookie is required if the new operator being used is 
   // ::operator new[](size_t, void*).
   const FunctionDecl *OperatorNew = E->getOperatorNew();
-  if (OperatorNew->getDeclContext()->getLookupContext()->isFileContext()) {
-    if (OperatorNew->getNumParams() == 2) {
-      CanQualType ParamType = 
-        Ctx.getCanonicalType(OperatorNew->getParamDecl(1)->getType());
-      
-      if (ParamType == Ctx.VoidPtrTy)
-        return CharUnits::Zero();
-    }
-  }
-      
-  return CalculateCookiePadding(Ctx, E->getAllocatedType());
+  if (IsPlacementOperatorNewArray(CGF.getContext(), OperatorNew))
+    return CharUnits::Zero();
+
+  return CGF.CGM.getCXXABI().GetArrayCookieSize(E->getAllocatedType());
 }
 
 static llvm::Value *EmitCXXNewAllocSize(ASTContext &Context,
-                                        CodeGenFunction &CGF, 
+                                        CodeGenFunction &CGF,
                                         const CXXNewExpr *E,
-                                        llvm::Value *& NumElements) {
-  QualType Type = E->getAllocatedType();
-  CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(Type);
-  const llvm::Type *SizeTy = CGF.ConvertType(CGF.getContext().getSizeType());
-  
-  if (!E->isArray())
-    return llvm::ConstantInt::get(SizeTy, TypeSize.getQuantity());
+                                        llvm::Value *&NumElements,
+                                        llvm::Value *&SizeWithoutCookie) {
+  QualType ElemType = E->getAllocatedType();
 
-  CharUnits CookiePadding = CalculateCookiePadding(CGF.getContext(), E);
+  const llvm::IntegerType *SizeTy =
+    cast<llvm::IntegerType>(CGF.ConvertType(CGF.getContext().getSizeType()));
   
-  Expr::EvalResult Result;
-  if (E->getArraySize()->Evaluate(Result, CGF.getContext()) &&
-      !Result.HasSideEffects && Result.Val.isInt()) {
+  CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(ElemType);
 
-    CharUnits AllocSize = 
-      Result.Val.getInt().getZExtValue() * TypeSize + CookiePadding;
-    
-    NumElements = 
-      llvm::ConstantInt::get(SizeTy, Result.Val.getInt().getZExtValue());
-    while (const ArrayType *AType = Context.getAsArrayType(Type)) {
-      const llvm::ArrayType *llvmAType =
-        cast<llvm::ArrayType>(CGF.ConvertType(Type));
-      NumElements =
-        CGF.Builder.CreateMul(NumElements, 
-                              llvm::ConstantInt::get(
-                                        SizeTy, llvmAType->getNumElements()));
-      Type = AType->getElementType();
-    }
-    
-    return llvm::ConstantInt::get(SizeTy, AllocSize.getQuantity());
+  if (!E->isArray()) {
+    SizeWithoutCookie = llvm::ConstantInt::get(SizeTy, TypeSize.getQuantity());
+    return SizeWithoutCookie;
   }
-  
+
+  // Figure out the cookie size.
+  CharUnits CookieSize = CalculateCookiePadding(CGF, E);
+
   // Emit the array size expression.
+  // We multiply the size of all dimensions for NumElements.
+  // e.g for 'int[2][3]', ElemType is 'int' and NumElements is 6.
   NumElements = CGF.EmitScalarExpr(E->getArraySize());
-  
-  // Multiply with the type size.
-  llvm::Value *V = 
-    CGF.Builder.CreateMul(NumElements, 
-                          llvm::ConstantInt::get(SizeTy, 
-                                                 TypeSize.getQuantity()));
-  
-  while (const ArrayType *AType = Context.getAsArrayType(Type)) {
-    const llvm::ArrayType *llvmAType =
-      cast<llvm::ArrayType>(CGF.ConvertType(Type));
-    NumElements =
-      CGF.Builder.CreateMul(NumElements, 
-                            llvm::ConstantInt::get(
-                                          SizeTy, llvmAType->getNumElements()));
-    Type = AType->getElementType();
+  assert(NumElements->getType() == SizeTy && "element count not a size_t");
+
+  uint64_t ArraySizeMultiplier = 1;
+  while (const ConstantArrayType *CAT
+             = CGF.getContext().getAsConstantArrayType(ElemType)) {
+    ElemType = CAT->getElementType();
+    ArraySizeMultiplier *= CAT->getSize().getZExtValue();
   }
 
-  // And add the cookie padding if necessary.
-  if (!CookiePadding.isZero())
-    V = CGF.Builder.CreateAdd(V, 
-        llvm::ConstantInt::get(SizeTy, CookiePadding.getQuantity()));
-  
-  return V;
+  llvm::Value *Size;
+  
+  // If someone is doing 'new int[42]' there is no need to do a dynamic check.
+  // Don't bloat the -O0 code.
+  if (llvm::ConstantInt *NumElementsC =
+        dyn_cast<llvm::ConstantInt>(NumElements)) {
+    llvm::APInt NEC = NumElementsC->getValue();
+    unsigned SizeWidth = NEC.getBitWidth();
+
+    // Determine if there is an overflow here by doing an extended multiply.
+    NEC.zext(SizeWidth*2);
+    llvm::APInt SC(SizeWidth*2, TypeSize.getQuantity());
+    SC *= NEC;
+
+    if (!CookieSize.isZero()) {
+      // Save the current size without a cookie.  We don't care if an
+      // overflow's already happened because SizeWithoutCookie isn't
+      // used if the allocator returns null or throws, as it should
+      // always do on an overflow.
+      llvm::APInt SWC = SC;
+      SWC.trunc(SizeWidth);
+      SizeWithoutCookie = llvm::ConstantInt::get(SizeTy, SWC);
+
+      // Add the cookie size.
+      SC += llvm::APInt(SizeWidth*2, CookieSize.getQuantity());
+    }
+    
+    if (SC.countLeadingZeros() >= SizeWidth) {
+      SC.trunc(SizeWidth);
+      Size = llvm::ConstantInt::get(SizeTy, SC);
+    } else {
+      // On overflow, produce a -1 so operator new throws.
+      Size = llvm::Constant::getAllOnesValue(SizeTy);
+    }
+
+    // Scale NumElements while we're at it.
+    uint64_t N = NEC.getZExtValue() * ArraySizeMultiplier;
+    NumElements = llvm::ConstantInt::get(SizeTy, N);
+
+  // Otherwise, we don't need to do an overflow-checked multiplication if
+  // we're multiplying by one.
+  } else if (TypeSize.isOne()) {
+    assert(ArraySizeMultiplier == 1);
+
+    Size = NumElements;
+
+    // If we need a cookie, add its size in with an overflow check.
+    // This is maybe a little paranoid.
+    if (!CookieSize.isZero()) {
+      SizeWithoutCookie = Size;
+
+      llvm::Value *CookieSizeV
+        = llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity());
+
+      const llvm::Type *Types[] = { SizeTy };
+      llvm::Value *UAddF
+        = CGF.CGM.getIntrinsic(llvm::Intrinsic::uadd_with_overflow, Types, 1);
+      llvm::Value *AddRes
+        = CGF.Builder.CreateCall2(UAddF, Size, CookieSizeV);
+
+      Size = CGF.Builder.CreateExtractValue(AddRes, 0);
+      llvm::Value *DidOverflow = CGF.Builder.CreateExtractValue(AddRes, 1);
+      Size = CGF.Builder.CreateSelect(DidOverflow,
+                                      llvm::ConstantInt::get(SizeTy, -1),
+                                      Size);
+    }
+
+  // Otherwise use the int.umul.with.overflow intrinsic.
+  } else {
+    llvm::Value *OutermostElementSize
+      = llvm::ConstantInt::get(SizeTy, TypeSize.getQuantity());
+
+    llvm::Value *NumOutermostElements = NumElements;
+
+    // Scale NumElements by the array size multiplier.  This might
+    // overflow, but only if the multiplication below also overflows,
+    // in which case this multiplication isn't used.
+    if (ArraySizeMultiplier != 1)
+      NumElements = CGF.Builder.CreateMul(NumElements,
+                         llvm::ConstantInt::get(SizeTy, ArraySizeMultiplier));
+
+    // The requested size of the outermost array is non-constant.
+    // Multiply that by the static size of the elements of that array;
+    // on unsigned overflow, set the size to -1 to trigger an
+    // exception from the allocation routine.  This is sufficient to
+    // prevent buffer overruns from the allocator returning a
+    // seemingly valid pointer to insufficient space.  This idea comes
+    // originally from MSVC, and GCC has an open bug requesting
+    // similar behavior:
+    //   http://gcc.gnu.org/bugzilla/show_bug.cgi?id=19351
+    //
+    // This will not be sufficient for C++0x, which requires a
+    // specific exception class (std::bad_array_new_length).
+    // That will require ABI support that has not yet been specified.
+    const llvm::Type *Types[] = { SizeTy };
+    llvm::Value *UMulF
+      = CGF.CGM.getIntrinsic(llvm::Intrinsic::umul_with_overflow, Types, 1);
+    llvm::Value *MulRes = CGF.Builder.CreateCall2(UMulF, NumOutermostElements,
+                                                  OutermostElementSize);
+
+    // The overflow bit.
+    llvm::Value *DidOverflow = CGF.Builder.CreateExtractValue(MulRes, 1);
+
+    // The result of the multiplication.
+    Size = CGF.Builder.CreateExtractValue(MulRes, 0);
+
+    // If we have a cookie, we need to add that size in, too.
+    if (!CookieSize.isZero()) {
+      SizeWithoutCookie = Size;
+
+      llvm::Value *CookieSizeV
+        = llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity());
+      llvm::Value *UAddF
+        = CGF.CGM.getIntrinsic(llvm::Intrinsic::uadd_with_overflow, Types, 1);
+      llvm::Value *AddRes
+        = CGF.Builder.CreateCall2(UAddF, SizeWithoutCookie, CookieSizeV);
+
+      Size = CGF.Builder.CreateExtractValue(AddRes, 0);
+
+      llvm::Value *AddDidOverflow = CGF.Builder.CreateExtractValue(AddRes, 1);
+      DidOverflow = CGF.Builder.CreateAnd(DidOverflow, AddDidOverflow);
+    }
+
+    Size = CGF.Builder.CreateSelect(DidOverflow,
+                                    llvm::ConstantInt::get(SizeTy, -1),
+                                    Size);
+  }
+
+  if (CookieSize.isZero())
+    SizeWithoutCookie = Size;
+  else
+    assert(SizeWithoutCookie && "didn't set SizeWithoutCookie?");
+
+  return Size;
 }
 
 static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const CXXNewExpr *E,
@@ -489,10 +521,13 @@ static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const CXXNewExpr *E,
   
   const Expr *Init = E->getConstructorArg(0);
   QualType AllocType = E->getAllocatedType();
-  
+
+  unsigned Alignment =
+    CGF.getContext().getTypeAlignInChars(AllocType).getQuantity();
   if (!CGF.hasAggregateLLVMType(AllocType)) 
     CGF.EmitStoreOfScalar(CGF.EmitScalarExpr(Init), NewPtr,
-                          AllocType.isVolatileQualified(), AllocType);
+                          AllocType.isVolatileQualified(), Alignment,
+                          AllocType);
   else if (AllocType->isAnyComplexType())
     CGF.EmitComplexExprIntoAddr(Init, NewPtr, 
                                 AllocType.isVolatileQualified());
@@ -554,18 +589,59 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
   EmitBlock(AfterFor, true);
 }
 
+static void EmitZeroMemSet(CodeGenFunction &CGF, QualType T,
+                           llvm::Value *NewPtr, llvm::Value *Size) {
+  llvm::LLVMContext &VMContext = CGF.CGM.getLLVMContext();
+  const llvm::Type *BP = llvm::Type::getInt8PtrTy(VMContext);
+  if (NewPtr->getType() != BP)
+    NewPtr = CGF.Builder.CreateBitCast(NewPtr, BP, "tmp");
+  
+  CGF.Builder.CreateCall5(CGF.CGM.getMemSetFn(BP, CGF.IntPtrTy), NewPtr,
+                llvm::Constant::getNullValue(llvm::Type::getInt8Ty(VMContext)),
+                          Size,
+                    llvm::ConstantInt::get(CGF.Int32Ty, 
+                                           CGF.getContext().getTypeAlign(T)/8),
+                          llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext),
+                                                 0));
+}
+                       
 static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
                                llvm::Value *NewPtr,
-                               llvm::Value *NumElements) {
+                               llvm::Value *NumElements,
+                               llvm::Value *AllocSizeWithoutCookie) {
   if (E->isArray()) {
     if (CXXConstructorDecl *Ctor = E->getConstructor()) {
-      if (!Ctor->getParent()->hasTrivialConstructor())
-        CGF.EmitCXXAggrConstructorCall(Ctor, NumElements, NewPtr, 
-                                       E->constructor_arg_begin(), 
-                                       E->constructor_arg_end());
+      bool RequiresZeroInitialization = false;
+      if (Ctor->getParent()->hasTrivialConstructor()) {
+        // If new expression did not specify value-initialization, then there
+        // is no initialization.
+        if (!E->hasInitializer() || Ctor->getParent()->isEmpty())
+          return;
+      
+        if (CGF.CGM.getTypes().isZeroInitializable(E->getAllocatedType())) {
+          // Optimization: since zero initialization will just set the memory
+          // to all zeroes, generate a single memset to do it in one shot.
+          EmitZeroMemSet(CGF, E->getAllocatedType(), NewPtr, 
+                         AllocSizeWithoutCookie);
+          return;
+        }
+
+        RequiresZeroInitialization = true;
+      }
+      
+      CGF.EmitCXXAggrConstructorCall(Ctor, NumElements, NewPtr, 
+                                     E->constructor_arg_begin(), 
+                                     E->constructor_arg_end(),
+                                     RequiresZeroInitialization);
       return;
-    }
-    else {
+    } else if (E->getNumConstructorArgs() == 1 &&
+               isa<ImplicitValueInitExpr>(E->getConstructorArg(0))) {
+      // Optimization: since zero initialization will just set the memory
+      // to all zeroes, generate a single memset to do it in one shot.
+      EmitZeroMemSet(CGF, E->getAllocatedType(), NewPtr, 
+                     AllocSizeWithoutCookie);
+      return;      
+    } else {
       CGF.EmitNewArrayInitializer(E, NewPtr, NumElements);
       return;
     }
@@ -595,6 +671,10 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
 
 llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   QualType AllocType = E->getAllocatedType();
+  if (AllocType->isArrayType())
+    while (const ArrayType *AType = getContext().getAsArrayType(AllocType))
+      AllocType = AType->getElementType();
+
   FunctionDecl *NewFD = E->getOperatorNew();
   const FunctionProtoType *NewFTy = NewFD->getType()->getAs<FunctionProtoType>();
 
@@ -604,8 +684,10 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   QualType SizeTy = getContext().getSizeType();
 
   llvm::Value *NumElements = 0;
+  llvm::Value *AllocSizeWithoutCookie = 0;
   llvm::Value *AllocSize = EmitCXXNewAllocSize(getContext(),
-                                               *this, E, NumElements);
+                                               *this, E, NumElements,
+                                               AllocSizeWithoutCookie);
   
   NewArgs.push_back(std::make_pair(RValue::get(AllocSize), SizeTy));
 
@@ -654,112 +736,69 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   bool NullCheckResult = NewFTy->hasEmptyExceptionSpec() &&
     !(AllocType->isPODType() && !E->hasInitializer());
 
-  llvm::BasicBlock *NewNull = 0;
+  llvm::BasicBlock *NullCheckSource = 0;
   llvm::BasicBlock *NewNotNull = 0;
   llvm::BasicBlock *NewEnd = 0;
 
   llvm::Value *NewPtr = RV.getScalarVal();
+  unsigned AS = cast<llvm::PointerType>(NewPtr->getType())->getAddressSpace();
 
   if (NullCheckResult) {
-    NewNull = createBasicBlock("new.null");
+    NullCheckSource = Builder.GetInsertBlock();
     NewNotNull = createBasicBlock("new.notnull");
     NewEnd = createBasicBlock("new.end");
 
-    llvm::Value *IsNull =
-      Builder.CreateICmpEQ(NewPtr,
-                           llvm::Constant::getNullValue(NewPtr->getType()),
-                           "isnull");
-
-    Builder.CreateCondBr(IsNull, NewNull, NewNotNull);
+    llvm::Value *IsNull = Builder.CreateIsNull(NewPtr, "new.isnull");
+    Builder.CreateCondBr(IsNull, NewEnd, NewNotNull);
     EmitBlock(NewNotNull);
   }
   
-  CharUnits CookiePadding = CalculateCookiePadding(getContext(), E);
-  if (!CookiePadding.isZero()) {
-    CharUnits CookieOffset = 
-      CookiePadding - getContext().getTypeSizeInChars(SizeTy);
-    
-    llvm::Value *NumElementsPtr = 
-      Builder.CreateConstInBoundsGEP1_64(NewPtr, CookieOffset.getQuantity());
-    
-    NumElementsPtr = Builder.CreateBitCast(NumElementsPtr, 
-                                           ConvertType(SizeTy)->getPointerTo());
-    Builder.CreateStore(NumElements, NumElementsPtr);
-
-    // Now add the padding to the new ptr.
-    NewPtr = Builder.CreateConstInBoundsGEP1_64(NewPtr, 
-                                                CookiePadding.getQuantity());
+  assert((AllocSize == AllocSizeWithoutCookie) ==
+         CalculateCookiePadding(*this, E).isZero());
+  if (AllocSize != AllocSizeWithoutCookie) {
+    assert(E->isArray());
+    NewPtr = CGM.getCXXABI().InitializeArrayCookie(CGF, NewPtr, NumElements,
+                                                   AllocType);
   }
-  
-  if (AllocType->isArrayType()) {
-    while (const ArrayType *AType = getContext().getAsArrayType(AllocType))
-      AllocType = AType->getElementType();
-    NewPtr = 
-      Builder.CreateBitCast(NewPtr, 
-                          ConvertType(getContext().getPointerType(AllocType)));
-    EmitNewInitializer(*this, E, NewPtr, NumElements);
-    NewPtr = Builder.CreateBitCast(NewPtr, ConvertType(E->getType()));
-  }
-  else {
-    NewPtr = Builder.CreateBitCast(NewPtr, ConvertType(E->getType()));
-    EmitNewInitializer(*this, E, NewPtr, NumElements);
+
+  const llvm::Type *ElementPtrTy
+    = ConvertTypeForMem(AllocType)->getPointerTo(AS);
+  NewPtr = Builder.CreateBitCast(NewPtr, ElementPtrTy);
+  if (E->isArray()) {
+    EmitNewInitializer(*this, E, NewPtr, NumElements, AllocSizeWithoutCookie);
+
+    // NewPtr is a pointer to the base element type.  If we're
+    // allocating an array of arrays, we'll need to cast back to the
+    // array pointer type.
+    const llvm::Type *ResultTy = ConvertTypeForMem(E->getType());
+    if (NewPtr->getType() != ResultTy)
+      NewPtr = Builder.CreateBitCast(NewPtr, ResultTy);
+  } else {
+    EmitNewInitializer(*this, E, NewPtr, NumElements, AllocSizeWithoutCookie);
   }
   
   if (NullCheckResult) {
     Builder.CreateBr(NewEnd);
-    NewNotNull = Builder.GetInsertBlock();
-    EmitBlock(NewNull);
-    Builder.CreateBr(NewEnd);
+    llvm::BasicBlock *NotNullSource = Builder.GetInsertBlock();
     EmitBlock(NewEnd);
 
     llvm::PHINode *PHI = Builder.CreatePHI(NewPtr->getType());
     PHI->reserveOperandSpace(2);
-    PHI->addIncoming(NewPtr, NewNotNull);
-    PHI->addIncoming(llvm::Constant::getNullValue(NewPtr->getType()), NewNull);
+    PHI->addIncoming(NewPtr, NotNullSource);
+    PHI->addIncoming(llvm::Constant::getNullValue(NewPtr->getType()),
+                     NullCheckSource);
 
     NewPtr = PHI;
   }
-
-  return NewPtr;
-}
-
-static std::pair<llvm::Value *, llvm::Value *>
-GetAllocatedObjectPtrAndNumElements(CodeGenFunction &CGF,
-                                    llvm::Value *Ptr, QualType DeleteTy) {
-  QualType SizeTy = CGF.getContext().getSizeType();
-  const llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
-  
-  CharUnits DeleteTypeAlign = CGF.getContext().getTypeAlignInChars(DeleteTy);
-  CharUnits CookiePadding = 
-    std::max(CGF.getContext().getTypeSizeInChars(SizeTy),
-             DeleteTypeAlign);
-  assert(!CookiePadding.isZero() && "CookiePadding should not be 0.");
-
-  const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
-  CharUnits CookieOffset = 
-    CookiePadding - CGF.getContext().getTypeSizeInChars(SizeTy);
-
-  llvm::Value *AllocatedObjectPtr = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
-  AllocatedObjectPtr = 
-    CGF.Builder.CreateConstInBoundsGEP1_64(AllocatedObjectPtr,
-                                           -CookiePadding.getQuantity());
-
-  llvm::Value *NumElementsPtr =
-    CGF.Builder.CreateConstInBoundsGEP1_64(AllocatedObjectPtr, 
-                                           CookieOffset.getQuantity());
-  NumElementsPtr = 
-    CGF.Builder.CreateBitCast(NumElementsPtr, SizeLTy->getPointerTo());
-  
-  llvm::Value *NumElements = CGF.Builder.CreateLoad(NumElementsPtr);
-  NumElements = 
-    CGF.Builder.CreateIntCast(NumElements, SizeLTy, /*isSigned=*/false);
   
-  return std::make_pair(AllocatedObjectPtr, NumElements);
+  return NewPtr;
 }
 
 void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
                                      llvm::Value *Ptr,
                                      QualType DeleteTy) {
+  assert(DeleteFD->getOverloadedOperator() == OO_Delete);
+
   const FunctionProtoType *DeleteFTy =
     DeleteFD->getType()->getAs<FunctionProtoType>();
 
@@ -775,21 +814,6 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
                                   DeleteTypeSize.getQuantity());
   }
   
-  if (DeleteFD->getOverloadedOperator() == OO_Array_Delete &&
-      !CalculateCookiePadding(getContext(), DeleteTy).isZero()) {
-    // We need to get the number of elements in the array from the cookie.
-    llvm::Value *AllocatedObjectPtr;
-    llvm::Value *NumElements;
-    llvm::tie(AllocatedObjectPtr, NumElements) =
-      GetAllocatedObjectPtrAndNumElements(*this, Ptr, DeleteTy);
-    
-    // Multiply the size with the number of elements.
-    if (Size)
-      Size = Builder.CreateMul(NumElements, Size);
-    
-    Ptr = AllocatedObjectPtr;
-  }
-  
   QualType ArgTy = DeleteFTy->getArgType(0);
   llvm::Value *DeletePtr = Builder.CreateBitCast(Ptr, ConvertType(ArgTy));
   DeleteArgs.push_back(std::make_pair(RValue::get(DeletePtr), ArgTy));
@@ -803,20 +827,169 @@ void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
            DeleteArgs, DeleteFD);
 }
 
+namespace {
+  /// Calls the given 'operator delete' on a single object.
+  struct CallObjectDelete : EHScopeStack::Cleanup {
+    llvm::Value *Ptr;
+    const FunctionDecl *OperatorDelete;
+    QualType ElementType;
+
+    CallObjectDelete(llvm::Value *Ptr,
+                     const FunctionDecl *OperatorDelete,
+                     QualType ElementType)
+      : Ptr(Ptr), OperatorDelete(OperatorDelete), ElementType(ElementType) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType);
+    }
+  };
+}
+
+/// Emit the code for deleting a single object.
+static void EmitObjectDelete(CodeGenFunction &CGF,
+                             const FunctionDecl *OperatorDelete,
+                             llvm::Value *Ptr,
+                             QualType ElementType) {
+  // Find the destructor for the type, if applicable.  If the
+  // destructor is virtual, we'll just emit the vcall and return.
+  const CXXDestructorDecl *Dtor = 0;
+  if (const RecordType *RT = ElementType->getAs<RecordType>()) {
+    CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    if (!RD->hasTrivialDestructor()) {
+      Dtor = RD->getDestructor();
+
+      if (Dtor->isVirtual()) {
+        const llvm::Type *Ty =
+          CGF.getTypes().GetFunctionType(CGF.getTypes().getFunctionInfo(Dtor,
+                                                               Dtor_Complete),
+                                         /*isVariadic=*/false);
+          
+        llvm::Value *Callee
+          = CGF.BuildVirtualCall(Dtor, Dtor_Deleting, Ptr, Ty);
+        CGF.EmitCXXMemberCall(Dtor, Callee, ReturnValueSlot(), Ptr, /*VTT=*/0,
+                              0, 0);
+
+        // The dtor took care of deleting the object.
+        return;
+      }
+    }
+  }
+
+  // Make sure that we call delete even if the dtor throws.
+  CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
+                                            Ptr, OperatorDelete, ElementType);
+
+  if (Dtor)
+    CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
+                              /*ForVirtualBase=*/false, Ptr);
+
+  CGF.PopCleanupBlock();
+}
+
+namespace {
+  /// Calls the given 'operator delete' on an array of objects.
+  struct CallArrayDelete : EHScopeStack::Cleanup {
+    llvm::Value *Ptr;
+    const FunctionDecl *OperatorDelete;
+    llvm::Value *NumElements;
+    QualType ElementType;
+    CharUnits CookieSize;
+
+    CallArrayDelete(llvm::Value *Ptr,
+                    const FunctionDecl *OperatorDelete,
+                    llvm::Value *NumElements,
+                    QualType ElementType,
+                    CharUnits CookieSize)
+      : Ptr(Ptr), OperatorDelete(OperatorDelete), NumElements(NumElements),
+        ElementType(ElementType), CookieSize(CookieSize) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      const FunctionProtoType *DeleteFTy =
+        OperatorDelete->getType()->getAs<FunctionProtoType>();
+      assert(DeleteFTy->getNumArgs() == 1 || DeleteFTy->getNumArgs() == 2);
+
+      CallArgList Args;
+      
+      // Pass the pointer as the first argument.
+      QualType VoidPtrTy = DeleteFTy->getArgType(0);
+      llvm::Value *DeletePtr
+        = CGF.Builder.CreateBitCast(Ptr, CGF.ConvertType(VoidPtrTy));
+      Args.push_back(std::make_pair(RValue::get(DeletePtr), VoidPtrTy));
+
+      // Pass the original requested size as the second argument.
+      if (DeleteFTy->getNumArgs() == 2) {
+        QualType size_t = DeleteFTy->getArgType(1);
+        const llvm::IntegerType *SizeTy
+          = cast<llvm::IntegerType>(CGF.ConvertType(size_t));
+        
+        CharUnits ElementTypeSize =
+          CGF.CGM.getContext().getTypeSizeInChars(ElementType);
+
+        // The size of an element, multiplied by the number of elements.
+        llvm::Value *Size
+          = llvm::ConstantInt::get(SizeTy, ElementTypeSize.getQuantity());
+        Size = CGF.Builder.CreateMul(Size, NumElements);
+
+        // Plus the size of the cookie if applicable.
+        if (!CookieSize.isZero()) {
+          llvm::Value *CookieSizeV
+            = llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity());
+          Size = CGF.Builder.CreateAdd(Size, CookieSizeV);
+        }
+
+        Args.push_back(std::make_pair(RValue::get(Size), size_t));
+      }
+
+      // Emit the call to delete.
+      CGF.EmitCall(CGF.getTypes().getFunctionInfo(Args, DeleteFTy),
+                   CGF.CGM.GetAddrOfFunction(OperatorDelete),
+                   ReturnValueSlot(), Args, OperatorDelete);
+    }
+  };
+}
+
+/// Emit the code for deleting an array of objects.
+static void EmitArrayDelete(CodeGenFunction &CGF,
+                            const FunctionDecl *OperatorDelete,
+                            llvm::Value *Ptr,
+                            QualType ElementType) {
+  llvm::Value *NumElements = 0;
+  llvm::Value *AllocatedPtr = 0;
+  CharUnits CookieSize;
+  CGF.CGM.getCXXABI().ReadArrayCookie(CGF, Ptr, ElementType,
+                                      NumElements, AllocatedPtr, CookieSize);
+
+  assert(AllocatedPtr && "ReadArrayCookie didn't set AllocatedPtr");
+
+  // Make sure that we call delete even if one of the dtors throws.
+  CGF.EHStack.pushCleanup<CallArrayDelete>(NormalAndEHCleanup,
+                                           AllocatedPtr, OperatorDelete,
+                                           NumElements, ElementType,
+                                           CookieSize);
+
+  if (const CXXRecordDecl *RD = ElementType->getAsCXXRecordDecl()) {
+    if (!RD->hasTrivialDestructor()) {
+      assert(NumElements && "ReadArrayCookie didn't find element count"
+                            " for a class with destructor");
+      CGF.EmitCXXAggrDestructorCall(RD->getDestructor(), NumElements, Ptr);
+    }
+  }
+
+  CGF.PopCleanupBlock();
+}
+
 void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
   
   // Get at the argument before we performed the implicit conversion
   // to void*.
   const Expr *Arg = E->getArgument();
   while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Arg)) {
-    if (ICE->getCastKind() != CastExpr::CK_UserDefinedConversion &&
+    if (ICE->getCastKind() != CK_UserDefinedConversion &&
         ICE->getType()->isVoidPointerType())
       Arg = ICE->getSubExpr();
     else
       break;
   }
-  
-  QualType DeleteTy = Arg->getType()->getAs<PointerType>()->getPointeeType();
 
   llvm::Value *Ptr = EmitScalarExpr(Arg);
 
@@ -830,41 +1003,38 @@ void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
 
   Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);
   EmitBlock(DeleteNotNull);
-  
-  bool ShouldCallDelete = true;
-  
-  // Call the destructor if necessary.
-  if (const RecordType *RT = DeleteTy->getAs<RecordType>()) {
-    if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
-      if (!RD->hasTrivialDestructor()) {
-        const CXXDestructorDecl *Dtor = RD->getDestructor();
-        if (E->isArrayForm()) {
-          llvm::Value *AllocatedObjectPtr;
-          llvm::Value *NumElements;
-          llvm::tie(AllocatedObjectPtr, NumElements) =
-            GetAllocatedObjectPtrAndNumElements(*this, Ptr, DeleteTy);
-          
-          EmitCXXAggrDestructorCall(Dtor, NumElements, Ptr);
-        } else if (Dtor->isVirtual()) {
-          const llvm::Type *Ty =
-            CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(Dtor),
-                                           /*isVariadic=*/false);
-          
-          llvm::Value *Callee = BuildVirtualCall(Dtor, Dtor_Deleting, Ptr, Ty);
-          EmitCXXMemberCall(Dtor, Callee, ReturnValueSlot(), Ptr, /*VTT=*/0,
-                            0, 0);
-
-          // The dtor took care of deleting the object.
-          ShouldCallDelete = false;
-        } else 
-          EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
-                                Ptr);
-      }
+
+  // We might be deleting a pointer to array.  If so, GEP down to the
+  // first non-array element.
+  // (this assumes that A(*)[3][7] is converted to [3 x [7 x %A]]*)
+  QualType DeleteTy = Arg->getType()->getAs<PointerType>()->getPointeeType();
+  if (DeleteTy->isConstantArrayType()) {
+    llvm::Value *Zero = Builder.getInt32(0);
+    llvm::SmallVector<llvm::Value*,8> GEP;
+
+    GEP.push_back(Zero); // point at the outermost array
+
+    // For each layer of array type we're pointing at:
+    while (const ConstantArrayType *Arr
+             = getContext().getAsConstantArrayType(DeleteTy)) {
+      // 1. Unpeel the array type.
+      DeleteTy = Arr->getElementType();
+
+      // 2. GEP to the first element of the array.
+      GEP.push_back(Zero);
     }
+
+    Ptr = Builder.CreateInBoundsGEP(Ptr, GEP.begin(), GEP.end(), "del.first");
   }
 
-  if (ShouldCallDelete)
-    EmitDeleteCall(E->getOperatorDelete(), Ptr, DeleteTy);
+  assert(ConvertTypeForMem(DeleteTy) ==
+         cast<llvm::PointerType>(Ptr->getType())->getElementType());
+
+  if (E->isArrayForm()) {
+    EmitArrayDelete(*this, E->getOperatorDelete(), Ptr, DeleteTy);
+  } else {
+    EmitObjectDelete(*this, E->getOperatorDelete(), Ptr, DeleteTy);
+  }
 
   EmitBlock(DeleteEnd);
 }
@@ -895,7 +1065,7 @@ llvm::Value * CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
       // FIXME: PointerType->hasAttr<NonNullAttr>()
       bool CanBeZero = false;
       if (UnaryOperator *UO = dyn_cast<UnaryOperator>(subE->IgnoreParens()))
-        if (UO->getOpcode() == UnaryOperator::Deref)
+        if (UO->getOpcode() == UO_Deref)
           CanBeZero = true;
       if (CanBeZero) {
         llvm::BasicBlock *NonZeroBlock = createBasicBlock();
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 0927319db776..79e9dd42ee28 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -347,7 +347,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op,
 
   // FIXME: We should be looking at all of the cast kinds here, not
   // cherry-picking the ones we have test cases for.
-  if (CK == CastExpr::CK_LValueBitCast) {
+  if (CK == CK_LValueBitCast) {
     llvm::Value *V = CGF.EmitLValue(Op).getAddress();
     V = Builder.CreateBitCast(V, 
                       CGF.ConvertType(CGF.getContext().getPointerType(DestTy)));
@@ -532,7 +532,7 @@ EmitCompoundAssign(const CompoundAssignOperator *E,
   // improve codegen a little.  It is possible for the RHS to be complex or
   // scalar.
   OpInfo.Ty = E->getComputationResultType();
-  OpInfo.RHS = EmitCast(CastExpr::CK_Unknown, E->getRHS(), OpInfo.Ty);
+  OpInfo.RHS = EmitCast(CK_Unknown, E->getRHS(), OpInfo.Ty);
   
   LValue LHS = CGF.EmitLValue(E->getLHS());
   // We know the LHS is a complex lvalue.
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index bbd256c4f1fa..9c31c2a3d538 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -13,6 +13,7 @@
 
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
 #include "CGRecordLayout.h"
 #include "clang/AST/APValue.h"
@@ -81,10 +82,6 @@ AppendField(const FieldDecl *Field, uint64_t FieldOffset,
   assert(NextFieldOffsetInBytes <= FieldOffsetInBytes
          && "Field offset mismatch!");
 
-  // Emit the field.
-  if (!InitCst)
-    return false;
-
   unsigned FieldAlignment = getAlignment(InitCst);
 
   // Round up the field offset to the alignment of the field type.
@@ -360,6 +357,9 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
                                      Field->getType(), CGF);
     else
       EltInit = CGM.EmitNullConstant(Field->getType());
+
+    if (!EltInit)
+      return false;
     
     if (!Field->isBitField()) {
       // Handle non-bitfield members.
@@ -455,37 +455,15 @@ public:
     return Visit(E->getInitializer());
   }
     
-  llvm::Constant *EmitMemberFunctionPointer(CXXMethodDecl *MD) {
-    assert(MD->isInstance() && "Member function must not be static!");
-    
-    MD = MD->getCanonicalDecl();
-
-    const llvm::Type *PtrDiffTy = 
-      CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType());
-    
-    llvm::Constant *Values[2];
-
-    Values[0] = CGM.GetCXXMemberFunctionPointerValue(MD);
-    
-    // The adjustment will always be 0.
-    Values[1] = llvm::ConstantInt::get(PtrDiffTy, 0);
-    
-    return llvm::ConstantStruct::get(CGM.getLLVMContext(),
-                                     Values, 2, /*Packed=*/false);
-  }
-
   llvm::Constant *VisitUnaryAddrOf(UnaryOperator *E) {
     if (const MemberPointerType *MPT = 
-        E->getType()->getAs<MemberPointerType>()) {
-      QualType T = MPT->getPointeeType();
+          E->getType()->getAs<MemberPointerType>()) {
       DeclRefExpr *DRE = cast<DeclRefExpr>(E->getSubExpr());
-
       NamedDecl *ND = DRE->getDecl();
-      if (T->isFunctionProtoType())
-        return EmitMemberFunctionPointer(cast<CXXMethodDecl>(ND));
-      
-      // We have a pointer to data member.
-      return CGM.EmitPointerToDataMember(cast<FieldDecl>(ND));
+      if (MPT->isMemberFunctionPointer())
+        return CGM.getCXXABI().EmitMemberPointer(cast<CXXMethodDecl>(ND));
+      else 
+        return CGM.getCXXABI().EmitMemberPointer(cast<FieldDecl>(ND));
     }
 
     return 0;
@@ -514,7 +492,7 @@ public:
     
   llvm::Constant *VisitCastExpr(CastExpr* E) {
     switch (E->getCastKind()) {
-    case CastExpr::CK_ToUnion: {
+    case CK_ToUnion: {
       // GCC cast to union extension
       assert(E->getType()->isUnionType() &&
              "Destination type is not union type!");
@@ -549,44 +527,21 @@ public:
         llvm::StructType::get(C->getType()->getContext(), Types, false);
       return llvm::ConstantStruct::get(STy, Elts);
     }
-    case CastExpr::CK_NullToMemberPointer:
-      return CGM.EmitNullConstant(E->getType());
+    case CK_NullToMemberPointer: {
+      const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>();
+      return CGM.getCXXABI().EmitNullMemberPointer(MPT);
+    }
       
-    case CastExpr::CK_BaseToDerivedMemberPointer: {
+    case CK_BaseToDerivedMemberPointer: {
       Expr *SubExpr = E->getSubExpr();
+      llvm::Constant *C = 
+        CGM.EmitConstantExpr(SubExpr, SubExpr->getType(), CGF);
+      if (!C) return 0;
 
-      const MemberPointerType *SrcTy = 
-        SubExpr->getType()->getAs<MemberPointerType>();
-      const MemberPointerType *DestTy = 
-        E->getType()->getAs<MemberPointerType>();
-      
-      const CXXRecordDecl *DerivedClass =
-        cast<CXXRecordDecl>(cast<RecordType>(DestTy->getClass())->getDecl());
-
-      if (SrcTy->getPointeeType()->isFunctionProtoType()) {
-        llvm::Constant *C = 
-          CGM.EmitConstantExpr(SubExpr, SubExpr->getType(), CGF);
-        if (!C)
-          return 0;
-        
-        llvm::ConstantStruct *CS = cast<llvm::ConstantStruct>(C);
-        
-        // Check if we need to update the adjustment.
-        if (llvm::Constant *Offset = 
-            CGM.GetNonVirtualBaseClassOffset(DerivedClass, E->getBasePath())) {
-          llvm::Constant *Values[2];
-        
-          Values[0] = CS->getOperand(0);
-          Values[1] = llvm::ConstantExpr::getAdd(CS->getOperand(1), Offset);
-          return llvm::ConstantStruct::get(CGM.getLLVMContext(), Values, 2, 
-                                           /*Packed=*/false);
-        }
-        
-        return CS;
-      }          
+      return CGM.getCXXABI().EmitMemberPointerConversion(C, E);
     }
 
-    case CastExpr::CK_BitCast: 
+    case CK_BitCast: 
       // This must be a member function pointer cast.
       return Visit(E->getSubExpr());
 
@@ -792,7 +747,7 @@ public:
     case Expr::DeclRefExprClass: {
       ValueDecl *Decl = cast<DeclRefExpr>(E)->getDecl();
       if (Decl->hasAttr<WeakRefAttr>())
-	return CGM.GetWeakRefReference(Decl);
+        return CGM.GetWeakRefReference(Decl);
       if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Decl))
         return CGM.GetAddrOfFunction(FD);
       if (const VarDecl* VD = dyn_cast<VarDecl>(Decl)) {
@@ -821,7 +776,7 @@ public:
     case Expr::PredefinedExprClass: {
       unsigned Type = cast<PredefinedExpr>(E)->getIdentType();
       if (CGF) {
-        LValue Res = CGF->EmitPredefinedFunctionName(Type);
+        LValue Res = CGF->EmitPredefinedLValue(cast<PredefinedExpr>(E));
         return cast<llvm::Constant>(Res.getAddress());
       } else if (Type == PredefinedExpr::PrettyFunction) {
         return CGM.GetAddrOfConstantCString("top level", ".tmp");
@@ -989,7 +944,7 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
                              uint64_t StartOffset) {
   assert(StartOffset % 8 == 0 && "StartOffset not byte aligned!");
 
-  if (!CGM.getTypes().ContainsPointerToDataMember(T))
+  if (CGM.getTypes().isZeroInitializable(T))
     return;
 
   if (const ConstantArrayType *CAT = 
@@ -1022,7 +977,7 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
         continue;
       
       // Ignore bases that don't have any pointer to data members.
-      if (!CGM.getTypes().ContainsPointerToDataMember(BaseDecl))
+      if (CGM.getTypes().isZeroInitializable(BaseDecl))
         continue;
 
       uint64_t BaseOffset = Layout.getBaseClassOffset(BaseDecl);
@@ -1036,7 +991,7 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
          E = RD->field_end(); I != E; ++I, ++FieldNo) {
       QualType FieldType = I->getType();
       
-      if (!CGM.getTypes().ContainsPointerToDataMember(FieldType))
+      if (CGM.getTypes().isZeroInitializable(FieldType))
         continue;
 
       uint64_t FieldOffset = StartOffset + Layout.getFieldOffset(FieldNo);
@@ -1061,7 +1016,7 @@ FillInNullDataMemberPointers(CodeGenModule &CGM, QualType T,
 }
 
 llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
-  if (!getTypes().ContainsPointerToDataMember(T))
+  if (getTypes().isZeroInitializable(T))
     return llvm::Constant::getNullValue(getTypes().ConvertTypeForMem(T));
     
   if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(T)) {
@@ -1105,7 +1060,7 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
         continue;
 
       // Ignore bases that don't have any pointer to data members.
-      if (!getTypes().ContainsPointerToDataMember(BaseDecl))
+      if (getTypes().isZeroInitializable(BaseDecl))
         continue;
 
       // Currently, all bases are arrays of i8. Figure out how many elements
@@ -1165,29 +1120,3 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
   return llvm::ConstantInt::get(getTypes().ConvertTypeForMem(T), -1ULL, 
                                 /*isSigned=*/true);
 }
-
-llvm::Constant *
-CodeGenModule::EmitPointerToDataMember(const FieldDecl *FD) {
-
-  // Itanium C++ ABI 2.3:
-  //   A pointer to data member is an offset from the base address of the class
-  //   object containing it, represented as a ptrdiff_t
-
-  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(FD->getParent());
-  QualType ClassType = 
-    getContext().getTypeDeclType(const_cast<CXXRecordDecl *>(ClassDecl));
-  
-  const llvm::StructType *ClassLTy =
-    cast<llvm::StructType>(getTypes().ConvertType(ClassType));
-
-  const CGRecordLayout &RL =
-    getTypes().getCGRecordLayout(FD->getParent());
-  unsigned FieldNo = RL.getLLVMFieldNo(FD);
-  uint64_t Offset = 
-    getTargetData().getStructLayout(ClassLTy)->getElementOffset(FieldNo);
-
-  const llvm::Type *PtrDiffTy = 
-    getTypes().ConvertType(getContext().getPointerDiffType());
-
-  return llvm::ConstantInt::get(PtrDiffTy, Offset);
-}
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index ef38209e1eaf..2318cc4e9aeb 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenFunction.h"
+#include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
 #include "CodeGenModule.h"
 #include "clang/AST/ASTContext.h"
@@ -137,7 +138,7 @@ public:
                                   CGF.getContext().typesAreCompatible(
                                     E->getArgType1(), E->getArgType2()));
   }
-  Value *VisitOffsetOfExpr(const OffsetOfExpr *E);
+  Value *VisitOffsetOfExpr(OffsetOfExpr *E);
   Value *VisitSizeOfAlignOfExpr(const SizeOfAlignOfExpr *E);
   Value *VisitAddrLabelExpr(const AddrLabelExpr *E) {
     llvm::Value *V = CGF.GetAddrOfLabel(E->getLabel());
@@ -149,7 +150,10 @@ public:
     Expr::EvalResult Result;
     if (E->Evaluate(Result, CGF.getContext()) && Result.Val.isInt()) {
       assert(!Result.HasSideEffects && "Constant declref with side-effect?!");
-      return llvm::ConstantInt::get(VMContext, Result.Val.getInt());
+      llvm::ConstantInt *CI 
+        = llvm::ConstantInt::get(VMContext, Result.Val.getInt());
+      CGF.EmitDeclRefExprDbgValue(E, CI);
+      return CI;
     }
     return EmitLoadOfLValue(E);
   }
@@ -235,6 +239,9 @@ public:
 
     
   Value *VisitUnaryAddrOf(const UnaryOperator *E) {
+    // If the sub-expression is an instance member reference,
+    // EmitDeclRefLValue will magically emit it with the appropriate
+    // value as the "address".
     return EmitLValue(E->getSubExpr()).getAddress();
   }
   Value *VisitUnaryDeref(const Expr *E) { return EmitLoadOfLValue(E); }
@@ -251,7 +258,6 @@ public:
   Value *VisitUnaryExtension(const UnaryOperator *E) {
     return Visit(E->getSubExpr());
   }
-  Value *VisitUnaryOffsetOf(const UnaryOperator *E);
     
   // C++
   Value *VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
@@ -297,7 +303,7 @@ public:
 
   // Binary Operators.
   Value *EmitMul(const BinOpInfo &Ops) {
-    if (Ops.Ty->isSignedIntegerType()) {
+    if (Ops.Ty->hasSignedIntegerRepresentation()) {
       switch (CGF.getContext().getLangOptions().getSignedOverflowBehavior()) {
       case LangOptions::SOB_Undefined:
         return Builder.CreateNSWMul(Ops.LHS, Ops.RHS, "mul");
@@ -409,11 +415,8 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
     return Builder.CreateFCmpUNE(Src, Zero, "tobool");
   }
 
-  if (SrcType->isMemberPointerType()) {
-    // Compare against -1.
-    llvm::Value *NegativeOne = llvm::Constant::getAllOnesValue(Src->getType());
-    return Builder.CreateICmpNE(Src, NegativeOne, "tobool");
-  }
+  if (const MemberPointerType *MPT = dyn_cast<MemberPointerType>(SrcType))
+    return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, Src, MPT);
 
   assert((SrcType->isIntegerType() || isa<llvm::PointerType>(Src->getType())) &&
          "Unknown scalar type to convert");
@@ -562,17 +565,10 @@ EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src,
 }
 
 Value *ScalarExprEmitter::EmitNullValue(QualType Ty) {
-  const llvm::Type *LTy = ConvertType(Ty);
-  
-  if (!Ty->isMemberPointerType())
-    return llvm::Constant::getNullValue(LTy);
-  
-  assert(!Ty->isMemberFunctionPointerType() &&
-         "member function pointers are not scalar!");
+  if (const MemberPointerType *MPT = Ty->getAs<MemberPointerType>())
+    return CGF.CGM.getCXXABI().EmitNullMemberPointer(MPT);
 
-  // Itanium C++ ABI 2.3:
-  //   A NULL pointer is represented as -1.
-  return llvm::ConstantInt::get(LTy, -1ULL, /*isSigned=*/true);  
+  return llvm::Constant::getNullValue(ConvertType(Ty));
 }
 
 //===----------------------------------------------------------------------===//
@@ -888,7 +884,7 @@ Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
 static bool ShouldNullCheckClassCastValue(const CastExpr *CE) {
   const Expr *E = CE->getSubExpr();
 
-  if (CE->getCastKind() == CastExpr::CK_UncheckedDerivedToBase)
+  if (CE->getCastKind() == CK_UncheckedDerivedToBase)
     return false;
   
   if (isa<CXXThisExpr>(E)) {
@@ -897,8 +893,8 @@ static bool ShouldNullCheckClassCastValue(const CastExpr *CE) {
   }
   
   if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(CE)) {
-    // And that lvalue casts are never null.
-    if (ICE->isLvalueCast())
+    // And that glvalue casts are never null.
+    if (ICE->getValueKind() != VK_RValue)
       return false;
   }
 
@@ -911,7 +907,7 @@ static bool ShouldNullCheckClassCastValue(const CastExpr *CE) {
 Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
   Expr *E = CE->getSubExpr();
   QualType DestTy = CE->getType();
-  CastExpr::CastKind Kind = CE->getCastKind();
+  CastKind Kind = CE->getCastKind();
   
   if (!DestTy->isVoidType())
     TestAndClearIgnoreResultAssign();
@@ -920,59 +916,58 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
   // a default case, so the compiler will warn on a missing case.  The cases
   // are in the same order as in the CastKind enum.
   switch (Kind) {
-  case CastExpr::CK_Unknown:
+  case CK_Unknown:
     // FIXME: All casts should have a known kind!
     //assert(0 && "Unknown cast kind!");
     break;
 
-  case CastExpr::CK_LValueBitCast: {
+  case CK_LValueBitCast: 
+  case CK_ObjCObjectLValueCast: {
     Value *V = EmitLValue(E).getAddress();
     V = Builder.CreateBitCast(V, 
                           ConvertType(CGF.getContext().getPointerType(DestTy)));
-    // FIXME: Are the qualifiers correct here?
-    return EmitLoadOfLValue(LValue::MakeAddr(V, CGF.MakeQualifiers(DestTy)), 
-                            DestTy);
+    return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy), DestTy);
   }
       
-  case CastExpr::CK_AnyPointerToObjCPointerCast:
-  case CastExpr::CK_AnyPointerToBlockPointerCast:
-  case CastExpr::CK_BitCast: {
+  case CK_AnyPointerToObjCPointerCast:
+  case CK_AnyPointerToBlockPointerCast:
+  case CK_BitCast: {
     Value *Src = Visit(const_cast<Expr*>(E));
     return Builder.CreateBitCast(Src, ConvertType(DestTy));
   }
-  case CastExpr::CK_NoOp:
-  case CastExpr::CK_UserDefinedConversion:
+  case CK_NoOp:
+  case CK_UserDefinedConversion:
     return Visit(const_cast<Expr*>(E));
 
-  case CastExpr::CK_BaseToDerived: {
+  case CK_BaseToDerived: {
     const CXXRecordDecl *DerivedClassDecl = 
       DestTy->getCXXRecordDeclForPointerType();
     
     return CGF.GetAddressOfDerivedClass(Visit(E), DerivedClassDecl, 
-                                        CE->getBasePath(), 
+                                        CE->path_begin(), CE->path_end(),
                                         ShouldNullCheckClassCastValue(CE));
   }
-  case CastExpr::CK_UncheckedDerivedToBase:
-  case CastExpr::CK_DerivedToBase: {
+  case CK_UncheckedDerivedToBase:
+  case CK_DerivedToBase: {
     const RecordType *DerivedClassTy = 
       E->getType()->getAs<PointerType>()->getPointeeType()->getAs<RecordType>();
     CXXRecordDecl *DerivedClassDecl = 
       cast<CXXRecordDecl>(DerivedClassTy->getDecl());
 
     return CGF.GetAddressOfBaseClass(Visit(E), DerivedClassDecl, 
-                                     CE->getBasePath(),
+                                     CE->path_begin(), CE->path_end(),
                                      ShouldNullCheckClassCastValue(CE));
   }
-  case CastExpr::CK_Dynamic: {
+  case CK_Dynamic: {
     Value *V = Visit(const_cast<Expr*>(E));
     const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(CE);
     return CGF.EmitDynamicCast(V, DCE);
   }
-  case CastExpr::CK_ToUnion:
+  case CK_ToUnion:
     assert(0 && "Should be unreachable!");
     break;
 
-  case CastExpr::CK_ArrayToPointerDecay: {
+  case CK_ArrayToPointerDecay: {
     assert(E->getType()->isArrayType() &&
            "Array to pointer decay must have array source type!");
 
@@ -990,62 +985,66 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
 
     return V;
   }
-  case CastExpr::CK_FunctionToPointerDecay:
+  case CK_FunctionToPointerDecay:
     return EmitLValue(E).getAddress();
 
-  case CastExpr::CK_NullToMemberPointer:
-    return CGF.CGM.EmitNullConstant(DestTy);
+  case CK_NullToMemberPointer: {
+    // If the subexpression's type is the C++0x nullptr_t, emit the
+    // subexpression, which may have side effects.
+    if (E->getType()->isNullPtrType())
+      (void) Visit(E);
 
-  case CastExpr::CK_BaseToDerivedMemberPointer:
-  case CastExpr::CK_DerivedToBaseMemberPointer: {
-    Value *Src = Visit(E);
+    const MemberPointerType *MPT = CE->getType()->getAs<MemberPointerType>();
+    return CGF.CGM.getCXXABI().EmitNullMemberPointer(MPT);
+  }
 
-    // See if we need to adjust the pointer.
-    const CXXRecordDecl *BaseDecl = 
-      cast<CXXRecordDecl>(E->getType()->getAs<MemberPointerType>()->
-                          getClass()->getAs<RecordType>()->getDecl());
-    const CXXRecordDecl *DerivedDecl = 
-      cast<CXXRecordDecl>(CE->getType()->getAs<MemberPointerType>()->
-                          getClass()->getAs<RecordType>()->getDecl());
-    if (CE->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
-      std::swap(DerivedDecl, BaseDecl);
-
-    if (llvm::Constant *Adj = 
-          CGF.CGM.GetNonVirtualBaseClassOffset(DerivedDecl, CE->getBasePath())){
-      if (CE->getCastKind() == CastExpr::CK_DerivedToBaseMemberPointer)
-        Src = Builder.CreateNSWSub(Src, Adj, "adj");
-      else
-        Src = Builder.CreateNSWAdd(Src, Adj, "adj");
-    }
+  case CK_BaseToDerivedMemberPointer:
+  case CK_DerivedToBaseMemberPointer: {
+    Value *Src = Visit(E);
     
-    return Src;
+    // Note that the AST doesn't distinguish between checked and
+    // unchecked member pointer conversions, so we always have to
+    // implement checked conversions here.  This is inefficient when
+    // actual control flow may be required in order to perform the
+    // check, which it is for data member pointers (but not member
+    // function pointers on Itanium and ARM).
+    return CGF.CGM.getCXXABI().EmitMemberPointerConversion(CGF, CE, Src);
   }
+  
 
-  case CastExpr::CK_ConstructorConversion:
+  case CK_ConstructorConversion:
     assert(0 && "Should be unreachable!");
     break;
 
-  case CastExpr::CK_IntegralToPointer: {
+  case CK_IntegralToPointer: {
     Value *Src = Visit(const_cast<Expr*>(E));
-    
+
     // First, convert to the correct width so that we control the kind of
     // extension.
     const llvm::Type *MiddleTy = CGF.IntPtrTy;
     bool InputSigned = E->getType()->isSignedIntegerType();
     llvm::Value* IntResult =
       Builder.CreateIntCast(Src, MiddleTy, InputSigned, "conv");
-    
+
     return Builder.CreateIntToPtr(IntResult, ConvertType(DestTy));
   }
-  case CastExpr::CK_PointerToIntegral: {
+  case CK_PointerToIntegral: {
     Value *Src = Visit(const_cast<Expr*>(E));
+
+    // Handle conversion to bool correctly.
+    if (DestTy->isBooleanType())
+      return EmitScalarConversion(Src, E->getType(), DestTy);
+
     return Builder.CreatePtrToInt(Src, ConvertType(DestTy));
   }
-  case CastExpr::CK_ToVoid: {
-    CGF.EmitAnyExpr(E, 0, false, true);
+  case CK_ToVoid: {
+    if (E->Classify(CGF.getContext()).isGLValue())
+      CGF.EmitLValue(E);
+    else
+      CGF.EmitAnyExpr(E, 0, false, true);
     return 0;
   }
-  case CastExpr::CK_VectorSplat: {
+  case CK_VectorSplat: {
     const llvm::Type *DstTy = ConvertType(DestTy);
     Value *Elt = Visit(const_cast<Expr*>(E));
 
@@ -1064,16 +1063,19 @@ Value *ScalarExprEmitter::EmitCastExpr(CastExpr *CE) {
     llvm::Value *Yay = Builder.CreateShuffleVector(UnV, UnV, Mask, "splat");
     return Yay;
   }
-  case CastExpr::CK_IntegralCast:
-  case CastExpr::CK_IntegralToFloating:
-  case CastExpr::CK_FloatingToIntegral:
-  case CastExpr::CK_FloatingCast:
+  case CK_IntegralCast:
+  case CK_IntegralToFloating:
+  case CK_FloatingToIntegral:
+  case CK_FloatingCast:
     return EmitScalarConversion(Visit(E), E->getType(), DestTy);
 
-  case CastExpr::CK_MemberPointerToBoolean:
-    return CGF.EvaluateExprAsBool(E);
+  case CK_MemberPointerToBoolean: {
+    llvm::Value *MemPtr = Visit(E);
+    const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>();
+    return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, MemPtr, MPT);
   }
-
+  }
+  
   // Handle cases where the source is an non-complex type.
 
   if (!CGF.hasAggregateLLVMType(E->getType())) {
@@ -1116,7 +1118,7 @@ Value *ScalarExprEmitter::VisitBlockDeclRefExpr(const BlockDeclRefExpr *E) {
   llvm::Value *V = CGF.GetAddrOfBlockDecl(E);
   if (E->getType().isObjCGCWeak())
     return CGF.CGM.getObjCRuntime().EmitObjCWeakRead(CGF, V);
-  return Builder.CreateLoad(V, "tmp");
+  return CGF.EmitLoadOfScalar(V, false, 0, E->getType());
 }
 
 //===----------------------------------------------------------------------===//
@@ -1156,7 +1158,7 @@ EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
         NextVal = Builder.CreateGEP(InVal, Inc, "add.ptr");
         llvm::Value *lhs = LV.getAddress();
         lhs = Builder.CreateBitCast(lhs, llvm::PointerType::getUnqual(i8Ty));
-        LV = LValue::MakeAddr(lhs, CGF.MakeQualifiers(ValTy));
+        LV = CGF.MakeAddrLValue(lhs, ValTy);
       } else
         NextVal = Builder.CreateInBoundsGEP(InVal, Inc, "ptrincdec");
     } else {
@@ -1191,9 +1193,10 @@ EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
         BinOp.LHS = InVal;
         BinOp.RHS = NextVal;
         BinOp.Ty = E->getType();
-        BinOp.Opcode = BinaryOperator::Add;
+        BinOp.Opcode = BO_Add;
         BinOp.E = E;
-        return EmitOverflowCheckedBinOp(BinOp);
+        NextVal = EmitOverflowCheckedBinOp(BinOp);
+        break;
       }
     }
   } else {
@@ -1240,7 +1243,7 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
   else 
     BinOp.LHS = llvm::Constant::getNullValue(BinOp.RHS->getType());
   BinOp.Ty = E->getType();
-  BinOp.Opcode = BinaryOperator::Sub;
+  BinOp.Opcode = BO_Sub;
   BinOp.E = E;
   return EmitSub(BinOp);
 }
@@ -1264,19 +1267,94 @@ Value *ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *E) {
   return Builder.CreateZExt(BoolVal, ConvertType(E->getType()), "lnot.ext");
 }
 
-Value *ScalarExprEmitter::VisitOffsetOfExpr(const OffsetOfExpr *E) {
-  Expr::EvalResult Result;
-  if(E->Evaluate(Result, CGF.getContext()))
-    return llvm::ConstantInt::get(VMContext, Result.Val.getInt());
-  
-  // FIXME: Cannot support code generation for non-constant offsetof.
-  unsigned DiagID = CGF.CGM.getDiags().getCustomDiagID(Diagnostic::Error,
-                             "cannot compile non-constant __builtin_offsetof");
-  CGF.CGM.getDiags().Report(CGF.getContext().getFullLoc(E->getLocStart()), 
-                            DiagID)
-    << E->getSourceRange();
-  
-  return llvm::Constant::getNullValue(ConvertType(E->getType()));
+Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
+  // Try folding the offsetof to a constant.
+  Expr::EvalResult EvalResult;
+  if (E->Evaluate(EvalResult, CGF.getContext()))
+    return llvm::ConstantInt::get(VMContext, EvalResult.Val.getInt());
+
+  // Loop over the components of the offsetof to compute the value.
+  unsigned n = E->getNumComponents();
+  const llvm::Type* ResultType = ConvertType(E->getType());
+  llvm::Value* Result = llvm::Constant::getNullValue(ResultType);
+  QualType CurrentType = E->getTypeSourceInfo()->getType();
+  for (unsigned i = 0; i != n; ++i) {
+    OffsetOfExpr::OffsetOfNode ON = E->getComponent(i);
+    llvm::Value *Offset = 0;
+    switch (ON.getKind()) {
+    case OffsetOfExpr::OffsetOfNode::Array: {
+      // Compute the index
+      Expr *IdxExpr = E->getIndexExpr(ON.getArrayExprIndex());
+      llvm::Value* Idx = CGF.EmitScalarExpr(IdxExpr);
+      bool IdxSigned = IdxExpr->getType()->isSignedIntegerType();
+      Idx = Builder.CreateIntCast(Idx, ResultType, IdxSigned, "conv");
+
+      // Save the element type
+      CurrentType =
+          CGF.getContext().getAsArrayType(CurrentType)->getElementType();
+
+      // Compute the element size
+      llvm::Value* ElemSize = llvm::ConstantInt::get(ResultType,
+          CGF.getContext().getTypeSizeInChars(CurrentType).getQuantity());
+
+      // Multiply out to compute the result
+      Offset = Builder.CreateMul(Idx, ElemSize);
+      break;
+    }
+
+    case OffsetOfExpr::OffsetOfNode::Field: {
+      FieldDecl *MemberDecl = ON.getField();
+      RecordDecl *RD = CurrentType->getAs<RecordType>()->getDecl();
+      const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD);
+
+      // Compute the index of the field in its parent.
+      unsigned i = 0;
+      // FIXME: It would be nice if we didn't have to loop here!
+      for (RecordDecl::field_iterator Field = RD->field_begin(),
+                                      FieldEnd = RD->field_end();
+           Field != FieldEnd; (void)++Field, ++i) {
+        if (*Field == MemberDecl)
+          break;
+      }
+      assert(i < RL.getFieldCount() && "offsetof field in wrong type");
+
+      // Compute the offset to the field
+      int64_t OffsetInt = RL.getFieldOffset(i) /
+                          CGF.getContext().getCharWidth();
+      Offset = llvm::ConstantInt::get(ResultType, OffsetInt);
+
+      // Save the element type.
+      CurrentType = MemberDecl->getType();
+      break;
+    }
+
+    case OffsetOfExpr::OffsetOfNode::Identifier:
+      llvm_unreachable("dependent __builtin_offsetof");
+
+    case OffsetOfExpr::OffsetOfNode::Base: {
+      if (ON.getBase()->isVirtual()) {
+        CGF.ErrorUnsupported(E, "virtual base in offsetof");
+        continue;
+      }
+
+      RecordDecl *RD = CurrentType->getAs<RecordType>()->getDecl();
+      const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD);
+
+      // Save the element type.
+      CurrentType = ON.getBase()->getType();
+      
+      // Compute the offset to the base.
+      const RecordType *BaseRT = CurrentType->getAs<RecordType>();
+      CXXRecordDecl *BaseRD = cast<CXXRecordDecl>(BaseRT->getDecl());
+      int64_t OffsetInt = RL.getBaseClassOffset(BaseRD) /
+                          CGF.getContext().getCharWidth();
+      Offset = llvm::ConstantInt::get(ResultType, OffsetInt);
+      break;
+    }
+    }
+    Result = Builder.CreateAdd(Result, Offset);
+  }
+  return Result;
 }
 
 /// VisitSizeOfAlignOfExpr - Return the size or alignment of the type of
@@ -1327,12 +1405,6 @@ Value *ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *E) {
   return llvm::Constant::getNullValue(ConvertType(E->getType()));
 }
 
-Value *ScalarExprEmitter::VisitUnaryOffsetOf(const UnaryOperator *E) {
-  Value* ResultAsPtr = EmitLValue(E->getSubExpr()).getAddress();
-  const llvm::Type* ResultType = ConvertType(E->getType());
-  return Builder.CreatePtrToInt(ResultAsPtr, ResultType, "offsetof");
-}
-
 //===----------------------------------------------------------------------===//
 //                           Binary Operators
 //===----------------------------------------------------------------------===//
@@ -1422,7 +1494,7 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
 Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
   if (Ops.LHS->getType()->isFPOrFPVectorTy())
     return Builder.CreateFDiv(Ops.LHS, Ops.RHS, "div");
-  else if (Ops.Ty->isUnsignedIntegerType())
+  else if (Ops.Ty->hasUnsignedIntegerRepresentation())
     return Builder.CreateUDiv(Ops.LHS, Ops.RHS, "div");
   else
     return Builder.CreateSDiv(Ops.LHS, Ops.RHS, "div");
@@ -1441,18 +1513,18 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   unsigned OpID = 0;
 
   switch (Ops.Opcode) {
-  case BinaryOperator::Add:
-  case BinaryOperator::AddAssign:
+  case BO_Add:
+  case BO_AddAssign:
     OpID = 1;
     IID = llvm::Intrinsic::sadd_with_overflow;
     break;
-  case BinaryOperator::Sub:
-  case BinaryOperator::SubAssign:
+  case BO_Sub:
+  case BO_SubAssign:
     OpID = 2;
     IID = llvm::Intrinsic::ssub_with_overflow;
     break;
-  case BinaryOperator::Mul:
-  case BinaryOperator::MulAssign:
+  case BO_Mul:
+  case BO_MulAssign:
     OpID = 3;
     IID = llvm::Intrinsic::smul_with_overflow;
     break;
@@ -1472,58 +1544,26 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   Value *overflow = Builder.CreateExtractValue(resultAndOverflow, 1);
 
   // Branch in case of overflow.
-  llvm::BasicBlock *initialBB = Builder.GetInsertBlock();
-  llvm::BasicBlock *overflowBB =
-    CGF.createBasicBlock("overflow", CGF.CurFn);
-  llvm::BasicBlock *continueBB =
-    CGF.createBasicBlock("overflow.continue", CGF.CurFn);
+  llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn);
+  llvm::BasicBlock *continueBB = CGF.createBasicBlock("nooverflow", CGF.CurFn);
 
   Builder.CreateCondBr(overflow, overflowBB, continueBB);
 
-  // Handle overflow
-
+  // Handle overflow with llvm.trap.
+  // TODO: it would be better to generate one of these blocks per function.
   Builder.SetInsertPoint(overflowBB);
-
-  // Handler is:
-  // long long *__overflow_handler)(long long a, long long b, char op,
-  // char width)
-  std::vector<const llvm::Type*> handerArgTypes;
-  handerArgTypes.push_back(CGF.Int64Ty);
-  handerArgTypes.push_back(CGF.Int64Ty);
-  handerArgTypes.push_back(llvm::Type::getInt8Ty(VMContext));
-  handerArgTypes.push_back(llvm::Type::getInt8Ty(VMContext));
-  llvm::FunctionType *handlerTy =
-    llvm::FunctionType::get(CGF.Int64Ty, handerArgTypes, false);
-  llvm::Value *handlerFunction =
-    CGF.CGM.getModule().getOrInsertGlobal("__overflow_handler",
-        llvm::PointerType::getUnqual(handlerTy));
-  handlerFunction = Builder.CreateLoad(handlerFunction);
-
-  llvm::Value *handlerResult = Builder.CreateCall4(handlerFunction,
-      Builder.CreateSExt(Ops.LHS, CGF.Int64Ty),
-      Builder.CreateSExt(Ops.RHS, CGF.Int64Ty),
-      llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext), OpID),
-      llvm::ConstantInt::get(llvm::Type::getInt8Ty(VMContext),
-        cast<llvm::IntegerType>(opTy)->getBitWidth()));
-
-  handlerResult = Builder.CreateTrunc(handlerResult, opTy);
-
-  Builder.CreateBr(continueBB);
-
-  // Set up the continuation
+  llvm::Function *Trap = CGF.CGM.getIntrinsic(llvm::Intrinsic::trap);
+  Builder.CreateCall(Trap);
+  Builder.CreateUnreachable();
+  
+  // Continue on.
   Builder.SetInsertPoint(continueBB);
-  // Get the correct result
-  llvm::PHINode *phi = Builder.CreatePHI(opTy);
-  phi->reserveOperandSpace(2);
-  phi->addIncoming(result, initialBB);
-  phi->addIncoming(handlerResult, overflowBB);
-
-  return phi;
+  return result;
 }
 
 Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &Ops) {
   if (!Ops.Ty->isAnyPointerType()) {
-    if (Ops.Ty->isSignedIntegerType()) {
+    if (Ops.Ty->hasSignedIntegerRepresentation()) {
       switch (CGF.getContext().getLangOptions().getSignedOverflowBehavior()) {
       case LangOptions::SOB_Undefined:
         return Builder.CreateNSWAdd(Ops.LHS, Ops.RHS, "add");
@@ -1606,7 +1646,7 @@ Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &Ops) {
 
 Value *ScalarExprEmitter::EmitSub(const BinOpInfo &Ops) {
   if (!isa<llvm::PointerType>(Ops.LHS->getType())) {
-    if (Ops.Ty->isSignedIntegerType()) {
+    if (Ops.Ty->hasSignedIntegerRepresentation()) {
       switch (CGF.getContext().getLangOptions().getSignedOverflowBehavior()) {
       case LangOptions::SOB_Undefined:
         return Builder.CreateNSWSub(Ops.LHS, Ops.RHS, "sub");
@@ -1747,7 +1787,7 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) {
     CGF.EmitBlock(Cont);
   }
 
-  if (Ops.Ty->isUnsignedIntegerType())
+  if (Ops.Ty->hasUnsignedIntegerRepresentation())
     return Builder.CreateLShr(Ops.LHS, RHS, "shr");
   return Builder.CreateAShr(Ops.LHS, RHS, "shr");
 }
@@ -1757,33 +1797,13 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
   TestAndClearIgnoreResultAssign();
   Value *Result;
   QualType LHSTy = E->getLHS()->getType();
-  if (LHSTy->isMemberFunctionPointerType()) {
-    Value *LHSPtr = CGF.EmitAnyExprToTemp(E->getLHS()).getAggregateAddr();
-    Value *RHSPtr = CGF.EmitAnyExprToTemp(E->getRHS()).getAggregateAddr();
-    llvm::Value *LHSFunc = Builder.CreateStructGEP(LHSPtr, 0);
-    LHSFunc = Builder.CreateLoad(LHSFunc);
-    llvm::Value *RHSFunc = Builder.CreateStructGEP(RHSPtr, 0);
-    RHSFunc = Builder.CreateLoad(RHSFunc);
-    Value *ResultF = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc,
-                                        LHSFunc, RHSFunc, "cmp.func");
-    Value *NullPtr = llvm::Constant::getNullValue(LHSFunc->getType());
-    Value *ResultNull = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc,
-                                           LHSFunc, NullPtr, "cmp.null");
-    llvm::Value *LHSAdj = Builder.CreateStructGEP(LHSPtr, 1);
-    LHSAdj = Builder.CreateLoad(LHSAdj);
-    llvm::Value *RHSAdj = Builder.CreateStructGEP(RHSPtr, 1);
-    RHSAdj = Builder.CreateLoad(RHSAdj);
-    Value *ResultA = Builder.CreateICmp((llvm::ICmpInst::Predicate)UICmpOpc,
-                                        LHSAdj, RHSAdj, "cmp.adj");
-    if (E->getOpcode() == BinaryOperator::EQ) {
-      Result = Builder.CreateOr(ResultNull, ResultA, "or.na");
-      Result = Builder.CreateAnd(Result, ResultF, "and.f");
-    } else {
-      assert(E->getOpcode() == BinaryOperator::NE &&
-             "Member pointer comparison other than == or != ?");
-      Result = Builder.CreateAnd(ResultNull, ResultA, "and.na");
-      Result = Builder.CreateOr(Result, ResultF, "or.f");
-    }
+  if (const MemberPointerType *MPT = LHSTy->getAs<MemberPointerType>()) {
+    assert(E->getOpcode() == BO_EQ ||
+           E->getOpcode() == BO_NE);
+    Value *LHS = CGF.EmitScalarExpr(E->getLHS());
+    Value *RHS = CGF.EmitScalarExpr(E->getRHS());
+    Result = CGF.CGM.getCXXABI().EmitMemberPointerComparison(
+                   CGF, LHS, RHS, MPT, E->getOpcode() == BO_NE);
   } else if (!LHSTy->isAnyComplexType()) {
     Value *LHS = Visit(E->getLHS());
     Value *RHS = Visit(E->getRHS());
@@ -1791,7 +1811,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
     if (LHS->getType()->isFPOrFPVectorTy()) {
       Result = Builder.CreateFCmp((llvm::CmpInst::Predicate)FCmpOpc,
                                   LHS, RHS, "cmp");
-    } else if (LHSTy->isSignedIntegerType()) {
+    } else if (LHSTy->hasSignedIntegerRepresentation()) {
       Result = Builder.CreateICmp((llvm::ICmpInst::Predicate)SICmpOpc,
                                   LHS, RHS, "cmp");
     } else {
@@ -1827,10 +1847,10 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,unsigned UICmpOpc,
                                    LHS.second, RHS.second, "cmp.i");
     }
 
-    if (E->getOpcode() == BinaryOperator::EQ) {
+    if (E->getOpcode() == BO_EQ) {
       Result = Builder.CreateAnd(ResultR, ResultI, "and.ri");
     } else {
-      assert(E->getOpcode() == BinaryOperator::NE &&
+      assert(E->getOpcode() == BO_NE &&
              "Complex comparison other than == or != ?");
       Result = Builder.CreateOr(ResultR, ResultI, "or.ri");
     }
@@ -2044,7 +2064,12 @@ VisitConditionalOperator(const ConditionalOperator *E) {
     return Builder.CreateSelect(CondV, LHS, RHS, "cond");
   }
 
-
+  if (!E->getLHS() && CGF.getContext().getLangOptions().CPlusPlus) {
+    // Does not support GNU missing condition extension in C++ yet (see #7726)
+    CGF.ErrorUnsupported(E, "conditional operator with missing LHS");
+    return llvm::UndefValue::get(ConvertType(E->getType()));
+  }
+  
   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
@@ -2188,21 +2213,19 @@ LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
     V = CreateTempAlloca(ClassPtrTy, "resval");
     llvm::Value *Src = EmitScalarExpr(BaseExpr);
     Builder.CreateStore(Src, V);
-    LValue LV = LValue::MakeAddr(V, MakeQualifiers(E->getType()));
-    V = ScalarExprEmitter(*this).EmitLoadOfLValue(LV, E->getType());
-  }
-  else {
-      if (E->isArrow())
-        V = ScalarExprEmitter(*this).EmitLoadOfLValue(BaseExpr);
-      else
-        V  = EmitLValue(BaseExpr).getAddress();
+    V = ScalarExprEmitter(*this).EmitLoadOfLValue(
+      MakeAddrLValue(V, E->getType()), E->getType());
+  } else {
+    if (E->isArrow())
+      V = ScalarExprEmitter(*this).EmitLoadOfLValue(BaseExpr);
+    else
+      V = EmitLValue(BaseExpr).getAddress();
   }
   
   // build Class* type
   ClassPtrTy = ClassPtrTy->getPointerTo();
   V = Builder.CreateBitCast(V, ClassPtrTy);
-  LValue LV = LValue::MakeAddr(V, MakeQualifiers(E->getType()));
-  return LV;
+  return MakeAddrLValue(V, E->getType());
 }
 
 
@@ -2212,7 +2235,7 @@ LValue CodeGenFunction::EmitCompoundAssignOperatorLValue(
   Value *Result = 0;
   switch (E->getOpcode()) {
 #define COMPOUND_OP(Op)                                                       \
-    case BinaryOperator::Op##Assign:                                          \
+    case BO_##Op##Assign:                                                     \
       return Scalar.EmitCompoundAssignLValue(E, &ScalarExprEmitter::Emit##Op, \
                                              Result)
   COMPOUND_OP(Mul);
@@ -2227,28 +2250,28 @@ LValue CodeGenFunction::EmitCompoundAssignOperatorLValue(
   COMPOUND_OP(Or);
 #undef COMPOUND_OP
       
-  case BinaryOperator::PtrMemD:
-  case BinaryOperator::PtrMemI:
-  case BinaryOperator::Mul:
-  case BinaryOperator::Div:
-  case BinaryOperator::Rem:
-  case BinaryOperator::Add:
-  case BinaryOperator::Sub:
-  case BinaryOperator::Shl:
-  case BinaryOperator::Shr:
-  case BinaryOperator::LT:
-  case BinaryOperator::GT:
-  case BinaryOperator::LE:
-  case BinaryOperator::GE:
-  case BinaryOperator::EQ:
-  case BinaryOperator::NE:
-  case BinaryOperator::And:
-  case BinaryOperator::Xor:
-  case BinaryOperator::Or:
-  case BinaryOperator::LAnd:
-  case BinaryOperator::LOr:
-  case BinaryOperator::Assign:
-  case BinaryOperator::Comma:
+  case BO_PtrMemD:
+  case BO_PtrMemI:
+  case BO_Mul:
+  case BO_Div:
+  case BO_Rem:
+  case BO_Add:
+  case BO_Sub:
+  case BO_Shl:
+  case BO_Shr:
+  case BO_LT:
+  case BO_GT:
+  case BO_LE:
+  case BO_GE:
+  case BO_EQ:
+  case BO_NE:
+  case BO_And:
+  case BO_Xor:
+  case BO_Or:
+  case BO_LAnd:
+  case BO_LOr:
+  case BO_Assign:
+  case BO_Comma:
     assert(false && "Not valid compound assignment operators");
     break;
   }
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index e735a617546c..6a6d63df8f52 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -403,13 +403,14 @@ void CodeGenFunction::GenerateObjCSetter(ObjCImplementationDecl *IMP,
     // Objective-C pointer types, we can always bit cast the RHS in these cases.
     if (getContext().getCanonicalType(Ivar->getType()) !=
         getContext().getCanonicalType(ArgDecl->getType())) {
-      ImplicitCastExpr ArgCasted(Ivar->getType(), CastExpr::CK_BitCast, &Arg,
-                                 CXXBaseSpecifierArray(), false);
-      BinaryOperator Assign(&IvarRef, &ArgCasted, BinaryOperator::Assign,
+      ImplicitCastExpr ArgCasted(ImplicitCastExpr::OnStack,
+                                 Ivar->getType(), CK_BitCast, &Arg,
+                                 VK_RValue);
+      BinaryOperator Assign(&IvarRef, &ArgCasted, BO_Assign,
                             Ivar->getType(), Loc);
       EmitStmt(&Assign);
     } else {
-      BinaryOperator Assign(&IvarRef, &Arg, BinaryOperator::Assign,
+      BinaryOperator Assign(&IvarRef, &Arg, BO_Assign,
                             Ivar->getType(), Loc);
       EmitStmt(&Assign);
     }
@@ -571,7 +572,7 @@ void CodeGenFunction::EmitObjCSuperPropertySet(const Expr *Exp,
   Args.push_back(std::make_pair(Src, Exp->getType()));
   CGM.getObjCRuntime().GenerateMessageSendSuper(*this,
                                                 ReturnValueSlot(),
-                                                Exp->getType(),
+                                                getContext().VoidTy,
                                                 S,
                                                 OMD->getClassInterface(),
                                                 isCategoryImpl,
@@ -792,7 +793,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
 
   BreakContinueStack.pop_back();
 
-  EmitBlock(AfterBody.Block);
+  EmitBlock(AfterBody.getBlock());
 
   llvm::BasicBlock *FetchMore = createBasicBlock("fetchmore");
 
@@ -828,7 +829,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
                         LV.getAddress());
   }
 
-  EmitBlock(LoopEnd.Block);
+  EmitBlock(LoopEnd.getBlock());
 }
 
 void CodeGenFunction::EmitObjCAtTryStmt(const ObjCAtTryStmt &S) {
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index f3c80bcf08e3..d7960beb1d1d 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -167,6 +167,7 @@ public:
                                    bool lval = false);
   virtual llvm::Value *GetSelector(CGBuilderTy &Builder, const ObjCMethodDecl
       *Method);
+  virtual llvm::Constant *GetEHType(QualType T);
 
   virtual llvm::Function *GenerateMethod(const ObjCMethodDecl *OMD,
                                          const ObjCContainerDecl *CD);
@@ -192,7 +193,8 @@ public:
   virtual void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dst);
   virtual void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                    llvm::Value *src, llvm::Value *dest);
+                                    llvm::Value *src, llvm::Value *dest,
+                                    bool threadlocal=false);
   virtual void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                     llvm::Value *src, llvm::Value *dest,
                                     llvm::Value *ivarOffset);
@@ -210,6 +212,10 @@ public:
   virtual llvm::Value *EmitIvarOffset(CodeGen::CodeGenFunction &CGF,
                                       const ObjCInterfaceDecl *Interface,
                                       const ObjCIvarDecl *Ivar);
+  virtual llvm::Constant *GCBlockLayout(CodeGen::CodeGenFunction &CGF,
+              const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &) {
+    return NULLPtr;
+  }
 };
 } // end anonymous namespace
 
@@ -402,6 +408,11 @@ llvm::Value *CGObjCGNU::GetSelector(CGBuilderTy &Builder, const ObjCMethodDecl
   return Builder.CreateLoad(Sel);
 }
 
+llvm::Constant *CGObjCGNU::GetEHType(QualType T) {
+  llvm_unreachable("asking for catch type for ObjC type in GNU runtime");
+  return 0;
+}
+
 llvm::Constant *CGObjCGNU::MakeConstantString(const std::string &Str,
                                               const std::string &Name) {
   llvm::Constant *ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
@@ -438,7 +449,7 @@ llvm::Constant *CGObjCGNU::MakeGlobal(const llvm::ArrayType *Ty,
 /// Generate an NSConstantString object.
 llvm::Constant *CGObjCGNU::GenerateConstantString(const StringLiteral *SL) {
 
-  std::string Str(SL->getStrData(), SL->getByteLength());
+  std::string Str = SL->getString().str();
 
   // Look for an existing one
   llvm::StringMap<llvm::Constant*>::iterator old = ObjCStrings.find(Str);
@@ -691,7 +702,7 @@ CGObjCGNU::GenerateMessageSend(CodeGen::CodeGenFunction &CGF,
     Params.push_back(SelectorTy);
     llvm::Value *self;
 
-    if (isa<ObjCMethodDecl>(CGF.CurFuncDecl)) {
+    if (isa<ObjCMethodDecl>(CGF.CurCodeDecl)) {
       self = CGF.LoadObjCSelf();
     } else {
       self = llvm::ConstantPointerNull::get(IdTy);
@@ -721,8 +732,8 @@ CGObjCGNU::GenerateMessageSend(CodeGen::CodeGenFunction &CGF,
 
     // The lookup function may have changed the receiver, so make sure we use
     // the new one.
-    ActualArgs[0] =
-        std::make_pair(RValue::get(Builder.CreateLoad(ReceiverPtr)), ASTIdTy);
+    ActualArgs[0] = std::make_pair(RValue::get(
+        Builder.CreateLoad(ReceiverPtr, true)), ASTIdTy);
   } else {
     std::vector<const llvm::Type*> Params;
     Params.push_back(Receiver->getType());
@@ -1854,6 +1865,19 @@ llvm::Constant *CGObjCGNU::EnumerationMutationFunction() {
   return CGM.CreateRuntimeFunction(FTy, "objc_enumerationMutation");
 }
 
+namespace {
+  struct CallSyncExit : EHScopeStack::Cleanup {
+    llvm::Value *SyncExitFn;
+    llvm::Value *SyncArg;
+    CallSyncExit(llvm::Value *SyncExitFn, llvm::Value *SyncArg)
+      : SyncExitFn(SyncExitFn), SyncArg(SyncArg) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) {
+      CGF.Builder.CreateCall(SyncExitFn, SyncArg)->setDoesNotThrow();
+    }
+  };
+}
+
 void CGObjCGNU::EmitSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
                                      const ObjCAtSynchronizedStmt &S) {
   std::vector<const llvm::Type*> Args(1, IdTy);
@@ -1870,13 +1894,8 @@ void CGObjCGNU::EmitSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   CGF.Builder.CreateCall(SyncEnter, SyncArg);
 
   // Register an all-paths cleanup to release the lock.
-  {
-    CodeGenFunction::CleanupBlock ReleaseScope(CGF, NormalAndEHCleanup);
-
-    llvm::Value *SyncExit = CGM.CreateRuntimeFunction(FTy, "objc_sync_exit");
-    SyncArg = CGF.Builder.CreateBitCast(SyncArg, IdTy);
-    CGF.Builder.CreateCall(SyncExit, SyncArg);
-  }
+  llvm::Value *SyncExit = CGM.CreateRuntimeFunction(FTy, "objc_sync_exit");
+  CGF.EHStack.pushCleanup<CallSyncExit>(NormalAndEHCleanup, SyncExit, SyncArg);
 
   // Emit the body of the statement.
   CGF.EmitStmt(S.getSynchBody());
@@ -2007,13 +2026,11 @@ void CGObjCGNU::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
   if (S.getFinallyStmt())
     CGF.ExitFinallyBlock(FinallyInfo);
 
-  if (Cont.Block) {
-    if (Cont.Block->use_empty())
-      delete Cont.Block;
-    else {
-      CGF.EmitBranch(Cont.Block);
-      CGF.EmitBlock(Cont.Block);
-    }
+  if (Cont.isValid()) {
+    if (Cont.getBlock()->use_empty())
+      delete Cont.getBlock();
+    else
+      CGF.EmitBlock(Cont.getBlock());
   }
 }
 
@@ -2075,11 +2092,16 @@ void CGObjCGNU::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
 }
 
 void CGObjCGNU::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                     llvm::Value *src, llvm::Value *dst) {
+                                     llvm::Value *src, llvm::Value *dst,
+                                     bool threadlocal) {
   CGBuilderTy B = CGF.Builder;
   src = EnforceType(B, src, IdTy);
   dst = EnforceType(B, dst, PtrToIdTy);
-  B.CreateCall2(GlobalAssignFn, src, dst);
+  if (!threadlocal)
+    B.CreateCall2(GlobalAssignFn, src, dst);
+  else
+    // FIXME. Add threadloca assign API
+    assert(false && "EmitObjCGlobalAssign - Threal Local API NYI");
 }
 
 void CGObjCGNU::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index 01ead9efe876..54d0ff21ea44 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -25,7 +25,8 @@
 #include "clang/Basic/LangOptions.h"
 #include "clang/Frontend/CodeGenOptions.h"
 
-#include "llvm/Intrinsics.h"
+#include "llvm/InlineAsm.h"
+#include "llvm/IntrinsicInst.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/DenseSet.h"
@@ -106,17 +107,33 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
   V = CGF.Builder.CreateGEP(V, Offset, "add.ptr");
   V = CGF.Builder.CreateBitCast(V, llvm::PointerType::getUnqual(LTy));
 
-  Qualifiers Quals = CGF.MakeQualifiers(IvarTy);
-  Quals.addCVRQualifiers(CVRQualifiers);
-
-  if (!Ivar->isBitField())
-    return LValue::MakeAddr(V, Quals);
+  if (!Ivar->isBitField()) {
+    LValue LV = CGF.MakeAddrLValue(V, IvarTy);
+    LV.getQuals().addCVRQualifiers(CVRQualifiers);
+    return LV;
+  }
 
-  // We need to compute the bit offset for the bit-field, the offset is to the
-  // byte. Note, there is a subtle invariant here: we can only call this routine
-  // on non-synthesized ivars but we may be called for synthesized ivars.
-  // However, a synthesized ivar can never be a bit-field, so this is safe.
-  uint64_t BitOffset = LookupFieldBitOffset(CGF.CGM, OID, 0, Ivar) % 8;
+  // We need to compute an access strategy for this bit-field. We are given the
+  // offset to the first byte in the bit-field, the sub-byte offset is taken
+  // from the original layout. We reuse the normal bit-field access strategy by
+  // treating this as an access to a struct where the bit-field is in byte 0,
+  // and adjust the containing type size as appropriate.
+  //
+  // FIXME: Note that currently we make a very conservative estimate of the
+  // alignment of the bit-field, because (a) it is not clear what guarantees the
+  // runtime makes us, and (b) we don't have a way to specify that the struct is
+  // at an alignment plus offset.
+  //
+  // Note, there is a subtle invariant here: we can only call this routine on
+  // non-synthesized ivars but we may be called for synthesized ivars.  However,
+  // a synthesized ivar can never be a bit-field, so this is safe.
+  const ASTRecordLayout &RL =
+    CGF.CGM.getContext().getASTObjCInterfaceLayout(OID);
+  uint64_t TypeSizeInBits = RL.getSize();
+  uint64_t FieldBitOffset = LookupFieldBitOffset(CGF.CGM, OID, 0, Ivar);
+  uint64_t BitOffset = FieldBitOffset % 8;
+  uint64_t ContainingTypeAlign = 8;
+  uint64_t ContainingTypeSize = TypeSizeInBits - (FieldBitOffset - BitOffset);
   uint64_t BitFieldSize =
     Ivar->getBitWidth()->EvaluateAsInt(CGF.getContext()).getZExtValue();
 
@@ -126,24 +143,12 @@ LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
   // layout object. However, this is blocked on other cleanups to the
   // Objective-C code, so for now we just live with allocating a bunch of these
   // objects.
+  CGBitFieldInfo *Info = new (CGF.CGM.getContext()) CGBitFieldInfo(
+    CGBitFieldInfo::MakeInfo(CGF.CGM.getTypes(), Ivar, BitOffset, BitFieldSize,
+                             ContainingTypeSize, ContainingTypeAlign));
 
-  // We always construct a single, possibly unaligned, access for this case.
-  CGBitFieldInfo::AccessInfo AI;
-  AI.FieldIndex = 0;
-  AI.FieldByteOffset = 0;
-  AI.FieldBitStart = BitOffset;
-  AI.AccessWidth = CGF.CGM.getContext().getTypeSize(IvarTy);
-  AI.AccessAlignment = 0;
-  AI.TargetBitOffset = 0;
-  AI.TargetBitWidth = BitFieldSize;
-
-  CGBitFieldInfo *Info =
-    new (CGF.CGM.getContext()) CGBitFieldInfo(BitFieldSize, 1, &AI,
-                                              IvarTy->isSignedIntegerType());
-
-  // FIXME: We need to set a very conservative alignment on this, or make sure
-  // that the runtime is doing the right thing.
-  return LValue::MakeBitfield(V, *Info, Quals.getCVRQualifiers());
+  return LValue::MakeBitfield(V, *Info,
+                              IvarTy.getCVRQualifiers() | CVRQualifiers);
 }
 
 ///
@@ -402,6 +407,16 @@ public:
     return CGM.CreateRuntimeFunction(FTy, "objc_assign_global");
   }
 
+  /// GcAssignThreadLocalFn -- LLVM objc_assign_threadlocal function.
+  llvm::Constant *getGcAssignThreadLocalFn() {
+    // id objc_assign_threadlocal(id src, id * dest)
+    std::vector<const llvm::Type*> Args(1, ObjectPtrTy);
+    Args.push_back(ObjectPtrTy->getPointerTo());
+    llvm::FunctionType *FTy =
+    llvm::FunctionType::get(ObjectPtrTy, Args, false);
+    return CGM.CreateRuntimeFunction(FTy, "objc_assign_threadlocal");
+  }
+  
   /// GcAssignIvarFn -- LLVM objc_assign_ivar function.
   llvm::Constant *getGcAssignIvarFn() {
     // id objc_assign_ivar(id, id *, ptrdiff_t)
@@ -425,7 +440,7 @@ public:
 
   /// GcAssignStrongCastFn -- LLVM objc_assign_strongCast function.
   llvm::Constant *getGcAssignStrongCastFn() {
-    // id objc_assign_global(id, id *)
+    // id objc_assign_strongCast(id, id *)
     std::vector<const llvm::Type*> Args(1, ObjectPtrTy);
     Args.push_back(ObjectPtrTy->getPointerTo());
     llvm::FunctionType *FTy =
@@ -719,25 +734,6 @@ public:
                                      "objc_msgSend_stret_fixup");
   }
 
-  llvm::Constant *getMessageSendIdFixupFn() {
-    // id objc_msgSendId_fixup(id, struct message_ref_t*, ...)
-    std::vector<const llvm::Type*> Params;
-    Params.push_back(ObjectPtrTy);
-    Params.push_back(MessageRefPtrTy);
-    return CGM.CreateRuntimeFunction(llvm::FunctionType::get(ObjectPtrTy,
-                                                             Params, true),
-                                     "objc_msgSendId_fixup");
-  }
-
-  llvm::Constant *getMessageSendIdStretFixupFn() {
-    // id objc_msgSendId_stret_fixup(id, struct message_ref_t*, ...)
-    std::vector<const llvm::Type*> Params;
-    Params.push_back(ObjectPtrTy);
-    Params.push_back(MessageRefPtrTy);
-    return CGM.CreateRuntimeFunction(llvm::FunctionType::get(ObjectPtrTy,
-                                                             Params, true),
-                                     "objc_msgSendId_stret_fixup");
-  }
   llvm::Constant *getMessageSendSuper2FixupFn() {
     // id objc_msgSendSuper2_fixup (struct objc_super *,
     //                              struct _super_message_ref_t*, ...)
@@ -760,28 +756,6 @@ public:
                                       "objc_msgSendSuper2_stret_fixup");
   }
 
-
-
-  /// EHPersonalityPtr - LLVM value for an i8* to the Objective-C
-  /// exception personality function.
-  llvm::Value *getEHPersonalityPtr() {
-    llvm::Constant *Personality =
-      CGM.CreateRuntimeFunction(llvm::FunctionType::get(llvm::Type::getInt32Ty(VMContext),
-                                                        true),
-                                "__objc_personality_v0");
-    return llvm::ConstantExpr::getBitCast(Personality, Int8PtrTy);
-  }
-
-  llvm::Constant *getUnwindResumeOrRethrowFn() {
-    std::vector<const llvm::Type*> Params;
-    Params.push_back(Int8PtrTy);
-    return CGM.CreateRuntimeFunction(
-      llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
-                              Params, false),
-      (CGM.getLangOptions().SjLjExceptions ? "_Unwind_SjLj_Resume" :
-       "_Unwind_Resume_or_Rethrow"));
-  }
-
   llvm::Constant *getObjCEndCatchFn() {
     return CGM.CreateRuntimeFunction(llvm::FunctionType::get(llvm::Type::getVoidTy(VMContext),
                                                              false),
@@ -905,7 +879,6 @@ protected:
   /// selector's name. The return value has type char *.
   llvm::Constant *GetMethodVarName(Selector Sel);
   llvm::Constant *GetMethodVarName(IdentifierInfo *Ident);
-  llvm::Constant *GetMethodVarName(const std::string &Name);
 
   /// GetMethodVarType - Return a unique constant for the given
   /// selector's name. The return value has type char *.
@@ -926,11 +899,15 @@ protected:
   /// name. The return value has type char *.
   llvm::Constant *GetClassName(IdentifierInfo *Ident);
 
+  llvm::Function *GetMethodDefinition(const ObjCMethodDecl *MD);
+
   /// BuildIvarLayout - Builds ivar layout bitmap for the class
   /// implementation for the __strong or __weak case.
   ///
   llvm::Constant *BuildIvarLayout(const ObjCImplementationDecl *OI,
                                   bool ForStrongLayout);
+  
+  llvm::Constant *BuildIvarLayoutBitmap(std::string &BitMap);
 
   void BuildAggrIvarRecordLayout(const RecordType *RT,
                                  unsigned int BytePos, bool ForStrongLayout,
@@ -1022,6 +999,9 @@ public:
   /// forward references will be filled in with empty bodies if no
   /// definition is seen. The return value has type ProtocolPtrTy.
   virtual llvm::Constant *GetOrEmitProtocolRef(const ObjCProtocolDecl *PD)=0;
+  virtual llvm::Constant *GCBlockLayout(CodeGen::CodeGenFunction &CGF,
+                      const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &);
+  
 };
 
 class CGObjCMac : public CGObjCCommonMac {
@@ -1053,15 +1033,6 @@ private:
   /// EmitSuperClassRef - Emits reference to class's main metadata class.
   llvm::Value *EmitSuperClassRef(const ObjCInterfaceDecl *ID);
 
-  CodeGen::RValue EmitMessageSend(CodeGen::CodeGenFunction &CGF,
-                                  ReturnValueSlot Return,
-                                  QualType ResultType,
-                                  Selector Sel,
-                                  llvm::Value *Arg0,
-                                  QualType Arg0Ty,
-                                  bool IsSuper,
-                                  const CallArgList &CallArgs);
-
   /// EmitIvarList - Emit the ivar list for the given
   /// implementation. If ForClass is true the list of class ivars
   /// (i.e. metaclass ivars) is emitted, otherwise the list of
@@ -1174,6 +1145,8 @@ public:
   virtual llvm::Value *GetSelector(CGBuilderTy &Builder,
                                    const ObjCMethodDecl *Method);
 
+  virtual llvm::Constant *GetEHType(QualType T);
+
   virtual void GenerateCategory(const ObjCCategoryImplDecl *CMD);
 
   virtual void GenerateClass(const ObjCImplementationDecl *ClassDecl);
@@ -1198,7 +1171,8 @@ public:
   virtual void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dst);
   virtual void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                    llvm::Value *src, llvm::Value *dest);
+                                    llvm::Value *src, llvm::Value *dest,
+                                    bool threadlocal = false);
   virtual void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dest,
                                   llvm::Value *ivarOffset);
@@ -1343,7 +1317,7 @@ private:
 
   /// GetInterfaceEHType - Get the cached ehtype for the given Objective-C
   /// interface. The return value has type EHTypePtrTy.
-  llvm::Value *GetInterfaceEHType(const ObjCInterfaceDecl *ID,
+  llvm::Constant *GetInterfaceEHType(const ObjCInterfaceDecl *ID,
                                   bool ForDefinition);
 
   const char *getMetaclassSymbolPrefix() const {
@@ -1418,6 +1392,8 @@ public:
   virtual llvm::Value *GenerateProtocolRef(CGBuilderTy &Builder,
                                            const ObjCProtocolDecl *PD);
 
+  virtual llvm::Constant *GetEHType(QualType T);
+
   virtual llvm::Constant *GetPropertyGetFunction() {
     return ObjCTypes.getGetPropertyFn();
   }
@@ -1444,7 +1420,8 @@ public:
   virtual void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dst);
   virtual void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                    llvm::Value *src, llvm::Value *dest);
+                                    llvm::Value *src, llvm::Value *dest,
+                                    bool threadlocal = false);
   virtual void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dest,
                                   llvm::Value *ivarOffset);
@@ -1515,6 +1492,11 @@ llvm::Value *CGObjCMac::GetSelector(CGBuilderTy &Builder, const ObjCMethodDecl
   return EmitSelector(Builder, Method->getSelector());
 }
 
+llvm::Constant *CGObjCMac::GetEHType(QualType T) {
+  llvm_unreachable("asking for catch type for ObjC type in fragile runtime");
+  return 0;
+}
+
 /// Generate a constant CFString object.
 /*
   struct __builtin_CFString {
@@ -1664,6 +1646,90 @@ CGObjCCommonMac::EmitLegacyMessageSend(CodeGen::CodeGenFunction &CGF,
   return CGF.EmitCall(FnInfo, Fn, Return, ActualArgs);
 }
 
+static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT) {
+  if (FQT.isObjCGCStrong())
+    return Qualifiers::Strong;
+  
+  if (FQT.isObjCGCWeak())
+    return Qualifiers::Weak;
+  
+  if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType())
+    return Qualifiers::Strong;
+  
+  if (const PointerType *PT = FQT->getAs<PointerType>())
+    return GetGCAttrTypeForType(Ctx, PT->getPointeeType());
+  
+  return Qualifiers::GCNone;
+}
+
+llvm::Constant *CGObjCCommonMac::GCBlockLayout(CodeGen::CodeGenFunction &CGF,
+              const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &DeclRefs) {
+  llvm::Constant *NullPtr = 
+    llvm::Constant::getNullValue(llvm::Type::getInt8PtrTy(VMContext));
+  if ((CGM.getLangOptions().getGCMode() == LangOptions::NonGC) ||
+      DeclRefs.empty())
+    return NullPtr;
+  bool hasUnion = false;
+  SkipIvars.clear();
+  IvarsInfo.clear();
+  unsigned WordSizeInBits = CGM.getContext().Target.getPointerWidth(0);
+  unsigned ByteSizeInBits = CGM.getContext().Target.getCharWidth();
+  
+  for (size_t i = 0; i < DeclRefs.size(); ++i) {
+    const BlockDeclRefExpr *BDRE = DeclRefs[i];
+    const ValueDecl *VD = BDRE->getDecl();
+    CharUnits Offset = CGF.BlockDecls[VD];
+    uint64_t FieldOffset = Offset.getQuantity();
+    QualType Ty = VD->getType();
+    assert(!Ty->isArrayType() && 
+           "Array block variable should have been caught");
+    if ((Ty->isRecordType() || Ty->isUnionType()) && !BDRE->isByRef()) {
+      BuildAggrIvarRecordLayout(Ty->getAs<RecordType>(),
+                                FieldOffset,
+                                true,
+                                hasUnion);
+      continue;
+    }
+      
+    Qualifiers::GC GCAttr = GetGCAttrTypeForType(CGM.getContext(), Ty);
+    unsigned FieldSize = CGM.getContext().getTypeSize(Ty);
+    // __block variables are passed by their descriptior address. So, size
+    // must reflect this.
+    if (BDRE->isByRef())
+      FieldSize = WordSizeInBits;
+    if (GCAttr == Qualifiers::Strong || BDRE->isByRef())
+      IvarsInfo.push_back(GC_IVAR(FieldOffset,
+                                  FieldSize / WordSizeInBits));
+    else if (GCAttr == Qualifiers::GCNone || GCAttr == Qualifiers::Weak)
+      SkipIvars.push_back(GC_IVAR(FieldOffset,
+                                  FieldSize / ByteSizeInBits));
+  }
+  
+  if (IvarsInfo.empty())
+    return NullPtr;
+  // Sort on byte position in case we encounterred a union nested in
+  // block variable type's aggregate type.
+  if (hasUnion && !IvarsInfo.empty())
+    std::sort(IvarsInfo.begin(), IvarsInfo.end());
+  if (hasUnion && !SkipIvars.empty())
+    std::sort(SkipIvars.begin(), SkipIvars.end());
+  
+  std::string BitMap;
+  llvm::Constant *C = BuildIvarLayoutBitmap(BitMap);
+  if (CGM.getLangOptions().ObjCGCBitmapPrint) {
+    printf("\n block variable layout for block: ");
+    const unsigned char *s = (unsigned char*)BitMap.c_str();
+    for (unsigned i = 0; i < BitMap.size(); i++)
+      if (!(s[i] & 0xf0))
+        printf("0x0%x%s", s[i], s[i] != 0 ? ", " : "");
+      else
+        printf("0x%x%s",  s[i], s[i] != 0 ? ", " : "");
+    printf("\n");
+  }
+  
+  return C;
+}
+
 llvm::Value *CGObjCMac::GenerateProtocolRef(CGBuilderTy &Builder,
                                             const ObjCProtocolDecl *PD) {
   // FIXME: I don't understand why gcc generates this, or where it is
@@ -1927,8 +1993,9 @@ llvm::Constant *CGObjCCommonMac::EmitPropertyList(llvm::Twine Name,
                                                    Prop));
   }
   if (const ObjCInterfaceDecl *OID = dyn_cast<ObjCInterfaceDecl>(OCD)) {
-    for (ObjCInterfaceDecl::protocol_iterator P = OID->protocol_begin(),
-         E = OID->protocol_end(); P != E; ++P)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+         P = OID->all_referenced_protocol_begin(),
+         E = OID->all_referenced_protocol_end(); P != E; ++P)
       PushProtocolProperties(PropertySet, Properties, Container, (*P), 
                              ObjCTypes);
   }
@@ -2116,8 +2183,8 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) {
     const_cast<ObjCInterfaceDecl*>(ID->getClassInterface());
   llvm::Constant *Protocols =
     EmitProtocolList("\01L_OBJC_CLASS_PROTOCOLS_" + ID->getName(),
-                     Interface->protocol_begin(),
-                     Interface->protocol_end());
+                     Interface->all_referenced_protocol_begin(),
+                     Interface->all_referenced_protocol_end());
   unsigned Flags = eClassFlags_Factory;
   if (ID->getNumIvarInitializers())
     Flags |= eClassFlags_HasCXXStructors;
@@ -2427,8 +2494,7 @@ llvm::Constant *CGObjCMac::EmitIvarList(const ObjCImplementationDecl *ID,
 /// given method if it has been defined. The result is null if the
 /// method has not been defined. The return value has type MethodPtrTy.
 llvm::Constant *CGObjCMac::GetMethodConstant(const ObjCMethodDecl *MD) {
-  // FIXME: Use DenseMap::lookup
-  llvm::Function *Fn = MethodDefinitions[MD];
+  llvm::Function *Fn = GetMethodDefinition(MD);
   if (!Fn)
     return 0;
 
@@ -2529,6 +2595,214 @@ void CGObjCMac::EmitSynchronizedStmt(CodeGenFunction &CGF,
   return EmitTryOrSynchronizedStmt(CGF, S);
 }
 
+namespace {
+  struct PerformFragileFinally : EHScopeStack::Cleanup {
+    const Stmt &S;
+    llvm::Value *SyncArgSlot;
+    llvm::Value *CallTryExitVar;
+    llvm::Value *ExceptionData;
+    ObjCTypesHelper &ObjCTypes;
+    PerformFragileFinally(const Stmt *S,
+                          llvm::Value *SyncArgSlot,
+                          llvm::Value *CallTryExitVar,
+                          llvm::Value *ExceptionData,
+                          ObjCTypesHelper *ObjCTypes)
+      : S(*S), SyncArgSlot(SyncArgSlot), CallTryExitVar(CallTryExitVar),
+        ExceptionData(ExceptionData), ObjCTypes(*ObjCTypes) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      // Check whether we need to call objc_exception_try_exit.
+      // In optimized code, this branch will always be folded.
+      llvm::BasicBlock *FinallyCallExit =
+        CGF.createBasicBlock("finally.call_exit");
+      llvm::BasicBlock *FinallyNoCallExit =
+        CGF.createBasicBlock("finally.no_call_exit");
+      CGF.Builder.CreateCondBr(CGF.Builder.CreateLoad(CallTryExitVar),
+                               FinallyCallExit, FinallyNoCallExit);
+
+      CGF.EmitBlock(FinallyCallExit);
+      CGF.Builder.CreateCall(ObjCTypes.getExceptionTryExitFn(), ExceptionData)
+        ->setDoesNotThrow();
+
+      CGF.EmitBlock(FinallyNoCallExit);
+
+      if (isa<ObjCAtTryStmt>(S)) {
+        if (const ObjCAtFinallyStmt* FinallyStmt =
+              cast<ObjCAtTryStmt>(S).getFinallyStmt()) {
+          // Save the current cleanup destination in case there's
+          // control flow inside the finally statement.
+          llvm::Value *CurCleanupDest =
+            CGF.Builder.CreateLoad(CGF.getNormalCleanupDestSlot());
+
+          CGF.EmitStmt(FinallyStmt->getFinallyBody());
+
+          if (CGF.HaveInsertPoint()) {
+            CGF.Builder.CreateStore(CurCleanupDest,
+                                    CGF.getNormalCleanupDestSlot());
+          } else {
+            // Currently, the end of the cleanup must always exist.
+            CGF.EnsureInsertPoint();
+          }
+        }
+      } else {
+        // Emit objc_sync_exit(expr); as finally's sole statement for
+        // @synchronized.
+        llvm::Value *SyncArg = CGF.Builder.CreateLoad(SyncArgSlot);
+        CGF.Builder.CreateCall(ObjCTypes.getSyncExitFn(), SyncArg)
+          ->setDoesNotThrow();
+      }
+    }
+  };
+
+  class FragileHazards {
+    CodeGenFunction &CGF;
+    llvm::SmallVector<llvm::Value*, 20> Locals;
+    llvm::DenseSet<llvm::BasicBlock*> BlocksBeforeTry;
+
+    llvm::InlineAsm *ReadHazard;
+    llvm::InlineAsm *WriteHazard;
+
+    llvm::FunctionType *GetAsmFnType();
+
+    void collectLocals();
+    void emitReadHazard(CGBuilderTy &Builder);
+
+  public:
+    FragileHazards(CodeGenFunction &CGF);
+
+    void emitWriteHazard();
+    void emitHazardsInNewBlocks();
+  };
+}
+
+/// Create the fragile-ABI read and write hazards based on the current
+/// state of the function, which is presumed to be immediately prior
+/// to a @try block.  These hazards are used to maintain correct
+/// semantics in the face of optimization and the fragile ABI's
+/// cavalier use of setjmp/longjmp.
+FragileHazards::FragileHazards(CodeGenFunction &CGF) : CGF(CGF) {
+  collectLocals();
+
+  if (Locals.empty()) return;
+
+  // Collect all the blocks in the function.
+  for (llvm::Function::iterator
+         I = CGF.CurFn->begin(), E = CGF.CurFn->end(); I != E; ++I)
+    BlocksBeforeTry.insert(&*I);
+
+  llvm::FunctionType *AsmFnTy = GetAsmFnType();
+
+  // Create a read hazard for the allocas.  This inhibits dead-store
+  // optimizations and forces the values to memory.  This hazard is
+  // inserted before any 'throwing' calls in the protected scope to
+  // reflect the possibility that the variables might be read from the
+  // catch block if the call throws.
+  {
+    std::string Constraint;
+    for (unsigned I = 0, E = Locals.size(); I != E; ++I) {
+      if (I) Constraint += ',';
+      Constraint += "*m";
+    }
+
+    ReadHazard = llvm::InlineAsm::get(AsmFnTy, "", Constraint, true, false);
+  }
+
+  // Create a write hazard for the allocas.  This inhibits folding
+  // loads across the hazard.  This hazard is inserted at the
+  // beginning of the catch path to reflect the possibility that the
+  // variables might have been written within the protected scope.
+  {
+    std::string Constraint;
+    for (unsigned I = 0, E = Locals.size(); I != E; ++I) {
+      if (I) Constraint += ',';
+      Constraint += "=*m";
+    }
+
+    WriteHazard = llvm::InlineAsm::get(AsmFnTy, "", Constraint, true, false);
+  }
+}
+
+/// Emit a write hazard at the current location.
+void FragileHazards::emitWriteHazard() {
+  if (Locals.empty()) return;
+
+  CGF.Builder.CreateCall(WriteHazard, Locals.begin(), Locals.end())
+    ->setDoesNotThrow();
+}
+
+void FragileHazards::emitReadHazard(CGBuilderTy &Builder) {
+  assert(!Locals.empty());
+  Builder.CreateCall(ReadHazard, Locals.begin(), Locals.end())
+    ->setDoesNotThrow();
+}
+
+/// Emit read hazards in all the protected blocks, i.e. all the blocks
+/// which have been inserted since the beginning of the try.
+void FragileHazards::emitHazardsInNewBlocks() {
+  if (Locals.empty()) return;
+
+  CGBuilderTy Builder(CGF.getLLVMContext());
+
+  // Iterate through all blocks, skipping those prior to the try.
+  for (llvm::Function::iterator
+         FI = CGF.CurFn->begin(), FE = CGF.CurFn->end(); FI != FE; ++FI) {
+    llvm::BasicBlock &BB = *FI;
+    if (BlocksBeforeTry.count(&BB)) continue;
+
+    // Walk through all the calls in the block.
+    for (llvm::BasicBlock::iterator
+           BI = BB.begin(), BE = BB.end(); BI != BE; ++BI) {
+      llvm::Instruction &I = *BI;
+
+      // Ignore instructions that aren't non-intrinsic calls.
+      // These are the only calls that can possibly call longjmp.
+      if (!isa<llvm::CallInst>(I) && !isa<llvm::InvokeInst>(I)) continue;
+      if (isa<llvm::IntrinsicInst>(I))
+        continue;
+
+      // Ignore call sites marked nounwind.  This may be questionable,
+      // since 'nounwind' doesn't necessarily mean 'does not call longjmp'.
+      llvm::CallSite CS(&I);
+      if (CS.doesNotThrow()) continue;
+
+      // Insert a read hazard before the call.  This will ensure that
+      // any writes to the locals are performed before making the
+      // call.  If the call throws, then this is sufficient to
+      // guarantee correctness as long as it doesn't also write to any
+      // locals.
+      Builder.SetInsertPoint(&BB, BI);
+      emitReadHazard(Builder);
+    }
+  }
+}
+
+static void addIfPresent(llvm::DenseSet<llvm::Value*> &S, llvm::Value *V) {
+  if (V) S.insert(V);
+}
+
+void FragileHazards::collectLocals() {
+  // Compute a set of allocas to ignore.
+  llvm::DenseSet<llvm::Value*> AllocasToIgnore;
+  addIfPresent(AllocasToIgnore, CGF.ReturnValue);
+  addIfPresent(AllocasToIgnore, CGF.NormalCleanupDest);
+  addIfPresent(AllocasToIgnore, CGF.EHCleanupDest);
+
+  // Collect all the allocas currently in the function.  This is
+  // probably way too aggressive.
+  llvm::BasicBlock &Entry = CGF.CurFn->getEntryBlock();
+  for (llvm::BasicBlock::iterator
+         I = Entry.begin(), E = Entry.end(); I != E; ++I)
+    if (isa<llvm::AllocaInst>(*I) && !AllocasToIgnore.count(&*I))
+      Locals.push_back(&*I);
+}
+
+llvm::FunctionType *FragileHazards::GetAsmFnType() {
+  std::vector<const llvm::Type *> Tys(Locals.size());
+  for (unsigned I = 0, E = Locals.size(); I != E; ++I)
+    Tys[I] = Locals[I]->getType();
+  return llvm::FunctionType::get(CGF.Builder.getVoidTy(), Tys, false);
+}
+
 /*
 
   Objective-C setjmp-longjmp (sjlj) Exception Handling
@@ -2657,66 +2931,49 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
   // For @synchronized, call objc_sync_enter(sync.expr). The
   // evaluation of the expression must occur before we enter the
-  // @synchronized. We can safely avoid a temp here because jumps into
-  // @synchronized are illegal & this will dominate uses.
-  llvm::Value *SyncArg = 0;
+  // @synchronized.  We can't avoid a temp here because we need the
+  // value to be preserved.  If the backend ever does liveness
+  // correctly after setjmp, this will be unnecessary.
+  llvm::Value *SyncArgSlot = 0;
   if (!isTry) {
-    SyncArg =
+    llvm::Value *SyncArg =
       CGF.EmitScalarExpr(cast<ObjCAtSynchronizedStmt>(S).getSynchExpr());
     SyncArg = CGF.Builder.CreateBitCast(SyncArg, ObjCTypes.ObjectPtrTy);
     CGF.Builder.CreateCall(ObjCTypes.getSyncEnterFn(), SyncArg)
       ->setDoesNotThrow();
+
+    SyncArgSlot = CGF.CreateTempAlloca(SyncArg->getType(), "sync.arg");
+    CGF.Builder.CreateStore(SyncArg, SyncArgSlot);
   }
 
-  // Allocate memory for the exception data and rethrow pointer.
+  // Allocate memory for the setjmp buffer.  This needs to be kept
+  // live throughout the try and catch blocks.
   llvm::Value *ExceptionData = CGF.CreateTempAlloca(ObjCTypes.ExceptionDataTy,
                                                     "exceptiondata.ptr");
-  llvm::Value *RethrowPtr = CGF.CreateTempAlloca(ObjCTypes.ObjectPtrTy,
-                                                 "_rethrow");
+
+  // Create the fragile hazards.  Note that this will not capture any
+  // of the allocas required for exception processing, but will
+  // capture the current basic block (which extends all the way to the
+  // setjmp call) as "before the @try".
+  FragileHazards Hazards(CGF);
 
   // Create a flag indicating whether the cleanup needs to call
   // objc_exception_try_exit.  This is true except when
   //   - no catches match and we're branching through the cleanup
   //     just to rethrow the exception, or
   //   - a catch matched and we're falling out of the catch handler.
+  // The setjmp-safety rule here is that we should always store to this
+  // variable in a place that dominates the branch through the cleanup
+  // without passing through any setjmps.
   llvm::Value *CallTryExitVar = CGF.CreateTempAlloca(CGF.Builder.getInt1Ty(),
                                                      "_call_try_exit");
-  CGF.Builder.CreateStore(llvm::ConstantInt::getTrue(VMContext),
-                          CallTryExitVar);
 
   // Push a normal cleanup to leave the try scope.
-  {
-    CodeGenFunction::CleanupBlock FinallyScope(CGF, NormalCleanup);
-
-    // Check whether we need to call objc_exception_try_exit.
-    // In optimized code, this branch will always be folded.
-    llvm::BasicBlock *FinallyCallExit =
-      CGF.createBasicBlock("finally.call_exit");
-    llvm::BasicBlock *FinallyNoCallExit =
-      CGF.createBasicBlock("finally.no_call_exit");
-    CGF.Builder.CreateCondBr(CGF.Builder.CreateLoad(CallTryExitVar),
-                             FinallyCallExit, FinallyNoCallExit);
-
-    CGF.EmitBlock(FinallyCallExit);
-    CGF.Builder.CreateCall(ObjCTypes.getExceptionTryExitFn(), ExceptionData)
-      ->setDoesNotThrow();
-
-    CGF.EmitBlock(FinallyNoCallExit);
-
-    if (isTry) {
-      if (const ObjCAtFinallyStmt* FinallyStmt =
-          cast<ObjCAtTryStmt>(S).getFinallyStmt())
-        CGF.EmitStmt(FinallyStmt->getFinallyBody());
-
-      // ~CleanupBlock requires there to be an exit block.
-      CGF.EnsureInsertPoint();
-    } else {
-      // Emit objc_sync_exit(expr); as finally's sole statement for
-      // @synchronized.
-      CGF.Builder.CreateCall(ObjCTypes.getSyncExitFn(), SyncArg)
-        ->setDoesNotThrow();
-    }
-  }
+  CGF.EHStack.pushCleanup<PerformFragileFinally>(NormalCleanup, &S,
+                                                 SyncArgSlot,
+                                                 CallTryExitVar,
+                                                 ExceptionData,
+                                                 &ObjCTypes);
 
   // Enter a try block:
   //  - Call objc_exception_try_enter to push ExceptionData on top of
@@ -2738,68 +2995,71 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
   llvm::BasicBlock *TryBlock = CGF.createBasicBlock("try");
   llvm::BasicBlock *TryHandler = CGF.createBasicBlock("try.handler");
   llvm::Value *DidCatch =
-    CGF.Builder.CreateIsNull(SetJmpResult, "did_catch_exception");
-  CGF.Builder.CreateCondBr(DidCatch, TryBlock, TryHandler);
+    CGF.Builder.CreateIsNotNull(SetJmpResult, "did_catch_exception");
+  CGF.Builder.CreateCondBr(DidCatch, TryHandler, TryBlock);
 
   // Emit the protected block.
   CGF.EmitBlock(TryBlock);
+  CGF.Builder.CreateStore(CGF.Builder.getTrue(), CallTryExitVar);
   CGF.EmitStmt(isTry ? cast<ObjCAtTryStmt>(S).getTryBody()
                      : cast<ObjCAtSynchronizedStmt>(S).getSynchBody());
-  CGF.EmitBranchThroughCleanup(FinallyEnd);
+
+  CGBuilderTy::InsertPoint TryFallthroughIP = CGF.Builder.saveAndClearIP();
 
   // Emit the exception handler block.
   CGF.EmitBlock(TryHandler);
 
-  // Retrieve the exception object.  We may emit multiple blocks but
-  // nothing can cross this so the value is already in SSA form.
-  llvm::CallInst *Caught =
-    CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
-                           ExceptionData, "caught");
-  Caught->setDoesNotThrow();
-
-  // Remember the exception to rethrow.
-  CGF.Builder.CreateStore(Caught, RethrowPtr);
-
-  // Note: at this point, objc_exception_throw already popped the
-  // catch handler, so anything that branches to the cleanup needs
-  // to set CallTryExitVar to false.
+  // Don't optimize loads of the in-scope locals across this point.
+  Hazards.emitWriteHazard();
 
   // For a @synchronized (or a @try with no catches), just branch
   // through the cleanup to the rethrow block.
   if (!isTry || !cast<ObjCAtTryStmt>(S).getNumCatchStmts()) {
     // Tell the cleanup not to re-pop the exit.
-    CGF.Builder.CreateStore(llvm::ConstantInt::getFalse(VMContext),
-                            CallTryExitVar);
-    
+    CGF.Builder.CreateStore(CGF.Builder.getFalse(), CallTryExitVar);
     CGF.EmitBranchThroughCleanup(FinallyRethrow);
 
   // Otherwise, we have to match against the caught exceptions.
   } else {
+    // Retrieve the exception object.  We may emit multiple blocks but
+    // nothing can cross this so the value is already in SSA form.
+    llvm::CallInst *Caught =
+      CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
+                             ExceptionData, "caught");
+    Caught->setDoesNotThrow();
+
     // Push the exception to rethrow onto the EH value stack for the
     // benefit of any @throws in the handlers.
     CGF.ObjCEHValueStack.push_back(Caught);
 
     const ObjCAtTryStmt* AtTryStmt = cast<ObjCAtTryStmt>(&S);
-    
-    // Enter a new exception try block (in case a @catch block throws
-    // an exception).  Now CallTryExitVar (currently true) is back in
-    // synch with reality.
-    CGF.Builder.CreateCall(ObjCTypes.getExceptionTryEnterFn(), ExceptionData)
-      ->setDoesNotThrow();
 
-    llvm::CallInst *SetJmpResult =
-      CGF.Builder.CreateCall(ObjCTypes.getSetJmpFn(), SetJmpBuffer,
-                             "setjmp.result");
-    SetJmpResult->setDoesNotThrow();
+    bool HasFinally = (AtTryStmt->getFinallyStmt() != 0);
 
-    llvm::Value *Threw =
-      CGF.Builder.CreateIsNotNull(SetJmpResult, "did_catch_exception");
+    llvm::BasicBlock *CatchBlock = 0;
+    llvm::BasicBlock *CatchHandler = 0;
+    if (HasFinally) {
+      // Enter a new exception try block (in case a @catch block
+      // throws an exception).
+      CGF.Builder.CreateCall(ObjCTypes.getExceptionTryEnterFn(), ExceptionData)
+        ->setDoesNotThrow();
+
+      llvm::CallInst *SetJmpResult =
+        CGF.Builder.CreateCall(ObjCTypes.getSetJmpFn(), SetJmpBuffer,
+                               "setjmp.result");
+      SetJmpResult->setDoesNotThrow();
+
+      llvm::Value *Threw =
+        CGF.Builder.CreateIsNotNull(SetJmpResult, "did_catch_exception");
 
-    llvm::BasicBlock *CatchBlock = CGF.createBasicBlock("catch");
-    llvm::BasicBlock *CatchHandler = CGF.createBasicBlock("catch_for_catch");
-    CGF.Builder.CreateCondBr(Threw, CatchHandler, CatchBlock);
+      CatchBlock = CGF.createBasicBlock("catch");
+      CatchHandler = CGF.createBasicBlock("catch_for_catch");
+      CGF.Builder.CreateCondBr(Threw, CatchHandler, CatchBlock);
+
+      CGF.EmitBlock(CatchBlock);
+    }
 
-    CGF.EmitBlock(CatchBlock);
+    CGF.Builder.CreateStore(CGF.Builder.getInt1(HasFinally), CallTryExitVar);
 
     // Handle catch list. As a special case we check if everything is
     // matched and avoid generating code for falling off the end if
@@ -2872,7 +3132,7 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
       // Collect any cleanups for the catch variable.  The scope lasts until
       // the end of the catch body.
-      CodeGenFunction::RunCleanupsScope CatchVarCleanups(CGF);      
+      CodeGenFunction::RunCleanupsScope CatchVarCleanups(CGF);
 
       CGF.EmitLocalBlockVarDecl(*CatchParam);
       assert(CGF.HaveInsertPoint() && "DeclStmt destroyed insert point?");
@@ -2896,43 +3156,55 @@ void CGObjCMac::EmitTryOrSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
 
     CGF.ObjCEHValueStack.pop_back();
 
-    if (!AllMatched) {
-      // None of the handlers caught the exception, so store it to be
-      // rethrown at the end of the @finally block.
-      CGF.Builder.CreateStore(Caught, RethrowPtr);
+    // If nothing wanted anything to do with the caught exception,
+    // kill the extract call.
+    if (Caught->use_empty())
+      Caught->eraseFromParent();
+
+    if (!AllMatched)
       CGF.EmitBranchThroughCleanup(FinallyRethrow);
-    }
 
-    // Emit the exception handler for the @catch blocks.
-    CGF.EmitBlock(CatchHandler);
+    if (HasFinally) {
+      // Emit the exception handler for the @catch blocks.
+      CGF.EmitBlock(CatchHandler);
 
-    // Rethrow the new exception, not the old one.
-    Caught = CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
-                                    ExceptionData);
-    Caught->setDoesNotThrow();
-    CGF.Builder.CreateStore(Caught, RethrowPtr);
+      // In theory we might now need a write hazard, but actually it's
+      // unnecessary because there's no local-accessing code between
+      // the try's write hazard and here.
+      //Hazards.emitWriteHazard();
 
-    // Don't pop the catch handler; the throw already did.
-    CGF.Builder.CreateStore(llvm::ConstantInt::getFalse(VMContext),
-                            CallTryExitVar);
-    CGF.EmitBranchThroughCleanup(FinallyRethrow);
+      // Don't pop the catch handler; the throw already did.
+      CGF.Builder.CreateStore(CGF.Builder.getFalse(), CallTryExitVar);
+      CGF.EmitBranchThroughCleanup(FinallyRethrow);
+    }
   }
 
+  // Insert read hazards as required in the new blocks.
+  Hazards.emitHazardsInNewBlocks();
+
   // Pop the cleanup.
+  CGF.Builder.restoreIP(TryFallthroughIP);
+  if (CGF.HaveInsertPoint())
+    CGF.Builder.CreateStore(CGF.Builder.getTrue(), CallTryExitVar);
   CGF.PopCleanupBlock();
-  CGF.EmitBlock(FinallyEnd.Block);
+  CGF.EmitBlock(FinallyEnd.getBlock(), true);
 
   // Emit the rethrow block.
-  CGF.Builder.ClearInsertionPoint();
-  CGF.EmitBlock(FinallyRethrow.Block, true);
+  CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
+  CGF.EmitBlock(FinallyRethrow.getBlock(), true);
   if (CGF.HaveInsertPoint()) {
-    CGF.Builder.CreateCall(ObjCTypes.getExceptionThrowFn(),
-                           CGF.Builder.CreateLoad(RethrowPtr))
+    // Just look in the buffer for the exception to throw.
+    llvm::CallInst *Caught =
+      CGF.Builder.CreateCall(ObjCTypes.getExceptionExtractFn(),
+                             ExceptionData);
+    Caught->setDoesNotThrow();
+
+    CGF.Builder.CreateCall(ObjCTypes.getExceptionThrowFn(), Caught)
       ->setDoesNotThrow();
     CGF.Builder.CreateUnreachable();
   }
 
-  CGF.Builder.SetInsertPoint(FinallyEnd.Block);
+  CGF.Builder.restoreIP(SavedIP);
 }
 
 void CGObjCMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
@@ -2996,7 +3268,8 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
 /// objc_assign_global (id src, id *dst)
 ///
 void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                     llvm::Value *src, llvm::Value *dst) {
+                                     llvm::Value *src, llvm::Value *dst,
+                                     bool threadlocal) {
   const llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
@@ -3007,8 +3280,12 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
   dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
-  CGF.Builder.CreateCall2(ObjCTypes.getGcAssignGlobalFn(),
-                          src, dst, "globalassign");
+  if (!threadlocal)
+    CGF.Builder.CreateCall2(ObjCTypes.getGcAssignGlobalFn(),
+                            src, dst, "globalassign");
+  else
+    CGF.Builder.CreateCall2(ObjCTypes.getGcAssignThreadLocalFn(),
+                            src, dst, "threadlocalassign");
   return;
 }
 
@@ -3260,6 +3537,22 @@ llvm::Constant *CGObjCCommonMac::GetClassName(IdentifierInfo *Ident) {
   return getConstantGEP(VMContext, Entry, 0, 0);
 }
 
+llvm::Function *CGObjCCommonMac::GetMethodDefinition(const ObjCMethodDecl *MD) {
+  llvm::DenseMap<const ObjCMethodDecl*, llvm::Function*>::iterator
+      I = MethodDefinitions.find(MD);
+  if (I != MethodDefinitions.end())
+    return I->second;
+
+  if (MD->hasBody() && MD->getPCHLevel() > 0) {
+    // MD isn't emitted yet because it comes from PCH.
+    CGM.EmitTopLevelDecl(const_cast<ObjCMethodDecl*>(MD));
+    assert(MethodDefinitions[MD] && "EmitTopLevelDecl didn't emit the method!");
+    return MethodDefinitions[MD];
+  }
+
+  return NULL;
+}
+
 /// GetIvarLayoutName - Returns a unique constant for the given
 /// ivar layout bitmap.
 llvm::Constant *CGObjCCommonMac::GetIvarLayoutName(IdentifierInfo *Ident,
@@ -3267,22 +3560,6 @@ llvm::Constant *CGObjCCommonMac::GetIvarLayoutName(IdentifierInfo *Ident,
   return llvm::Constant::getNullValue(ObjCTypes.Int8PtrTy);
 }
 
-static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT) {
-  if (FQT.isObjCGCStrong())
-    return Qualifiers::Strong;
-
-  if (FQT.isObjCGCWeak())
-    return Qualifiers::Weak;
-
-  if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType())
-    return Qualifiers::Strong;
-
-  if (const PointerType *PT = FQT->getAs<PointerType>())
-    return GetGCAttrTypeForType(Ctx, PT->getPointeeType());
-
-  return Qualifiers::GCNone;
-}
-
 void CGObjCCommonMac::BuildAggrIvarRecordLayout(const RecordType *RT,
                                                 unsigned int BytePos,
                                                 bool ForStrongLayout,
@@ -3446,63 +3723,19 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
                                 MaxSkippedUnionIvarSize));
 }
 
-/// BuildIvarLayout - Builds ivar layout bitmap for the class
-/// implementation for the __strong or __weak case.
-/// The layout map displays which words in ivar list must be skipped
-/// and which must be scanned by GC (see below). String is built of bytes.
-/// Each byte is divided up in two nibbles (4-bit each). Left nibble is count
-/// of words to skip and right nibble is count of words to scan. So, each
-/// nibble represents up to 15 workds to skip or scan. Skipping the rest is
-/// represented by a 0x00 byte which also ends the string.
-/// 1. when ForStrongLayout is true, following ivars are scanned:
-/// - id, Class
-/// - object *
-/// - __strong anything
-///
-/// 2. When ForStrongLayout is false, following ivars are scanned:
-/// - __weak anything
-///
-llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
-  const ObjCImplementationDecl *OMD,
-  bool ForStrongLayout) {
-  bool hasUnion = false;
-
+/// BuildIvarLayoutBitmap - This routine is the horsework for doing all
+/// the computations and returning the layout bitmap (for ivar or blocks) in
+/// the given argument BitMap string container. Routine reads
+/// two containers, IvarsInfo and SkipIvars which are assumed to be
+/// filled already by the caller.
+llvm::Constant *CGObjCCommonMac::BuildIvarLayoutBitmap(std::string& BitMap) {
   unsigned int WordsToScan, WordsToSkip;
   const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC)
-    return llvm::Constant::getNullValue(PtrTy);
-
-  llvm::SmallVector<FieldDecl*, 32> RecFields;
-  const ObjCInterfaceDecl *OI = OMD->getClassInterface();
-  CGM.getContext().CollectObjCIvars(OI, RecFields);
-
-  // Add this implementations synthesized ivars.
-  llvm::SmallVector<ObjCIvarDecl*, 16> Ivars;
-  CGM.getContext().CollectNonClassIvars(OI, Ivars);
-  for (unsigned k = 0, e = Ivars.size(); k != e; ++k)
-    RecFields.push_back(cast<FieldDecl>(Ivars[k]));
-
-  if (RecFields.empty())
-    return llvm::Constant::getNullValue(PtrTy);
-
-  SkipIvars.clear();
-  IvarsInfo.clear();
-
-  BuildAggrIvarLayout(OMD, 0, 0, RecFields, 0, ForStrongLayout, hasUnion);
-  if (IvarsInfo.empty())
-    return llvm::Constant::getNullValue(PtrTy);
-
-  // Sort on byte position in case we encounterred a union nested in
-  // the ivar list.
-  if (hasUnion && !IvarsInfo.empty())
-    std::sort(IvarsInfo.begin(), IvarsInfo.end());
-  if (hasUnion && !SkipIvars.empty())
-    std::sort(SkipIvars.begin(), SkipIvars.end());
-
+  
   // Build the string of skip/scan nibbles
   llvm::SmallVector<SKIP_SCAN, 32> SkipScanIvars;
   unsigned int WordSize =
-    CGM.getTypes().getTargetData().getTypeAllocSize(PtrTy);
+  CGM.getTypes().getTargetData().getTypeAllocSize(PtrTy);
   if (IvarsInfo[0].ivar_bytepos == 0) {
     WordsToSkip = 0;
     WordsToScan = IvarsInfo[0].ivar_size;
@@ -3512,7 +3745,7 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
   }
   for (unsigned int i=1, Last=IvarsInfo.size(); i != Last; i++) {
     unsigned int TailPrevGCObjC =
-      IvarsInfo[i-1].ivar_bytepos + IvarsInfo[i-1].ivar_size * WordSize;
+    IvarsInfo[i-1].ivar_bytepos + IvarsInfo[i-1].ivar_size * WordSize;
     if (IvarsInfo[i].ivar_bytepos == TailPrevGCObjC) {
       // consecutive 'scanned' object pointers.
       WordsToScan += IvarsInfo[i].ivar_size;
@@ -3526,7 +3759,7 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
       SkScan.skip = WordsToSkip;
       SkScan.scan = WordsToScan;
       SkipScanIvars.push_back(SkScan);
-
+      
       // Skip the hole.
       SkScan.skip = (IvarsInfo[i].ivar_bytepos - TailPrevGCObjC) / WordSize;
       SkScan.scan = 0;
@@ -3541,15 +3774,15 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
     SkScan.scan = WordsToScan;
     SkipScanIvars.push_back(SkScan);
   }
-
+  
   if (!SkipIvars.empty()) {
     unsigned int LastIndex = SkipIvars.size()-1;
     int LastByteSkipped =
-      SkipIvars[LastIndex].ivar_bytepos + SkipIvars[LastIndex].ivar_size;
+    SkipIvars[LastIndex].ivar_bytepos + SkipIvars[LastIndex].ivar_size;
     LastIndex = IvarsInfo.size()-1;
     int LastByteScanned =
-      IvarsInfo[LastIndex].ivar_bytepos +
-      IvarsInfo[LastIndex].ivar_size * WordSize;
+    IvarsInfo[LastIndex].ivar_bytepos +
+    IvarsInfo[LastIndex].ivar_size * WordSize;
     // Compute number of bytes to skip at the tail end of the last ivar scanned.
     if (LastByteSkipped > LastByteScanned) {
       unsigned int TotalWords = (LastByteSkipped + (WordSize -1)) / WordSize;
@@ -3572,20 +3805,19 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
       --SkipScan;
     }
   }
-
+  
   // Generate the string.
-  std::string BitMap;
   for (int i = 0; i <= SkipScan; i++) {
     unsigned char byte;
     unsigned int skip_small = SkipScanIvars[i].skip % 0xf;
     unsigned int scan_small = SkipScanIvars[i].scan % 0xf;
     unsigned int skip_big  = SkipScanIvars[i].skip / 0xf;
     unsigned int scan_big  = SkipScanIvars[i].scan / 0xf;
-
+    
     // first skip big.
     for (unsigned int ix = 0; ix < skip_big; ix++)
       BitMap += (unsigned char)(0xf0);
-
+    
     // next (skip small, scan)
     if (skip_small) {
       byte = skip_small << 4;
@@ -3610,11 +3842,71 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
   // null terminate string.
   unsigned char zero = 0;
   BitMap += zero;
+  
+  llvm::GlobalVariable * Entry =
+  CreateMetadataVar("\01L_OBJC_CLASS_NAME_",
+                    llvm::ConstantArray::get(VMContext, BitMap.c_str()),
+                    "__TEXT,__cstring,cstring_literals",
+                    1, true);
+  return getConstantGEP(VMContext, Entry, 0, 0);
+}
 
-  if (CGM.getLangOptions().ObjCGCBitmapPrint) {
+/// BuildIvarLayout - Builds ivar layout bitmap for the class
+/// implementation for the __strong or __weak case.
+/// The layout map displays which words in ivar list must be skipped
+/// and which must be scanned by GC (see below). String is built of bytes.
+/// Each byte is divided up in two nibbles (4-bit each). Left nibble is count
+/// of words to skip and right nibble is count of words to scan. So, each
+/// nibble represents up to 15 workds to skip or scan. Skipping the rest is
+/// represented by a 0x00 byte which also ends the string.
+/// 1. when ForStrongLayout is true, following ivars are scanned:
+/// - id, Class
+/// - object *
+/// - __strong anything
+///
+/// 2. When ForStrongLayout is false, following ivars are scanned:
+/// - __weak anything
+///
+llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
+  const ObjCImplementationDecl *OMD,
+  bool ForStrongLayout) {
+  bool hasUnion = false;
+
+  const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
+  if (CGM.getLangOptions().getGCMode() == LangOptions::NonGC)
+    return llvm::Constant::getNullValue(PtrTy);
+
+  llvm::SmallVector<ObjCIvarDecl*, 32> Ivars;
+  const ObjCInterfaceDecl *OI = OMD->getClassInterface();
+  CGM.getContext().DeepCollectObjCIvars(OI, true, Ivars);
+
+  llvm::SmallVector<FieldDecl*, 32> RecFields;
+  for (unsigned k = 0, e = Ivars.size(); k != e; ++k)
+    RecFields.push_back(cast<FieldDecl>(Ivars[k]));
+
+  if (RecFields.empty())
+    return llvm::Constant::getNullValue(PtrTy);
+
+  SkipIvars.clear();
+  IvarsInfo.clear();
+
+  BuildAggrIvarLayout(OMD, 0, 0, RecFields, 0, ForStrongLayout, hasUnion);
+  if (IvarsInfo.empty())
+    return llvm::Constant::getNullValue(PtrTy);
+  // Sort on byte position in case we encounterred a union nested in
+  // the ivar list.
+  if (hasUnion && !IvarsInfo.empty())
+    std::sort(IvarsInfo.begin(), IvarsInfo.end());
+  if (hasUnion && !SkipIvars.empty())
+    std::sort(SkipIvars.begin(), SkipIvars.end());
+  
+  std::string BitMap;
+  llvm::Constant *C = BuildIvarLayoutBitmap(BitMap);
+  
+   if (CGM.getLangOptions().ObjCGCBitmapPrint) {
     printf("\n%s ivar layout for class '%s': ",
            ForStrongLayout ? "strong" : "weak",
-           OMD->getClassInterface()->getNameAsCString());
+           OMD->getClassInterface()->getName().data());
     const unsigned char *s = (unsigned char*)BitMap.c_str();
     for (unsigned i = 0; i < BitMap.size(); i++)
       if (!(s[i] & 0xf0))
@@ -3623,12 +3915,7 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
         printf("0x%x%s",  s[i], s[i] != 0 ? ", " : "");
     printf("\n");
   }
-  llvm::GlobalVariable * Entry =
-    CreateMetadataVar("\01L_OBJC_CLASS_NAME_",
-                      llvm::ConstantArray::get(VMContext, BitMap.c_str()),
-                      "__TEXT,__cstring,cstring_literals",
-                      1, true);
-  return getConstantGEP(VMContext, Entry, 0, 0);
+  return C;
 }
 
 llvm::Constant *CGObjCCommonMac::GetMethodVarName(Selector Sel) {
@@ -3649,11 +3936,6 @@ llvm::Constant *CGObjCCommonMac::GetMethodVarName(IdentifierInfo *ID) {
   return GetMethodVarName(CGM.getContext().Selectors.getNullarySelector(ID));
 }
 
-// FIXME: Merge into a single cstring creation function.
-llvm::Constant *CGObjCCommonMac::GetMethodVarName(const std::string &Name) {
-  return GetMethodVarName(&CGM.getContext().Idents.get(Name));
-}
-
 llvm::Constant *CGObjCCommonMac::GetMethodVarType(const FieldDecl *Field) {
   std::string TypeStr;
   CGM.getContext().getObjCEncodingForType(Field->getType(), TypeStr, Field);
@@ -4526,8 +4808,8 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer(
   assert(OID && "CGObjCNonFragileABIMac::BuildClassRoTInitializer");
   Values[ 6] = EmitProtocolList("\01l_OBJC_CLASS_PROTOCOLS_$_"
                                 + OID->getName(),
-                                OID->protocol_begin(),
-                                OID->protocol_end());
+                                OID->all_referenced_protocol_begin(),
+                                OID->all_referenced_protocol_end());
 
   if (flags & CLS_META)
     Values[ 7] = llvm::Constant::getNullValue(ObjCTypes.IvarListnfABIPtrTy);
@@ -4741,7 +5023,7 @@ llvm::Value *CGObjCNonFragileABIMac::GenerateProtocolRef(CGBuilderTy &Builder,
                                    ObjCTypes.ExternalProtocolPtrTy);
 
   std::string ProtocolName("\01l_OBJC_PROTOCOL_REFERENCE_$_");
-  ProtocolName += PD->getNameAsCString();
+  ProtocolName += PD->getName();
 
   llvm::GlobalVariable *PTGV = CGM.getModule().getGlobalVariable(ProtocolName);
   if (PTGV)
@@ -4855,8 +5137,7 @@ void CGObjCNonFragileABIMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
 /// method has not been defined. The return value has type MethodPtrTy.
 llvm::Constant *CGObjCNonFragileABIMac::GetMethodConstant(
   const ObjCMethodDecl *MD) {
-  // FIXME: Use DenseMap::lookup
-  llvm::Function *Fn = MethodDefinitions[MD];
+  llvm::Function *Fn = GetMethodDefinition(MD);
   if (!Fn)
     return 0;
 
@@ -5284,40 +5565,24 @@ CodeGen::RValue CGObjCNonFragileABIMac::EmitMessageSend(
   llvm::Constant *Fn = 0;
   std::string Name("\01l_");
   if (CGM.ReturnTypeUsesSRet(FnInfo)) {
-#if 0
-    // unlike what is documented. gcc never generates this API!!
-    if (Receiver->getType() == ObjCTypes.ObjectPtrTy) {
-      Fn = ObjCTypes.getMessageSendIdStretFixupFn();
-      // FIXME. Is there a better way of getting these names.
-      // They are available in RuntimeFunctions vector pair.
-      Name += "objc_msgSendId_stret_fixup";
-    } else
-#endif
-      if (IsSuper) {
-        Fn = ObjCTypes.getMessageSendSuper2StretFixupFn();
-        Name += "objc_msgSendSuper2_stret_fixup";
-      } else {
-        Fn = ObjCTypes.getMessageSendStretFixupFn();
-        Name += "objc_msgSend_stret_fixup";
-      }
+    if (IsSuper) {
+      Fn = ObjCTypes.getMessageSendSuper2StretFixupFn();
+      Name += "objc_msgSendSuper2_stret_fixup";
+    } else {
+      Fn = ObjCTypes.getMessageSendStretFixupFn();
+      Name += "objc_msgSend_stret_fixup";
+    }
   } else if (!IsSuper && CGM.ReturnTypeUsesFPRet(ResultType)) {
     Fn = ObjCTypes.getMessageSendFpretFixupFn();
     Name += "objc_msgSend_fpret_fixup";
   } else {
-#if 0
-// unlike what is documented. gcc never generates this API!!
-    if (Receiver->getType() == ObjCTypes.ObjectPtrTy) {
-      Fn = ObjCTypes.getMessageSendIdFixupFn();
-      Name += "objc_msgSendId_fixup";
-    } else
-#endif
-      if (IsSuper) {
-        Fn = ObjCTypes.getMessageSendSuper2FixupFn();
-        Name += "objc_msgSendSuper2_fixup";
-      } else {
-        Fn = ObjCTypes.getMessageSendFixupFn();
-        Name += "objc_msgSend_fixup";
-      }
+    if (IsSuper) {
+      Fn = ObjCTypes.getMessageSendSuper2FixupFn();
+      Name += "objc_msgSendSuper2_fixup";
+    } else {
+      Fn = ObjCTypes.getMessageSendFixupFn();
+      Name += "objc_msgSend_fixup";
+    }
   }
   assert(Fn && "CGObjCNonFragileABIMac::EmitMessageSend");
   Name += '_';
@@ -5647,7 +5912,8 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
 /// objc_assign_global (id src, id *dst)
 ///
 void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                                  llvm::Value *src, llvm::Value *dst) {
+                                          llvm::Value *src, llvm::Value *dst,
+                                          bool threadlocal) {
   const llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
     unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
@@ -5658,11 +5924,28 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
   }
   src = CGF.Builder.CreateBitCast(src, ObjCTypes.ObjectPtrTy);
   dst = CGF.Builder.CreateBitCast(dst, ObjCTypes.PtrObjectPtrTy);
-  CGF.Builder.CreateCall2(ObjCTypes.getGcAssignGlobalFn(),
-                          src, dst, "globalassign");
+  if (!threadlocal)
+    CGF.Builder.CreateCall2(ObjCTypes.getGcAssignGlobalFn(),
+                            src, dst, "globalassign");
+  else
+    CGF.Builder.CreateCall2(ObjCTypes.getGcAssignThreadLocalFn(),
+                            src, dst, "threadlocalassign");
   return;
 }
 
+namespace {
+  struct CallSyncExit : EHScopeStack::Cleanup {
+    llvm::Value *SyncExitFn;
+    llvm::Value *SyncArg;
+    CallSyncExit(llvm::Value *SyncExitFn, llvm::Value *SyncArg)
+      : SyncExitFn(SyncExitFn), SyncArg(SyncArg) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEHCleanup) {
+      CGF.Builder.CreateCall(SyncExitFn, SyncArg)->setDoesNotThrow();
+    }
+  };
+}
+
 void
 CGObjCNonFragileABIMac::EmitSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
                                              const ObjCAtSynchronizedStmt &S) {
@@ -5675,12 +5958,9 @@ CGObjCNonFragileABIMac::EmitSynchronizedStmt(CodeGen::CodeGenFunction &CGF,
     ->setDoesNotThrow();
 
   // Register an all-paths cleanup to release the lock.
-  {
-    CodeGenFunction::CleanupBlock ReleaseScope(CGF, NormalAndEHCleanup);
-
-    CGF.Builder.CreateCall(ObjCTypes.getSyncExitFn(), SyncArg)
-      ->setDoesNotThrow();
-  }
+  CGF.EHStack.pushCleanup<CallSyncExit>(NormalAndEHCleanup,
+                                        ObjCTypes.getSyncExitFn(),
+                                        SyncArg);
 
   // Emit the body of the statement.
   CGF.EmitStmt(S.getSynchBody());
@@ -5697,7 +5977,7 @@ namespace {
     llvm::Value *TypeInfo;
   };
 
-  struct CallObjCEndCatch : EHScopeStack::LazyCleanup {
+  struct CallObjCEndCatch : EHScopeStack::Cleanup {
     CallObjCEndCatch(bool MightThrow, llvm::Value *Fn) :
       MightThrow(MightThrow), Fn(Fn) {}
     bool MightThrow;
@@ -5714,6 +5994,31 @@ namespace {
   };
 }
 
+llvm::Constant *
+CGObjCNonFragileABIMac::GetEHType(QualType T) {
+  // There's a particular fixed type info for 'id'.
+  if (T->isObjCIdType() ||
+      T->isObjCQualifiedIdType()) {
+    llvm::Constant *IDEHType =
+      CGM.getModule().getGlobalVariable("OBJC_EHTYPE_id");
+    if (!IDEHType)
+      IDEHType =
+        new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.EHTypeTy,
+                                 false,
+                                 llvm::GlobalValue::ExternalLinkage,
+                                 0, "OBJC_EHTYPE_id");
+    return IDEHType;
+  }
+
+  // All other types should be Objective-C interface pointer types.
+  const ObjCObjectPointerType *PT =
+    T->getAs<ObjCObjectPointerType>();
+  assert(PT && "Invalid @catch type.");
+  const ObjCInterfaceType *IT = PT->getInterfaceType();
+  assert(IT && "Invalid @catch type.");
+  return GetInterfaceEHType(IT->getDecl(), false);
+}                                                  
+
 void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
                                          const ObjCAtTryStmt &S) {
   // Jump destination for falling out of catch bodies.
@@ -5749,27 +6054,7 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
         break;
       }
 
-      // There's a particular fixed type info for 'id'.
-      if (CatchDecl->getType()->isObjCIdType() ||
-          CatchDecl->getType()->isObjCQualifiedIdType()) {
-        llvm::Value *IDEHType =
-          CGM.getModule().getGlobalVariable("OBJC_EHTYPE_id");
-        if (!IDEHType)
-          IDEHType =
-            new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.EHTypeTy,
-                                     false,
-                                     llvm::GlobalValue::ExternalLinkage,
-                                     0, "OBJC_EHTYPE_id");
-        Handler.TypeInfo = IDEHType;
-      } else {
-        // All other types should be Objective-C interface pointer types.
-        const ObjCObjectPointerType *PT =
-          CatchDecl->getType()->getAs<ObjCObjectPointerType>();
-        assert(PT && "Invalid @catch type.");
-        const ObjCInterfaceType *IT = PT->getInterfaceType();
-        assert(IT && "Invalid @catch type.");
-        Handler.TypeInfo = GetInterfaceEHType(IT->getDecl(), false);
-      }
+      Handler.TypeInfo = GetEHType(CatchDecl->getType());
     }
 
     EHCatchScope *Catch = CGF.EHStack.pushCatch(Handlers.size());
@@ -5802,9 +6087,9 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
 
     // Add a cleanup to leave the catch.
     bool EndCatchMightThrow = (Handler.Variable == 0);
-    CGF.EHStack.pushLazyCleanup<CallObjCEndCatch>(NormalAndEHCleanup,
-                                                  EndCatchMightThrow,
-                                                  ObjCTypes.getObjCEndCatchFn());
+    CGF.EHStack.pushCleanup<CallObjCEndCatch>(NormalAndEHCleanup,
+                                              EndCatchMightThrow,
+                                              ObjCTypes.getObjCEndCatchFn());
 
     // Bind the catch parameter if it exists.
     if (const VarDecl *CatchParam = Handler.Variable) {
@@ -5832,8 +6117,8 @@ void CGObjCNonFragileABIMac::EmitTryStmt(CodeGen::CodeGenFunction &CGF,
   if (S.getFinallyStmt())
     CGF.ExitFinallyBlock(FinallyInfo);
 
-  if (Cont.Block)
-    CGF.EmitBlock(Cont.Block);
+  if (Cont.isValid())
+    CGF.EmitBlock(Cont.getBlock());
 }
 
 /// EmitThrowStmt - Generate code for a throw statement.
@@ -5868,7 +6153,7 @@ void CGObjCNonFragileABIMac::EmitThrowStmt(CodeGen::CodeGenFunction &CGF,
   CGF.Builder.ClearInsertionPoint();
 }
 
-llvm::Value *
+llvm::Constant *
 CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID,
                                            bool ForDefinition) {
   llvm::GlobalVariable * &Entry = EHTypeReferences[ID->getIdentifier()];
diff --git a/lib/CodeGen/CGObjCRuntime.h b/lib/CodeGen/CGObjCRuntime.h
index eb79f09de13f..584760f6f343 100644
--- a/lib/CodeGen/CGObjCRuntime.h
+++ b/lib/CodeGen/CGObjCRuntime.h
@@ -52,6 +52,7 @@ namespace CodeGen {
   class Selector;
   class ObjCIvarDecl;
   class ObjCStringLiteral;
+  class BlockDeclRefExpr;
 
 namespace CodeGen {
   class CodeGenModule;
@@ -103,6 +104,12 @@ public:
   virtual llvm::Value *GetSelector(CGBuilderTy &Builder,
                                    const ObjCMethodDecl *Method) = 0;
 
+  /// Get the type constant to catch for the given ObjC pointer type.
+  /// This is used externally to implement catching ObjC types in C++.
+  /// Runtimes which don't support this should add the appropriate
+  /// error to Sema.
+  virtual llvm::Constant *GetEHType(QualType T) = 0;
+
   /// Generate a constant string object.
   virtual llvm::Constant *GenerateConstantString(const StringLiteral *) = 0;
 
@@ -192,7 +199,8 @@ public:
   virtual void EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dest) = 0;
   virtual void EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
-                                    llvm::Value *src, llvm::Value *dest) = 0;
+                                    llvm::Value *src, llvm::Value *dest,
+                                    bool threadlocal=false) = 0;
   virtual void EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                   llvm::Value *src, llvm::Value *dest,
                                   llvm::Value *ivarOffset) = 0;
@@ -211,6 +219,9 @@ public:
                                         llvm::Value *DestPtr,
                                         llvm::Value *SrcPtr,
                                         llvm::Value *Size) = 0;
+  virtual llvm::Constant *GCBlockLayout(CodeGen::CodeGenFunction &CGF,
+                  const llvm::SmallVectorImpl<const BlockDeclRefExpr *> &) = 0;
+                                        
 };
 
 /// Creates an instance of an Objective-C runtime class.
diff --git a/lib/CodeGen/CGRTTI.cpp b/lib/CodeGen/CGRTTI.cpp
index 1cca97702ddd..60df613f65e8 100644
--- a/lib/CodeGen/CGRTTI.cpp
+++ b/lib/CodeGen/CGRTTI.cpp
@@ -11,9 +11,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/Type.h"
-#include "clang/AST/RecordLayout.h"
 #include "CodeGenModule.h"
+#include "CGCXXABI.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/AST/Type.h"
+#include "clang/Frontend/CodeGenOptions.h"
+
 using namespace clang;
 using namespace CodeGen;
 
@@ -45,7 +48,11 @@ class RTTIBuilder {
   
   /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct, used
   /// for pointer types.
-  void BuildPointerTypeInfo(const PointerType *Ty);
+  void BuildPointerTypeInfo(QualType PointeeTy);
+
+  /// BuildObjCObjectTypeInfo - Build the appropriate kind of
+  /// type_info for an object type.
+  void BuildObjCObjectTypeInfo(const ObjCObjectType *Ty);
   
   /// BuildPointerToMemberTypeInfo - Build an abi::__pointer_to_member_type_info 
   /// struct, used for member pointer types.
@@ -59,7 +66,7 @@ public:
   llvm::Constant *BuildName(QualType Ty, bool Hidden, 
                             llvm::GlobalVariable::LinkageTypes Linkage) {
     llvm::SmallString<256> OutName;
-    CGM.getMangleContext().mangleCXXRTTIName(Ty, OutName);
+    CGM.getCXXABI().getMangleContext().mangleCXXRTTIName(Ty, OutName);
     llvm::StringRef Name = OutName.str();
 
     llvm::GlobalVariable *OGV = CGM.getModule().getNamedGlobal(Name);
@@ -158,7 +165,7 @@ public:
 llvm::Constant *RTTIBuilder::GetAddrOfExternalRTTIDescriptor(QualType Ty) {
   // Mangle the RTTI name.
   llvm::SmallString<256> OutName;
-  CGM.getMangleContext().mangleCXXRTTI(Ty, OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, OutName);
   llvm::StringRef Name = OutName.str();
 
   // Look for an existing global.
@@ -244,11 +251,9 @@ static bool TypeInfoIsInStandardLibrary(const PointerType *PointerTy) {
   return TypeInfoIsInStandardLibrary(BuiltinTy);
 }
 
-/// ShouldUseExternalRTTIDescriptor - Returns whether the type information for
-/// the given type exists somewhere else, and that we should not emit the type
-/// information in this translation unit.
-static bool ShouldUseExternalRTTIDescriptor(ASTContext &Context,
-                                            QualType Ty) {
+/// IsStandardLibraryRTTIDescriptor - Returns whether the type
+/// information for the given type exists in the standard library.
+static bool IsStandardLibraryRTTIDescriptor(QualType Ty) {
   // Type info for builtin types is defined in the standard library.
   if (const BuiltinType *BuiltinTy = dyn_cast<BuiltinType>(Ty))
     return TypeInfoIsInStandardLibrary(BuiltinTy);
@@ -258,6 +263,15 @@ static bool ShouldUseExternalRTTIDescriptor(ASTContext &Context,
   if (const PointerType *PointerTy = dyn_cast<PointerType>(Ty))
     return TypeInfoIsInStandardLibrary(PointerTy);
 
+  return false;
+}
+
+/// ShouldUseExternalRTTIDescriptor - Returns whether the type information for
+/// the given type exists somewhere else, and that we should not emit the type
+/// information in this translation unit.  Assumes that it is not a
+/// standard-library type.
+static bool ShouldUseExternalRTTIDescriptor(ASTContext &Context,
+                                            QualType Ty) {
   // If RTTI is disabled, don't consider key functions.
   if (!Context.getLangOptions().RTTI) return false;
 
@@ -270,7 +284,7 @@ static bool ShouldUseExternalRTTIDescriptor(ASTContext &Context,
       return false;
 
     // Get the key function.
-    const CXXMethodDecl *KeyFunction = RD->getASTContext().getKeyFunction(RD);
+    const CXXMethodDecl *KeyFunction = Context.getKeyFunction(RD);
     if (KeyFunction && !KeyFunction->hasBody()) {
       // The class has a key function, but it is not defined in this translation
       // unit, so we should use the external descriptor for it.
@@ -383,21 +397,45 @@ static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
 }
 
 void RTTIBuilder::BuildVTablePointer(const Type *Ty) {
-  const char *VTableName;
+  // abi::__class_type_info.
+  static const char * const ClassTypeInfo =
+    "_ZTVN10__cxxabiv117__class_type_infoE";
+  // abi::__si_class_type_info.
+  static const char * const SIClassTypeInfo =
+    "_ZTVN10__cxxabiv120__si_class_type_infoE";
+  // abi::__vmi_class_type_info.
+  static const char * const VMIClassTypeInfo =
+    "_ZTVN10__cxxabiv121__vmi_class_type_infoE";
+
+  const char *VTableName = 0;
 
   switch (Ty->getTypeClass()) {
-  default: assert(0 && "Unhandled type!");
+#define TYPE(Class, Base)
+#define ABSTRACT_TYPE(Class, Base)
+#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
+#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
+#define DEPENDENT_TYPE(Class, Base) case Type::Class:
+#include "clang/AST/TypeNodes.def"
+    assert(false && "Non-canonical and dependent types shouldn't get here");
+
+  case Type::LValueReference:
+  case Type::RValueReference:
+    assert(false && "References shouldn't get here");
 
   case Type::Builtin:
-  // GCC treats vector types as fundamental types.
+  // GCC treats vector and complex types as fundamental types.
   case Type::Vector:
   case Type::ExtVector:
+  case Type::Complex:
+  // FIXME: GCC treats block pointers as fundamental types?!
+  case Type::BlockPointer:
     // abi::__fundamental_type_info.
     VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE";
     break;
 
   case Type::ConstantArray:
   case Type::IncompleteArray:
+  case Type::VariableArray:
     // abi::__array_type_info.
     VTableName = "_ZTVN10__cxxabiv117__array_type_infoE";
     break;
@@ -412,25 +450,44 @@ void RTTIBuilder::BuildVTablePointer(const Type *Ty) {
     // abi::__enum_type_info.
     VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE";
     break;
-      
+
   case Type::Record: {
     const CXXRecordDecl *RD = 
       cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
     
     if (!RD->hasDefinition() || !RD->getNumBases()) {
-      // abi::__class_type_info.
-      VTableName = "_ZTVN10__cxxabiv117__class_type_infoE";
+      VTableName = ClassTypeInfo;
     } else if (CanUseSingleInheritance(RD)) {
-      // abi::__si_class_type_info.
-      VTableName = "_ZTVN10__cxxabiv120__si_class_type_infoE";
+      VTableName = SIClassTypeInfo;
     } else {
-      // abi::__vmi_class_type_info.
-      VTableName = "_ZTVN10__cxxabiv121__vmi_class_type_infoE";
+      VTableName = VMIClassTypeInfo;
     }
     
     break;
   }
 
+  case Type::ObjCObject:
+    // Ignore protocol qualifiers.
+    Ty = cast<ObjCObjectType>(Ty)->getBaseType().getTypePtr();
+
+    // Handle id and Class.
+    if (isa<BuiltinType>(Ty)) {
+      VTableName = ClassTypeInfo;
+      break;
+    }
+
+    assert(isa<ObjCInterfaceType>(Ty));
+    // Fall through.
+
+  case Type::ObjCInterface:
+    if (cast<ObjCInterfaceType>(Ty)->getDecl()->getSuperClass()) {
+      VTableName = SIClassTypeInfo;
+    } else {
+      VTableName = ClassTypeInfo;
+    }
+    break;
+
+  case Type::ObjCObjectPointer:
   case Type::Pointer:
     // abi::__pointer_type_info.
     VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE";
@@ -456,45 +513,64 @@ void RTTIBuilder::BuildVTablePointer(const Type *Ty) {
   Fields.push_back(VTable);
 }
 
-llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty,
-                                           bool Force) {
+llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty, bool Force) {
   // We want to operate on the canonical type.
   Ty = CGM.getContext().getCanonicalType(Ty);
 
   // Check if we've already emitted an RTTI descriptor for this type.
   llvm::SmallString<256> OutName;
-  CGM.getMangleContext().mangleCXXRTTI(Ty, OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty, OutName);
   llvm::StringRef Name = OutName.str();
   
   llvm::GlobalVariable *OldGV = CGM.getModule().getNamedGlobal(Name);
   if (OldGV && !OldGV->isDeclaration())
     return llvm::ConstantExpr::getBitCast(OldGV, Int8PtrTy);
-  
+
   // Check if there is already an external RTTI descriptor for this type.
-  if (!Force && ShouldUseExternalRTTIDescriptor(CGM.getContext(), Ty))
+  bool IsStdLib = IsStandardLibraryRTTIDescriptor(Ty);
+  if (!Force &&
+      (IsStdLib || ShouldUseExternalRTTIDescriptor(CGM.getContext(), Ty)))
     return GetAddrOfExternalRTTIDescriptor(Ty);
 
-  llvm::GlobalVariable::LinkageTypes Linkage = getTypeInfoLinkage(Ty);
+  // Emit the standard library with external linkage.
+  llvm::GlobalVariable::LinkageTypes Linkage;
+  if (IsStdLib)
+    Linkage = llvm::GlobalValue::ExternalLinkage;
+  else
+    Linkage = getTypeInfoLinkage(Ty);
 
   // Add the vtable pointer.
   BuildVTablePointer(cast<Type>(Ty));
   
   // And the name.
   Fields.push_back(BuildName(Ty, DecideHidden(Ty), Linkage));
-  
+
   switch (Ty->getTypeClass()) {
-  default: assert(false && "Unhandled type class!");
+#define TYPE(Class, Base)
+#define ABSTRACT_TYPE(Class, Base)
+#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
+#define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
+#define DEPENDENT_TYPE(Class, Base) case Type::Class:
+#include "clang/AST/TypeNodes.def"
+    assert(false && "Non-canonical and dependent types shouldn't get here");
 
   // GCC treats vector types as fundamental types.
   case Type::Builtin:
   case Type::Vector:
   case Type::ExtVector:
+  case Type::Complex:
+  case Type::BlockPointer:
     // Itanium C++ ABI 2.9.5p4:
     // abi::__fundamental_type_info adds no data members to std::type_info.
     break;
-      
+
+  case Type::LValueReference:
+  case Type::RValueReference:
+    assert(false && "References shouldn't get here");
+
   case Type::ConstantArray:
   case Type::IncompleteArray:
+  case Type::VariableArray:
     // Itanium C++ ABI 2.9.5p5:
     // abi::__array_type_info adds no data members to std::type_info.
     break;
@@ -525,11 +601,20 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty,
 
     break;
   }
+
+  case Type::ObjCObject:
+  case Type::ObjCInterface:
+    BuildObjCObjectTypeInfo(cast<ObjCObjectType>(Ty));
+    break;
+
+  case Type::ObjCObjectPointer:
+    BuildPointerTypeInfo(cast<ObjCObjectPointerType>(Ty)->getPointeeType());
+    break; 
       
   case Type::Pointer:
-    BuildPointerTypeInfo(cast<PointerType>(Ty));
+    BuildPointerTypeInfo(cast<PointerType>(Ty)->getPointeeType());
     break;
-  
+
   case Type::MemberPointer:
     BuildPointerToMemberTypeInfo(cast<MemberPointerType>(Ty));
     break;
@@ -551,7 +636,18 @@ llvm::Constant *RTTIBuilder::BuildTypeInfo(QualType Ty,
     OldGV->replaceAllUsesWith(NewPtr);
     OldGV->eraseFromParent();
   }
-    
+
+  // GCC only relies on the uniqueness of the type names, not the
+  // type_infos themselves, so we can emit these as hidden symbols.
+  // But don't do this if we're worried about strict visibility
+  // compatibility.
+  if (const RecordType *RT = dyn_cast<RecordType>(Ty))
+    CGM.setTypeVisibility(GV, cast<CXXRecordDecl>(RT->getDecl()),
+                          /*ForRTTI*/ true);
+  else if (CGM.getCodeGenOpts().HiddenWeakVTables &&
+           Linkage == llvm::GlobalValue::WeakODRLinkage)
+    GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
+  
   return llvm::ConstantExpr::getBitCast(GV, Int8PtrTy);
 }
 
@@ -570,6 +666,30 @@ static unsigned ComputeQualifierFlags(Qualifiers Quals) {
   return Flags;
 }
 
+/// BuildObjCObjectTypeInfo - Build the appropriate kind of type_info
+/// for the given Objective-C object type.
+void RTTIBuilder::BuildObjCObjectTypeInfo(const ObjCObjectType *OT) {
+  // Drop qualifiers.
+  const Type *T = OT->getBaseType().getTypePtr();
+  assert(isa<BuiltinType>(T) || isa<ObjCInterfaceType>(T));
+
+  // The builtin types are abi::__class_type_infos and don't require
+  // extra fields.
+  if (isa<BuiltinType>(T)) return;
+
+  ObjCInterfaceDecl *Class = cast<ObjCInterfaceType>(T)->getDecl();
+  ObjCInterfaceDecl *Super = Class->getSuperClass();
+
+  // Root classes are also __class_type_info.
+  if (!Super) return;
+
+  QualType SuperTy = CGM.getContext().getObjCInterfaceType(Super);
+
+  // Everything else is single inheritance.
+  llvm::Constant *BaseTypeInfo = RTTIBuilder(CGM).BuildTypeInfo(SuperTy);
+  Fields.push_back(BaseTypeInfo);
+}
+
 /// BuildSIClassTypeInfo - Build an abi::__si_class_type_info, used for single
 /// inheritance, according to the Itanium C++ ABI, 2.95p6b.
 void RTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
@@ -677,7 +797,7 @@ void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
   //   direct proper base. Each description is of the type:
   //
   //   struct abi::__base_class_type_info {
-	//   public:
+  //   public:
   //     const __class_type_info *__base_type;
   //     long __offset_flags;
   //
@@ -725,9 +845,7 @@ void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
 
 /// BuildPointerTypeInfo - Build an abi::__pointer_type_info struct,
 /// used for pointer types.
-void RTTIBuilder::BuildPointerTypeInfo(const PointerType *Ty) {
-  QualType PointeeTy = Ty->getPointeeType();
-  
+void RTTIBuilder::BuildPointerTypeInfo(QualType PointeeTy) {  
   Qualifiers Quals;
   QualType UnqualifiedPointeeTy = 
     CGM.getContext().getUnqualifiedArrayType(PointeeTy, Quals);
diff --git a/lib/CodeGen/CGRecordLayout.h b/lib/CodeGen/CGRecordLayout.h
index e95591e5cc01..9b4e9f86c6da 100644
--- a/lib/CodeGen/CGRecordLayout.h
+++ b/lib/CodeGen/CGRecordLayout.h
@@ -144,6 +144,21 @@ public:
 
   void print(llvm::raw_ostream &OS) const;
   void dump() const;
+
+  /// \brief Given a bit-field decl, build an appropriate helper object for
+  /// accessing that field (which is expected to have the given offset and
+  /// size).
+  static CGBitFieldInfo MakeInfo(class CodeGenTypes &Types, const FieldDecl *FD,
+                                 uint64_t FieldOffset, uint64_t FieldSize);
+
+  /// \brief Given a bit-field decl, build an appropriate helper object for
+  /// accessing that field (which is expected to have the given offset and
+  /// size). The field decl should be known to be contained within a type of at
+  /// least the given size and with the given alignment.
+  static CGBitFieldInfo MakeInfo(CodeGenTypes &Types, const FieldDecl *FD,
+                                 uint64_t FieldOffset, uint64_t FieldSize,
+                                 uint64_t ContainingTypeSizeInBits,
+                                 unsigned ContainingTypeAlign);
 };
 
 /// CGRecordLayout - This class handles struct and union layout info while
@@ -174,20 +189,21 @@ private:
 
   /// Whether one of the fields in this record layout is a pointer to data
   /// member, or a struct that contains pointer to data member.
-  bool ContainsPointerToDataMember : 1;
+  bool IsZeroInitializable : 1;
 
 public:
-  CGRecordLayout(const llvm::Type *T, bool ContainsPointerToDataMember)
-    : LLVMType(T), ContainsPointerToDataMember(ContainsPointerToDataMember) {}
+  CGRecordLayout(const llvm::Type *T, bool IsZeroInitializable)
+    : LLVMType(T), IsZeroInitializable(IsZeroInitializable) {}
 
   /// \brief Return the LLVM type associated with this record.
   const llvm::Type *getLLVMType() const {
     return LLVMType;
   }
 
-  /// \brief Check whether this struct contains pointers to data members.
-  bool containsPointerToDataMember() const {
-    return ContainsPointerToDataMember;
+  /// \brief Check whether this struct can be C++ zero-initialized
+  /// with a zeroinitializer.
+  bool isZeroInitializable() const {
+    return IsZeroInitializable;
   }
 
   /// \brief Return llvm::StructType element number that corresponds to the
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 9f1687577c3e..77a319fa3ab1 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -18,6 +18,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/RecordLayout.h"
 #include "CodeGenTypes.h"
+#include "CGCXXABI.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Type.h"
 #include "llvm/Support/Debug.h"
@@ -45,10 +46,9 @@ public:
   typedef std::pair<const CXXRecordDecl *, unsigned> LLVMBaseInfo;
   llvm::SmallVector<LLVMBaseInfo, 16> LLVMNonVirtualBases;
   
-  /// ContainsPointerToDataMember - Whether one of the fields in this record
-  /// layout is a pointer to data member, or a struct that contains pointer to
-  /// data member.
-  bool ContainsPointerToDataMember;
+  /// IsZeroInitializable - Whether this struct can be C++
+  /// zero-initialized with an LLVM zeroinitializer.
+  bool IsZeroInitializable;
 
   /// Packed - Whether the resulting LLVM struct will be packed or not.
   bool Packed;
@@ -115,14 +115,14 @@ private:
 
   unsigned getTypeAlignment(const llvm::Type *Ty) const;
 
-  /// CheckForPointerToDataMember - Check if the given type contains a pointer
+  /// CheckZeroInitializable - Check if the given type contains a pointer
   /// to data member.
-  void CheckForPointerToDataMember(QualType T);
-  void CheckForPointerToDataMember(const CXXRecordDecl *RD);
+  void CheckZeroInitializable(QualType T);
+  void CheckZeroInitializable(const CXXRecordDecl *RD);
 
 public:
   CGRecordLayoutBuilder(CodeGenTypes &Types)
-    : ContainsPointerToDataMember(false), Packed(false), Types(Types),
+    : IsZeroInitializable(true), Packed(false), Types(Types),
       Alignment(0), AlignmentAsLLVMStruct(1),
       BitsAvailableInLastField(0), NextFieldOffsetInBytes(0) { }
 
@@ -157,15 +157,12 @@ void CGRecordLayoutBuilder::Layout(const RecordDecl *D) {
   LayoutFields(D);
 }
 
-static CGBitFieldInfo ComputeBitFieldInfo(CodeGenTypes &Types,
-                                          const FieldDecl *FD,
-                                          uint64_t FieldOffset,
-                                          uint64_t FieldSize) {
-  const RecordDecl *RD = FD->getParent();
-  const ASTRecordLayout &RL = Types.getContext().getASTRecordLayout(RD);
-  uint64_t ContainingTypeSizeInBits = RL.getSize();
-  unsigned ContainingTypeAlign = RL.getAlignment();
-
+CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
+                               const FieldDecl *FD,
+                               uint64_t FieldOffset,
+                               uint64_t FieldSize,
+                               uint64_t ContainingTypeSizeInBits,
+                               unsigned ContainingTypeAlign) {
   const llvm::Type *Ty = Types.ConvertTypeForMemRecursive(FD->getType());
   uint64_t TypeSizeInBytes = Types.getTargetData().getTypeAllocSize(Ty);
   uint64_t TypeSizeInBits = TypeSizeInBytes * 8;
@@ -255,6 +252,19 @@ static CGBitFieldInfo ComputeBitFieldInfo(CodeGenTypes &Types,
   return CGBitFieldInfo(FieldSize, NumComponents, Components, IsSigned);
 }
 
+CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
+                                        const FieldDecl *FD,
+                                        uint64_t FieldOffset,
+                                        uint64_t FieldSize) {
+  const RecordDecl *RD = FD->getParent();
+  const ASTRecordLayout &RL = Types.getContext().getASTRecordLayout(RD);
+  uint64_t ContainingTypeSizeInBits = RL.getSize();
+  unsigned ContainingTypeAlign = RL.getAlignment();
+
+  return MakeInfo(Types, FD, FieldOffset, FieldSize, ContainingTypeSizeInBits,
+                  ContainingTypeAlign);
+}
+
 void CGRecordLayoutBuilder::LayoutBitField(const FieldDecl *D,
                                            uint64_t FieldOffset) {
   uint64_t FieldSize =
@@ -287,7 +297,8 @@ void CGRecordLayoutBuilder::LayoutBitField(const FieldDecl *D,
 
   // Add the bit field info.
   LLVMBitFields.push_back(
-    LLVMBitFieldInfo(D, ComputeBitFieldInfo(Types, D, FieldOffset, FieldSize)));
+    LLVMBitFieldInfo(D, CGBitFieldInfo::MakeInfo(Types, D, FieldOffset,
+                                                 FieldSize)));
 
   AppendBytes(NumBytesToAppend);
 
@@ -311,8 +322,7 @@ bool CGRecordLayoutBuilder::LayoutField(const FieldDecl *D,
     return true;
   }
 
-  // Check if we have a pointer to data member in this field.
-  CheckForPointerToDataMember(D->getType());
+  CheckZeroInitializable(D->getType());
 
   assert(FieldOffset % 8 == 0 && "FieldOffset is not on a byte boundary!");
   uint64_t FieldOffsetInBytes = FieldOffset / 8;
@@ -380,7 +390,8 @@ CGRecordLayoutBuilder::LayoutUnionField(const FieldDecl *Field,
 
     // Add the bit field info.
     LLVMBitFields.push_back(
-      LLVMBitFieldInfo(Field, ComputeBitFieldInfo(Types, Field, 0, FieldSize)));
+      LLVMBitFieldInfo(Field, CGBitFieldInfo::MakeInfo(Types, Field,
+                                                       0, FieldSize)));
     return FieldTy;
   }
 
@@ -458,7 +469,7 @@ void CGRecordLayoutBuilder::LayoutNonVirtualBase(const CXXRecordDecl *BaseDecl,
     return;
   }
 
-  CheckForPointerToDataMember(BaseDecl);
+  CheckZeroInitializable(BaseDecl);
 
   // FIXME: Actually use a better type than [sizeof(BaseDecl) x i8] when we can.
   AppendPadding(BaseOffset / 8, 1);
@@ -603,9 +614,9 @@ unsigned CGRecordLayoutBuilder::getTypeAlignment(const llvm::Type *Ty) const {
   return Types.getTargetData().getABITypeAlignment(Ty);
 }
 
-void CGRecordLayoutBuilder::CheckForPointerToDataMember(QualType T) {
+void CGRecordLayoutBuilder::CheckZeroInitializable(QualType T) {
   // This record already contains a member pointer.
-  if (ContainsPointerToDataMember)
+  if (!IsZeroInitializable)
     return;
 
   // Can only have member pointers if we're compiling C++.
@@ -615,21 +626,17 @@ void CGRecordLayoutBuilder::CheckForPointerToDataMember(QualType T) {
   T = Types.getContext().getBaseElementType(T);
 
   if (const MemberPointerType *MPT = T->getAs<MemberPointerType>()) {
-    if (!MPT->getPointeeType()->isFunctionType()) {
-      // We have a pointer to data member.
-      ContainsPointerToDataMember = true;
-    }
+    if (!Types.getCXXABI().isZeroInitializable(MPT))
+      IsZeroInitializable = false;
   } else if (const RecordType *RT = T->getAs<RecordType>()) {
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-
-    return CheckForPointerToDataMember(RD);
+    CheckZeroInitializable(RD);
   }
 }
 
-void
-CGRecordLayoutBuilder::CheckForPointerToDataMember(const CXXRecordDecl *RD) {
+void CGRecordLayoutBuilder::CheckZeroInitializable(const CXXRecordDecl *RD) {
   // This record already contains a member pointer.
-  if (ContainsPointerToDataMember)
+  if (!IsZeroInitializable)
     return;
 
   // FIXME: It would be better if there was a way to explicitly compute the
@@ -638,8 +645,8 @@ CGRecordLayoutBuilder::CheckForPointerToDataMember(const CXXRecordDecl *RD) {
   
   const CGRecordLayout &Layout = Types.getCGRecordLayout(RD);
   
-  if (Layout.containsPointerToDataMember())
-    ContainsPointerToDataMember = true;
+  if (!Layout.isZeroInitializable())
+    IsZeroInitializable = false;
 }
 
 CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D) {
@@ -652,7 +659,7 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D) {
                                                Builder.Packed);
 
   CGRecordLayout *RL =
-    new CGRecordLayout(Ty, Builder.ContainsPointerToDataMember);
+    new CGRecordLayout(Ty, Builder.IsZeroInitializable);
 
   // Add all the non-virtual base field numbers.
   RL->NonVirtualBaseFields.insert(Builder.LLVMNonVirtualBases.begin(),
@@ -723,7 +730,7 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D) {
 void CGRecordLayout::print(llvm::raw_ostream &OS) const {
   OS << "<CGRecordLayout\n";
   OS << "  LLVMType:" << *LLVMType << "\n";
-  OS << "  ContainsPointerToDataMember:" << ContainsPointerToDataMember << "\n";
+  OS << "  IsZeroInitializable:" << IsZeroInitializable << "\n";
   OS << "  BitFields:[\n";
 
   // Print bit-field infos in declaration order.
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index b72725edca7e..16145f766af2 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -34,7 +34,8 @@ void CodeGenFunction::EmitStopPoint(const Stmt *S) {
       DI->setLocation(S->getLocEnd());
     else
       DI->setLocation(S->getLocStart());
-    DI->EmitStopPoint(CurFn, Builder);
+    DI->UpdateLineDirectiveRegion(Builder);
+    DI->EmitStopPoint(Builder);
   }
 }
 
@@ -152,7 +153,7 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
   CGDebugInfo *DI = getDebugInfo();
   if (DI) {
     DI->setLocation(S.getLBracLoc());
-    DI->EmitRegionStart(CurFn, Builder);
+    DI->EmitRegionStart(Builder);
   }
 
   // Keep track of the current cleanup stack depth.
@@ -164,7 +165,7 @@ RValue CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
 
   if (DI) {
     DI->setLocation(S.getRBracLoc());
-    DI->EmitRegionEnd(CurFn, Builder);
+    DI->EmitRegionEnd(Builder);
   }
 
   RValue RV;
@@ -247,32 +248,35 @@ void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
 CodeGenFunction::JumpDest
 CodeGenFunction::getJumpDestForLabel(const LabelStmt *S) {
   JumpDest &Dest = LabelMap[S];
-  if (Dest.Block) return Dest;
+  if (Dest.isValid()) return Dest;
 
   // Create, but don't insert, the new block.
-  Dest.Block = createBasicBlock(S->getName());
-  Dest.ScopeDepth = EHScopeStack::stable_iterator::invalid();
+  Dest = JumpDest(createBasicBlock(S->getName()),
+                  EHScopeStack::stable_iterator::invalid(),
+                  NextCleanupDestIndex++);
   return Dest;
 }
 
 void CodeGenFunction::EmitLabel(const LabelStmt &S) {
   JumpDest &Dest = LabelMap[&S];
 
-  // If we didn't needed a forward reference to this label, just go
+  // If we didn't need a forward reference to this label, just go
   // ahead and create a destination at the current scope.
-  if (!Dest.Block) {
+  if (!Dest.isValid()) {
     Dest = getJumpDestInCurrentScope(S.getName());
 
   // Otherwise, we need to give this label a target depth and remove
   // it from the branch-fixups list.
   } else {
-    assert(!Dest.ScopeDepth.isValid() && "already emitted label!");
-    Dest.ScopeDepth = EHStack.stable_begin();
+    assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
+    Dest = JumpDest(Dest.getBlock(),
+                    EHStack.stable_begin(),
+                    Dest.getDestIndex());
 
-    EHStack.resolveBranchFixups(Dest.Block);
+    ResolveBranchFixups(Dest.getBlock());
   }
 
-  EmitBlock(Dest.Block);
+  EmitBlock(Dest.getBlock());
 }
 
 
@@ -372,7 +376,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
   // Emit the header for the loop, which will also become
   // the continue target.
   JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
-  EmitBlock(LoopHeader.Block);
+  EmitBlock(LoopHeader.getBlock());
 
   // Create an exit block for when the condition fails, which will
   // also become the break target.
@@ -408,13 +412,13 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
   // As long as the condition is true, go to the loop body.
   llvm::BasicBlock *LoopBody = createBasicBlock("while.body");
   if (EmitBoolCondBranch) {
-    llvm::BasicBlock *ExitBlock = LoopExit.Block;
+    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
     if (ConditionScope.requiresCleanups())
       ExitBlock = createBasicBlock("while.exit");
 
     Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
 
-    if (ExitBlock != LoopExit.Block) {
+    if (ExitBlock != LoopExit.getBlock()) {
       EmitBlock(ExitBlock);
       EmitBranchThroughCleanup(LoopExit);
     }
@@ -434,15 +438,15 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
   ConditionScope.ForceCleanup();
 
   // Branch to the loop header again.
-  EmitBranch(LoopHeader.Block);
+  EmitBranch(LoopHeader.getBlock());
 
   // Emit the exit block.
-  EmitBlock(LoopExit.Block, true);
+  EmitBlock(LoopExit.getBlock(), true);
 
   // The LoopHeader typically is just a branch if we skipped emitting
   // a branch, try to erase it.
   if (!EmitBoolCondBranch)
-    SimplifyForwardingBlocks(LoopHeader.Block);
+    SimplifyForwardingBlocks(LoopHeader.getBlock());
 }
 
 void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
@@ -462,7 +466,7 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
 
   BreakContinueStack.pop_back();
 
-  EmitBlock(LoopCond.Block);
+  EmitBlock(LoopCond.getBlock());
 
   // C99 6.8.5.2: "The evaluation of the controlling expression takes place
   // after each execution of the loop body."
@@ -481,15 +485,15 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
 
   // As long as the condition is true, iterate the loop.
   if (EmitBoolCondBranch)
-    Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.Block);
+    Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock());
 
   // Emit the exit block.
-  EmitBlock(LoopExit.Block);
+  EmitBlock(LoopExit.getBlock());
 
   // The DoCond block typically is just a branch if we skipped
   // emitting a branch, try to erase it.
   if (!EmitBoolCondBranch)
-    SimplifyForwardingBlocks(LoopCond.Block);
+    SimplifyForwardingBlocks(LoopCond.getBlock());
 }
 
 void CodeGenFunction::EmitForStmt(const ForStmt &S) {
@@ -497,6 +501,12 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
 
   RunCleanupsScope ForScope(*this);
 
+  CGDebugInfo *DI = getDebugInfo();
+  if (DI) {
+    DI->setLocation(S.getSourceRange().getBegin());
+    DI->EmitRegionStart(Builder);
+  }
+
   // Evaluate the first part before the loop.
   if (S.getInit())
     EmitStmt(S.getInit());
@@ -505,7 +515,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
   // If there's an increment, the continue scope will be overwritten
   // later.
   JumpDest Continue = getJumpDestInCurrentScope("for.cond");
-  llvm::BasicBlock *CondBlock = Continue.Block;
+  llvm::BasicBlock *CondBlock = Continue.getBlock();
   EmitBlock(CondBlock);
 
   // Create a cleanup scope for the condition variable cleanups.
@@ -515,7 +525,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
   if (S.getCond()) {
     // If the for statement has a condition scope, emit the local variable
     // declaration.
-    llvm::BasicBlock *ExitBlock = LoopExit.Block;
+    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
     if (S.getConditionVariable()) {
       EmitLocalBlockVarDecl(*S.getConditionVariable());
     }
@@ -533,7 +543,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
     BoolCondVal = EvaluateExprAsBool(S.getCond());
     Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock);
 
-    if (ExitBlock != LoopExit.Block) {
+    if (ExitBlock != LoopExit.getBlock()) {
       EmitBlock(ExitBlock);
       EmitBranchThroughCleanup(LoopExit);
     }
@@ -554,12 +564,6 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
   // Store the blocks to use for break and continue.
   BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
 
-  CGDebugInfo *DI = getDebugInfo();
-  if (DI) {
-    DI->setLocation(S.getSourceRange().getBegin());
-    DI->EmitRegionStart(CurFn, Builder);
-  }
-
   {
     // Create a separate cleanup scope for the body, in case it is not
     // a compound statement.
@@ -569,7 +573,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
 
   // If there is an increment, emit it next.
   if (S.getInc()) {
-    EmitBlock(Continue.Block);
+    EmitBlock(Continue.getBlock());
     EmitStmt(S.getInc());
   }
 
@@ -582,11 +586,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
 
   if (DI) {
     DI->setLocation(S.getSourceRange().getEnd());
-    DI->EmitRegionEnd(CurFn, Builder);
+    DI->EmitRegionEnd(Builder);
   }
 
   // Emit the fall-through block.
-  EmitBlock(LoopExit.Block, true);
+  EmitBlock(LoopExit.getBlock(), true);
 }
 
 void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
@@ -839,13 +843,15 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
 
     // Otherwise, just forward the default block to the switch end.
     } else {
-      DefaultBlock->replaceAllUsesWith(SwitchExit.Block);
+      DefaultBlock->replaceAllUsesWith(SwitchExit.getBlock());
       delete DefaultBlock;
     }
   }
 
+  ConditionScope.ForceCleanup();
+
   // Emit continuation.
-  EmitBlock(SwitchExit.Block, true);
+  EmitBlock(SwitchExit.getBlock(), true);
 
   SwitchInsn = SavedSwitchInsn;
   CaseRangeBlock = SavedCRBlock;
@@ -855,16 +861,24 @@ static std::string
 SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
                  llvm::SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=0) {
   std::string Result;
+  std::string tmp;
 
   while (*Constraint) {
     switch (*Constraint) {
     default:
-      Result += Target.convertConstraint(*Constraint);
+      tmp = Target.convertConstraint(*Constraint);
+      if (Result.find(tmp) == std::string::npos) // Combine unique constraints
+        Result += tmp;
       break;
     // Ignore these
     case '*':
     case '?':
     case '!':
+    case '=': // Will see this and the following in mult-alt constraints.
+    case '+':
+      break;
+    case ',':                 // FIXME - Until the back-end properly supports
+              return Result;  // multiple alternative constraints, we stop here.
       break;
     case 'g':
       Result += "imr";
@@ -888,40 +902,50 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
   return Result;
 }
 
-llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
-                                         const TargetInfo::ConstraintInfo &Info,
-                                           const Expr *InputExpr,
-                                           std::string &ConstraintStr) {
+llvm::Value*
+CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
+                                    const TargetInfo::ConstraintInfo &Info,
+                                    LValue InputValue, QualType InputType,
+                                    std::string &ConstraintStr) {
   llvm::Value *Arg;
   if (Info.allowsRegister() || !Info.allowsMemory()) {
-    if (!CodeGenFunction::hasAggregateLLVMType(InputExpr->getType())) {
-      Arg = EmitScalarExpr(InputExpr);
+    if (!CodeGenFunction::hasAggregateLLVMType(InputType)) {
+      Arg = EmitLoadOfLValue(InputValue, InputType).getScalarVal();
     } else {
-      InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
-      LValue Dest = EmitLValue(InputExpr);
-
-      const llvm::Type *Ty = ConvertType(InputExpr->getType());
+      const llvm::Type *Ty = ConvertType(InputType);
       uint64_t Size = CGM.getTargetData().getTypeSizeInBits(Ty);
       if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
         Ty = llvm::IntegerType::get(VMContext, Size);
         Ty = llvm::PointerType::getUnqual(Ty);
 
-        Arg = Builder.CreateLoad(Builder.CreateBitCast(Dest.getAddress(), Ty));
+        Arg = Builder.CreateLoad(Builder.CreateBitCast(InputValue.getAddress(),
+                                                       Ty));
       } else {
-        Arg = Dest.getAddress();
+        Arg = InputValue.getAddress();
         ConstraintStr += '*';
       }
     }
   } else {
-    InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
-    LValue Dest = EmitLValue(InputExpr);
-    Arg = Dest.getAddress();
+    Arg = InputValue.getAddress();
     ConstraintStr += '*';
   }
 
   return Arg;
 }
 
+llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
+                                         const TargetInfo::ConstraintInfo &Info,
+                                           const Expr *InputExpr,
+                                           std::string &ConstraintStr) {
+  if (Info.allowsRegister() || !Info.allowsMemory())
+    if (!CodeGenFunction::hasAggregateLLVMType(InputExpr->getType()))
+      return EmitScalarExpr(InputExpr);
+
+  InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
+  LValue Dest = EmitLValue(InputExpr);
+  return EmitAsmInputLValue(S, Info, Dest, InputExpr->getType(), ConstraintStr);
+}
+
 void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   // Analyze the asm string to decompose it into its pieces.  We know that Sema
   // has already done this, so it is guaranteed to be successful.
@@ -1031,7 +1055,8 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
       InOutConstraints += ',';
 
       const Expr *InputExpr = S.getOutputExpr(i);
-      llvm::Value *Arg = EmitAsmInput(S, Info, InputExpr, InOutConstraints);
+      llvm::Value *Arg = EmitAsmInputLValue(S, Info, Dest, InputExpr->getType(),
+                                            InOutConstraints);
 
       if (Info.allowsRegister())
         InOutConstraints += llvm::utostr(i);
diff --git a/lib/CodeGen/CGTemporaries.cpp b/lib/CodeGen/CGTemporaries.cpp
index fd7c616b1162..dfb8dc63c5c5 100644
--- a/lib/CodeGen/CGTemporaries.cpp
+++ b/lib/CodeGen/CGTemporaries.cpp
@@ -15,34 +15,43 @@
 using namespace clang;
 using namespace CodeGen;
 
-static void EmitTemporaryCleanup(CodeGenFunction &CGF,
-                                 const CXXTemporary *Temporary,
-                                 llvm::Value *Addr,
-                                 llvm::Value *CondPtr) {
-  llvm::BasicBlock *CondEnd = 0;
+namespace {
+  struct DestroyTemporary : EHScopeStack::Cleanup {
+    const CXXTemporary *Temporary;
+    llvm::Value *Addr;
+    llvm::Value *CondPtr;
+
+    DestroyTemporary(const CXXTemporary *Temporary, llvm::Value *Addr,
+                     llvm::Value *CondPtr)
+      : Temporary(Temporary), Addr(Addr), CondPtr(CondPtr) {}
+
+    void Emit(CodeGenFunction &CGF, bool IsForEH) {
+      llvm::BasicBlock *CondEnd = 0;
     
-  // If this is a conditional temporary, we need to check the condition
-  // boolean and only call the destructor if it's true.
-  if (CondPtr) {
-    llvm::BasicBlock *CondBlock = CGF.createBasicBlock("temp.cond-dtor.call");
-    CondEnd = CGF.createBasicBlock("temp.cond-dtor.cont");
+      // If this is a conditional temporary, we need to check the condition
+      // boolean and only call the destructor if it's true.
+      if (CondPtr) {
+        llvm::BasicBlock *CondBlock =
+          CGF.createBasicBlock("temp.cond-dtor.call");
+        CondEnd = CGF.createBasicBlock("temp.cond-dtor.cont");
 
-    llvm::Value *Cond = CGF.Builder.CreateLoad(CondPtr);
-    CGF.Builder.CreateCondBr(Cond, CondBlock, CondEnd);
-    CGF.EmitBlock(CondBlock);
-  }
+        llvm::Value *Cond = CGF.Builder.CreateLoad(CondPtr);
+        CGF.Builder.CreateCondBr(Cond, CondBlock, CondEnd);
+        CGF.EmitBlock(CondBlock);
+      }
 
-  CGF.EmitCXXDestructorCall(Temporary->getDestructor(),
-                            Dtor_Complete, /*ForVirtualBase=*/false,
-                            Addr);
+      CGF.EmitCXXDestructorCall(Temporary->getDestructor(),
+                                Dtor_Complete, /*ForVirtualBase=*/false,
+                                Addr);
 
-  if (CondPtr) {
-    // Reset the condition to false.
-    CGF.Builder.CreateStore(llvm::ConstantInt::getFalse(CGF.getLLVMContext()),
-                            CondPtr);
-    CGF.EmitBlock(CondEnd);
-  }
-}                                 
+      if (CondPtr) {
+        // Reset the condition to false.
+        CGF.Builder.CreateStore(CGF.Builder.getFalse(), CondPtr);
+        CGF.EmitBlock(CondEnd);
+      }
+    }
+  };
+}
 
 /// Emits all the code to cause the given temporary to be cleaned up.
 void CodeGenFunction::EmitCXXTemporary(const CXXTemporary *Temporary,
@@ -59,16 +68,11 @@ void CodeGenFunction::EmitCXXTemporary(const CXXTemporary *Temporary,
     InitTempAlloca(CondPtr, llvm::ConstantInt::getFalse(VMContext));
 
     // Now set it to true.
-    Builder.CreateStore(llvm::ConstantInt::getTrue(VMContext), CondPtr);
+    Builder.CreateStore(Builder.getTrue(), CondPtr);
   }
 
-  CleanupBlock Cleanup(*this, NormalCleanup);
-  EmitTemporaryCleanup(*this, Temporary, Ptr, CondPtr);
-
-  if (Exceptions) {
-    Cleanup.beginEHCleanup();
-    EmitTemporaryCleanup(*this, Temporary, Ptr, CondPtr);
-  }
+  EHStack.pushCleanup<DestroyTemporary>(NormalAndEHCleanup,
+                                        Temporary, Ptr, CondPtr);
 }
 
 RValue
@@ -76,23 +80,13 @@ CodeGenFunction::EmitCXXExprWithTemporaries(const CXXExprWithTemporaries *E,
                                             llvm::Value *AggLoc,
                                             bool IsAggLocVolatile,
                                             bool IsInitializer) {
-  RValue RV;
-  {
-    RunCleanupsScope Scope(*this);
-
-    RV = EmitAnyExpr(E->getSubExpr(), AggLoc, IsAggLocVolatile,
+  RunCleanupsScope Scope(*this);
+  return EmitAnyExpr(E->getSubExpr(), AggLoc, IsAggLocVolatile,
                      /*IgnoreResult=*/false, IsInitializer);
-  }
-  return RV;
 }
 
 LValue CodeGenFunction::EmitCXXExprWithTemporariesLValue(
                                               const CXXExprWithTemporaries *E) {
-  LValue LV;
-  {
-    RunCleanupsScope Scope(*this);
-
-    LV = EmitLValue(E->getSubExpr());
-  }
-  return LV;
+  RunCleanupsScope Scope(*this);
+  return EmitLValue(E->getSubExpr());
 }
diff --git a/lib/CodeGen/CGVTT.cpp b/lib/CodeGen/CGVTT.cpp
index 61c74230e118..56acfc84802e 100644
--- a/lib/CodeGen/CGVTT.cpp
+++ b/lib/CodeGen/CGVTT.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenModule.h"
+#include "CGCXXABI.h"
 #include "clang/AST/RecordLayout.h"
 using namespace clang;
 using namespace CodeGen;
@@ -373,7 +374,7 @@ CodeGenVTables::GenerateVTT(llvm::GlobalVariable::LinkageTypes Linkage,
     return 0;
 
   llvm::SmallString<256> OutName;
-  CGM.getMangleContext().mangleCXXVTT(RD, OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXVTT(RD, OutName);
   llvm::StringRef Name = OutName.str();
 
   D1(printf("vtt %s\n", RD->getNameAsCString()));
diff --git a/lib/CodeGen/CGVTables.cpp b/lib/CodeGen/CGVTables.cpp
index 6abac2609f5f..bed4670f7f95 100644
--- a/lib/CodeGen/CGVTables.cpp
+++ b/lib/CodeGen/CGVTables.cpp
@@ -13,8 +13,10 @@
 
 #include "CodeGenModule.h"
 #include "CodeGenFunction.h"
+#include "CGCXXABI.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/RecordLayout.h"
+#include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/Compiler.h"
@@ -2408,12 +2410,12 @@ llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD,
   // Compute the mangled name.
   llvm::SmallString<256> Name;
   if (const CXXDestructorDecl* DD = dyn_cast<CXXDestructorDecl>(MD))
-    getMangleContext().mangleCXXDtorThunk(DD, GD.getDtorType(), Thunk.This,
-                                          Name);
+    getCXXABI().getMangleContext().mangleCXXDtorThunk(DD, GD.getDtorType(),
+                                                      Thunk.This, Name);
   else
-    getMangleContext().mangleThunk(MD, Thunk, Name);
+    getCXXABI().getMangleContext().mangleThunk(MD, Thunk, Name);
   
-  const llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(MD);
+  const llvm::Type *Ty = getTypes().GetFunctionTypeForVTable(GD);
   return GetOrCreateLLVMFunction(Name, Ty, GD);
 }
 
@@ -2460,6 +2462,54 @@ static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
   return CGF.Builder.CreateBitCast(V, Ptr->getType());
 }
 
+static void setThunkVisibility(CodeGenModule &CGM, const CXXMethodDecl *MD,
+                               const ThunkInfo &Thunk, llvm::Function *Fn) {
+  CGM.setGlobalVisibility(Fn, MD);
+
+  if (!CGM.getCodeGenOpts().HiddenWeakVTables)
+    return;
+
+  // If the thunk has weak/linkonce linkage, but the function must be
+  // emitted in every translation unit that references it, then we can
+  // emit its thunks with hidden visibility, since its thunks must be
+  // emitted when the function is.
+
+  // This follows CodeGenModule::setTypeVisibility; see the comments
+  // there for explanation.
+
+  if ((Fn->getLinkage() != llvm::GlobalVariable::LinkOnceODRLinkage &&
+       Fn->getLinkage() != llvm::GlobalVariable::WeakODRLinkage) ||
+      Fn->getVisibility() != llvm::GlobalVariable::DefaultVisibility)
+    return;
+
+  if (MD->hasAttr<VisibilityAttr>())
+    return;
+
+  switch (MD->getTemplateSpecializationKind()) {
+  case TSK_ExplicitInstantiationDefinition:
+  case TSK_ExplicitInstantiationDeclaration:
+    return;
+
+  case TSK_Undeclared:
+    break;
+
+  case TSK_ExplicitSpecialization:
+  case TSK_ImplicitInstantiation:
+    if (!CGM.getCodeGenOpts().HiddenWeakTemplateVTables)
+      return;
+    break;
+  }
+
+  // If there's an explicit definition, and that definition is
+  // out-of-line, then we can't assume that all users will have a
+  // definition to emit.
+  const FunctionDecl *Def = 0;
+  if (MD->hasBody(Def) && Def->isOutOfLine())
+    return;
+
+  Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
+}
+
 void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
                                     const ThunkInfo &Thunk) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
@@ -2473,13 +2523,8 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
   // CodeGenFunction::GenerateCode.
 
   // Create the implicit 'this' parameter declaration.
-  CXXThisDecl = ImplicitParamDecl::Create(getContext(), 0,
-                                          MD->getLocation(),
-                                          &getContext().Idents.get("this"),
-                                          ThisType);
-
-  // Add the 'this' parameter.
-  FunctionArgs.push_back(std::make_pair(CXXThisDecl, CXXThisDecl->getType()));
+  CurGD = GD;
+  CGM.getCXXABI().BuildInstanceFunctionParams(*this, ResultType, FunctionArgs);
 
   // Add the rest of the parameters.
   for (FunctionDecl::param_const_iterator I = MD->param_begin(),
@@ -2491,6 +2536,8 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
   
   StartFunction(GlobalDecl(), ResultType, Fn, FunctionArgs, SourceLocation());
 
+  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
+
   // Adjust the 'this' pointer if necessary.
   llvm::Value *AdjustedThisPtr = 
     PerformTypeAdjustment(*this, LoadCXXThis(), 
@@ -2514,7 +2561,7 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
 
   // Get our callee.
   const llvm::Type *Ty =
-    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(MD),
+    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(GD),
                                    FPT->isVariadic());
   llvm::Value *Callee = CGM.GetAddrOfFunction(GD, Ty);
 
@@ -2574,24 +2621,20 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn, GlobalDecl GD,
   }
 
   if (!ResultType->isVoidType() && Slot.isNull())
-    EmitReturnOfRValue(RV, ResultType);
+    CGM.getCXXABI().EmitReturnFromThunk(CGF, RV, ResultType);
 
   FinishFunction();
 
-  // Destroy the 'this' declaration.
-  CXXThisDecl->Destroy(getContext());
-  
   // Set the right linkage.
   CGM.setFunctionLinkage(MD, Fn);
   
   // Set the right visibility.
-  CGM.setGlobalVisibility(Fn, MD);
+  setThunkVisibility(CGM, MD, Thunk, Fn);
 }
 
 void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk)
 {
   llvm::Constant *Entry = CGM.GetAddrOfThunk(GD, Thunk);
-  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
   
   // Strip off a bitcast if we got one back.
   if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Entry)) {
@@ -2602,7 +2645,7 @@ void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk)
   // There's already a declaration with the same name, check if it has the same
   // type or if we need to replace it.
   if (cast<llvm::GlobalValue>(Entry)->getType()->getElementType() != 
-      CGM.getTypes().GetFunctionTypeForVTable(MD)) {
+      CGM.getTypes().GetFunctionTypeForVTable(GD)) {
     llvm::GlobalValue *OldThunkFn = cast<llvm::GlobalValue>(Entry);
     
     // If the types mismatch then we have to rewrite the definition.
@@ -2821,8 +2864,7 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
         
           NextVTableThunkIndex++;
         } else {
-          const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
-          const llvm::Type *Ty = CGM.getTypes().GetFunctionTypeForVTable(MD);
+          const llvm::Type *Ty = CGM.getTypes().GetFunctionTypeForVTable(GD);
         
           Init = CGM.GetAddrOfFunction(GD, Ty);
         }
@@ -2889,7 +2931,7 @@ GetGlobalVariable(llvm::Module &Module, llvm::StringRef Name,
 
 llvm::GlobalVariable *CodeGenVTables::GetAddrOfVTable(const CXXRecordDecl *RD) {
   llvm::SmallString<256> OutName;
-  CGM.getMangleContext().mangleCXXVTable(RD, OutName);
+  CGM.getCXXABI().getMangleContext().mangleCXXVTable(RD, OutName);
   llvm::StringRef Name = OutName.str();
 
   ComputeVTableRelatedInformation(RD, true);
@@ -2928,7 +2970,7 @@ CodeGenVTables::EmitVTableDefinition(llvm::GlobalVariable *VTable,
   VTable->setLinkage(Linkage);
   
   // Set the right visibility.
-  CGM.setGlobalVisibility(VTable, RD);
+  CGM.setTypeVisibility(VTable, RD, /*ForRTTI*/ false);
 }
 
 llvm::GlobalVariable *
@@ -2949,8 +2991,8 @@ CodeGenVTables::GenerateConstructionVTable(const CXXRecordDecl *RD,
 
   // Get the mangled construction vtable name.
   llvm::SmallString<256> OutName;
-  CGM.getMangleContext().mangleCXXCtorVTable(RD, Base.getBaseOffset() / 8, 
-                                             Base.getBase(), OutName);
+  CGM.getCXXABI().getMangleContext().
+    mangleCXXCtorVTable(RD, Base.getBaseOffset() / 8, Base.getBase(), OutName);
   llvm::StringRef Name = OutName.str();
 
   const llvm::Type *Int8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext());
diff --git a/lib/CodeGen/CGValue.h b/lib/CodeGen/CGValue.h
index 92ef9dc4a11f..f57ecd245f09 100644
--- a/lib/CodeGen/CGValue.h
+++ b/lib/CodeGen/CGValue.h
@@ -15,6 +15,7 @@
 #ifndef CLANG_CODEGEN_CGVALUE_H
 #define CLANG_CODEGEN_CGVALUE_H
 
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/Type.h"
 
 namespace llvm {
@@ -136,6 +137,9 @@ class LValue {
   // 'const' is unused here
   Qualifiers Quals;
 
+  /// The alignment to use when accessing this lvalue.
+  unsigned short Alignment;
+
   // objective-c's ivar
   bool Ivar:1;
   
@@ -148,15 +152,20 @@ class LValue {
 
   // Lvalue is a global reference of an objective-c object
   bool GlobalObjCRef : 1;
+  
+  // Lvalue is a thread local reference
+  bool ThreadLocalRef : 1;
 
   Expr *BaseIvarExp;
 private:
-  void SetQualifiers(Qualifiers Quals) {
+  void Initialize(Qualifiers Quals, unsigned Alignment = 0) {
     this->Quals = Quals;
-    
-    // FIXME: Convenient place to set objc flags to 0. This should really be
-    // done in a user-defined constructor instead.
+    this->Alignment = Alignment;
+    assert(this->Alignment == Alignment && "Alignment exceeds allowed max!");
+
+    // Initialize Objective-C flags.
     this->Ivar = this->ObjIsArray = this->NonGC = this->GlobalObjCRef = false;
+    this->ThreadLocalRef = false;
     this->BaseIvarExp = 0;
   }
 
@@ -175,30 +184,36 @@ public:
   }
 
   bool isObjCIvar() const { return Ivar; }
+  void setObjCIvar(bool Value) { Ivar = Value; }
+
   bool isObjCArray() const { return ObjIsArray; }
+  void setObjCArray(bool Value) { ObjIsArray = Value; }
+
   bool isNonGC () const { return NonGC; }
+  void setNonGC(bool Value) { NonGC = Value; }
+
   bool isGlobalObjCRef() const { return GlobalObjCRef; }
-  bool isObjCWeak() const { return Quals.getObjCGCAttr() == Qualifiers::Weak; }
-  bool isObjCStrong() const { return Quals.getObjCGCAttr() == Qualifiers::Strong; }
+  void setGlobalObjCRef(bool Value) { GlobalObjCRef = Value; }
+
+  bool isThreadLocalRef() const { return ThreadLocalRef; }
+  void setThreadLocalRef(bool Value) { ThreadLocalRef = Value;}
+
+  bool isObjCWeak() const {
+    return Quals.getObjCGCAttr() == Qualifiers::Weak;
+  }
+  bool isObjCStrong() const {
+    return Quals.getObjCGCAttr() == Qualifiers::Strong;
+  }
   
   Expr *getBaseIvarExp() const { return BaseIvarExp; }
   void setBaseIvarExp(Expr *V) { BaseIvarExp = V; }
 
-  unsigned getAddressSpace() const { return Quals.getAddressSpace(); }
+  const Qualifiers &getQuals() const { return Quals; }
+  Qualifiers &getQuals() { return Quals; }
 
-  static void SetObjCIvar(LValue& R, bool iValue) {
-    R.Ivar = iValue;
-  }
-  static void SetObjCArray(LValue& R, bool iValue) {
-    R.ObjIsArray = iValue;
-  }
-  static void SetGlobalObjCRef(LValue& R, bool iValue) {
-    R.GlobalObjCRef = iValue;
-  }
+  unsigned getAddressSpace() const { return Quals.getAddressSpace(); }
 
-  static void SetObjCNonGC(LValue& R, bool iValue) {
-    R.NonGC = iValue;
-  }
+  unsigned getAlignment() const { return Alignment; }
 
   // simple lvalue
   llvm::Value *getAddress() const { assert(isSimple()); return V; }
@@ -236,11 +251,15 @@ public:
     return KVCRefExpr;
   }
 
-  static LValue MakeAddr(llvm::Value *V, Qualifiers Quals) {
+  static LValue MakeAddr(llvm::Value *V, QualType T, unsigned Alignment,
+                         ASTContext &Context) {
+    Qualifiers Quals = Context.getCanonicalType(T).getQualifiers();
+    Quals.setObjCGCAttr(Context.getObjCGCAttrKind(T));
+
     LValue R;
     R.LVType = Simple;
     R.V = V;
-    R.SetQualifiers(Quals);
+    R.Initialize(Quals, Alignment);
     return R;
   }
 
@@ -250,7 +269,7 @@ public:
     R.LVType = VectorElt;
     R.V = Vec;
     R.VectorIdx = Idx;
-    R.SetQualifiers(Qualifiers::fromCVRMask(CVR));
+    R.Initialize(Qualifiers::fromCVRMask(CVR));
     return R;
   }
 
@@ -260,7 +279,7 @@ public:
     R.LVType = ExtVectorElt;
     R.V = Vec;
     R.VectorElts = Elts;
-    R.SetQualifiers(Qualifiers::fromCVRMask(CVR));
+    R.Initialize(Qualifiers::fromCVRMask(CVR));
     return R;
   }
 
@@ -276,7 +295,7 @@ public:
     R.LVType = BitField;
     R.V = BaseValue;
     R.BitFieldInfo = &Info;
-    R.SetQualifiers(Qualifiers::fromCVRMask(CVR));
+    R.Initialize(Qualifiers::fromCVRMask(CVR));
     return R;
   }
 
@@ -288,7 +307,7 @@ public:
     LValue R;
     R.LVType = PropertyRef;
     R.PropertyRefExpr = E;
-    R.SetQualifiers(Qualifiers::fromCVRMask(CVR));
+    R.Initialize(Qualifiers::fromCVRMask(CVR));
     return R;
   }
 
@@ -297,7 +316,7 @@ public:
     LValue R;
     R.LVType = KVCRef;
     R.KVCRefExpr = E;
-    R.SetQualifiers(Qualifiers::fromCVRMask(CVR));
+    R.Initialize(Qualifiers::fromCVRMask(CVR));
     return R;
   }
 };
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index eb6c4361be89..51d084e1d301 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -13,6 +13,7 @@
 
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
+#include "CGCXXABI.h"
 #include "CGDebugInfo.h"
 #include "CGException.h"
 #include "clang/Basic/TargetInfo.h"
@@ -31,8 +32,9 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm)
   : BlockFunction(cgm, *this, Builder), CGM(cgm),
     Target(CGM.getContext().Target),
     Builder(cgm.getModule().getContext()),
+    NormalCleanupDest(0), EHCleanupDest(0), NextCleanupDestIndex(1),
     ExceptionSlot(0), DebugInfo(0), IndirectBranch(0),
-    SwitchInsn(0), CaseRangeBlock(0), InvokeDest(0),
+    SwitchInsn(0), CaseRangeBlock(0),
     DidCallStackSave(false), UnreachableBlock(0),
     CXXThisDecl(0), CXXThisValue(0), CXXVTTDecl(0), CXXVTTValue(0),
     ConditionalBranchLevel(0), TerminateLandingPad(0), TerminateHandler(0),
@@ -47,7 +49,7 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm)
       
   Exceptions = getContext().getLangOptions().Exceptions;
   CatchUndefined = getContext().getLangOptions().CatchUndefined;
-  CGM.getMangleContext().startNewFunction();
+  CGM.getCXXABI().getMangleContext().startNewFunction();
 }
 
 ASTContext &CodeGenFunction::getContext() const {
@@ -55,17 +57,6 @@ ASTContext &CodeGenFunction::getContext() const {
 }
 
 
-llvm::Value *CodeGenFunction::GetAddrOfLocalVar(const VarDecl *VD) {
-  llvm::Value *Res = LocalDeclMap[VD];
-  assert(Res && "Invalid argument to GetAddrOfLocalVar(), no decl!");
-  return Res;
-}
-
-llvm::Constant *
-CodeGenFunction::GetAddrOfStaticLocalVar(const VarDecl *BVD) {
-  return cast<llvm::Constant>(GetAddrOfLocalVar(BVD));
-}
-
 const llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
   return CGM.getTypes().ConvertTypeForMem(T);
 }
@@ -76,7 +67,7 @@ const llvm::Type *CodeGenFunction::ConvertType(QualType T) {
 
 bool CodeGenFunction::hasAggregateLLVMType(QualType T) {
   return T->isRecordType() || T->isArrayType() || T->isAnyComplexType() ||
-    T->isMemberFunctionPointerType();
+    T->isObjCObjectType();
 }
 
 void CodeGenFunction::EmitReturnBlock() {
@@ -89,26 +80,26 @@ void CodeGenFunction::EmitReturnBlock() {
 
     // We have a valid insert point, reuse it if it is empty or there are no
     // explicit jumps to the return block.
-    if (CurBB->empty() || ReturnBlock.Block->use_empty()) {
-      ReturnBlock.Block->replaceAllUsesWith(CurBB);
-      delete ReturnBlock.Block;
+    if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
+      ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
+      delete ReturnBlock.getBlock();
     } else
-      EmitBlock(ReturnBlock.Block);
+      EmitBlock(ReturnBlock.getBlock());
     return;
   }
 
   // Otherwise, if the return block is the target of a single direct
   // branch then we can just put the code in that block instead. This
   // cleans up functions which started with a unified return block.
-  if (ReturnBlock.Block->hasOneUse()) {
+  if (ReturnBlock.getBlock()->hasOneUse()) {
     llvm::BranchInst *BI =
-      dyn_cast<llvm::BranchInst>(*ReturnBlock.Block->use_begin());
+      dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->use_begin());
     if (BI && BI->isUnconditional() &&
-        BI->getSuccessor(0) == ReturnBlock.Block) {
+        BI->getSuccessor(0) == ReturnBlock.getBlock()) {
       // Reset insertion point and delete the branch.
       Builder.SetInsertPoint(BI->getParent());
       BI->eraseFromParent();
-      delete ReturnBlock.Block;
+      delete ReturnBlock.getBlock();
       return;
     }
   }
@@ -117,7 +108,7 @@ void CodeGenFunction::EmitReturnBlock() {
   // unless it has uses. However, we still need a place to put the debug
   // region.end for now.
 
-  EmitBlock(ReturnBlock.Block);
+  EmitBlock(ReturnBlock.getBlock());
 }
 
 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
@@ -139,7 +130,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   // Emit debug descriptor for function end.
   if (CGDebugInfo *DI = getDebugInfo()) {
     DI->setLocation(EndLoc);
-    DI->EmitRegionEnd(CurFn, Builder);
+    DI->EmitFunctionEnd(Builder);
   }
 
   EmitFunctionEpilog(*CurFnInfo);
@@ -170,6 +161,7 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
     }
   }
 
+  EmitIfUsed(*this, RethrowBlock.getBlock());
   EmitIfUsed(*this, TerminateLandingPad);
   EmitIfUsed(*this, TerminateHandler);
   EmitIfUsed(*this, UnreachableBlock);
@@ -287,10 +279,8 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
   EmitStartEHSpec(CurCodeDecl);
   EmitFunctionProlog(*CurFnInfo, CurFn, Args);
 
-  if (CXXThisDecl)
-    CXXThisValue = Builder.CreateLoad(LocalDeclMap[CXXThisDecl], "this");
-  if (CXXVTTDecl)
-    CXXVTTValue = Builder.CreateLoad(LocalDeclMap[CXXVTTDecl], "vtt");
+  if (D && isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance())
+    CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
 
   // If any of the arguments have a variably modified type, make sure to
   // emit the type size.
@@ -309,6 +299,23 @@ void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args) {
   EmitStmt(FD->getBody());
 }
 
+/// Tries to mark the given function nounwind based on the
+/// non-existence of any throwing calls within it.  We believe this is
+/// lightweight enough to do at -O0.
+static void TryMarkNoThrow(llvm::Function *F) {
+  // LLVM treats 'nounwind' on a function as part of the type, so we
+  // can't do this on functions that can be overwritten.
+  if (F->mayBeOverridden()) return;
+
+  for (llvm::Function::iterator FI = F->begin(), FE = F->end(); FI != FE; ++FI)
+    for (llvm::BasicBlock::iterator
+           BI = FI->begin(), BE = FI->end(); BI != BE; ++BI)
+      if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(&*BI))
+        if (!Call->doesNotThrow())
+          return;
+  F->setDoesNotThrow(true);
+}
+
 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn) {
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
   
@@ -317,30 +324,11 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn) {
     DebugInfo = CGM.getDebugInfo();
 
   FunctionArgList Args;
+  QualType ResTy = FD->getResultType();
 
   CurGD = GD;
-  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) {
-    if (MD->isInstance()) {
-      // Create the implicit 'this' decl.
-      // FIXME: I'm not entirely sure I like using a fake decl just for code
-      // generation. Maybe we can come up with a better way?
-      CXXThisDecl = ImplicitParamDecl::Create(getContext(), 0,
-                                              FD->getLocation(),
-                                              &getContext().Idents.get("this"),
-                                              MD->getThisType(getContext()));
-      Args.push_back(std::make_pair(CXXThisDecl, CXXThisDecl->getType()));
-      
-      // Check if we need a VTT parameter as well.
-      if (CodeGenVTables::needsVTTParameter(GD)) {
-        // FIXME: The comment about using a fake decl above applies here too.
-        QualType T = getContext().getPointerType(getContext().VoidPtrTy);
-        CXXVTTDecl = 
-          ImplicitParamDecl::Create(getContext(), 0, FD->getLocation(),
-                                    &getContext().Idents.get("vtt"), T);
-        Args.push_back(std::make_pair(CXXVTTDecl, CXXVTTDecl->getType()));
-      }
-    }
-  }
+  if (isa<CXXMethodDecl>(FD) && cast<CXXMethodDecl>(FD)->isInstance())
+    CGM.getCXXABI().BuildInstanceFunctionParams(*this, ResTy, Args);
 
   if (FD->getNumParams()) {
     const FunctionProtoType* FProto = FD->getType()->getAs<FunctionProtoType>();
@@ -355,7 +343,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn) {
   if (Stmt *Body = FD->getBody()) BodyRange = Body->getSourceRange();
 
   // Emit the standard function prologue.
-  StartFunction(GD, FD->getResultType(), Fn, Args, BodyRange.getBegin());
+  StartFunction(GD, ResTy, Fn, Args, BodyRange.getBegin());
 
   // Generate the body of the function.
   if (isa<CXXDestructorDecl>(FD))
@@ -368,13 +356,10 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn) {
   // Emit the standard function epilogue.
   FinishFunction(BodyRange.getEnd());
 
-  // Destroy the 'this' declaration.
-  if (CXXThisDecl)
-    CXXThisDecl->Destroy(getContext());
-  
-  // Destroy the VTT declaration.
-  if (CXXVTTDecl)
-    CXXVTTDecl->Destroy(getContext());
+  // If we haven't marked the function nothrow through other means, do
+  // a quick pass now to see if we can.
+  if (!CurFn->doesNotThrow())
+    TryMarkNoThrow(CurFn);
 }
 
 /// ContainsLabel - Return true if the statement contains a label in it.  If
@@ -439,7 +424,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
 
   if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
     // Handle X && Y in a condition.
-    if (CondBOp->getOpcode() == BinaryOperator::LAnd) {
+    if (CondBOp->getOpcode() == BO_LAnd) {
       // If we have "1 && X", simplify the code.  "0 && X" would have constant
       // folded if the case was simple enough.
       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS()) == 1) {
@@ -466,7 +451,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
       EndConditionalBranch();
 
       return;
-    } else if (CondBOp->getOpcode() == BinaryOperator::LOr) {
+    } else if (CondBOp->getOpcode() == BO_LOr) {
       // If we have "0 || X", simplify the code.  "1 || X" would have constant
       // folded if the case was simple enough.
       if (ConstantFoldsToSimpleInteger(CondBOp->getLHS()) == -1) {
@@ -498,7 +483,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
 
   if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
     // br(!x, t, f) -> br(x, f, t)
-    if (CondUOp->getOpcode() == UnaryOperator::LNot)
+    if (CondUOp->getOpcode() == UO_LNot)
       return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock);
   }
 
@@ -533,21 +518,6 @@ void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type,
 
 void
 CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
-  // If the type contains a pointer to data member we can't memset it to zero.
-  // Instead, create a null constant and copy it to the destination.
-  if (CGM.getTypes().ContainsPointerToDataMember(Ty)) {
-    llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
-    
-    llvm::GlobalVariable *NullVariable = 
-      new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
-                               /*isConstant=*/true, 
-                               llvm::GlobalVariable::PrivateLinkage,
-                               NullConstant, llvm::Twine());
-    EmitAggregateCopy(DestPtr, NullVariable, Ty, /*isVolatile=*/false);
-    return;
-  } 
-  
-
   // Ignore empty classes in C++.
   if (getContext().getLangOptions().CPlusPlus) {
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
@@ -555,29 +525,58 @@ CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
         return;
     }
   }
-  
-  // Otherwise, just memset the whole thing to zero.  This is legal
-  // because in LLVM, all default initializers (other than the ones we just
-  // handled above) are guaranteed to have a bit pattern of all zeros.
-  const llvm::Type *BP = llvm::Type::getInt8PtrTy(VMContext);
+
+  // Cast the dest ptr to the appropriate i8 pointer type.
+  unsigned DestAS =
+    cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+  const llvm::Type *BP =
+    llvm::Type::getInt8PtrTy(VMContext, DestAS);
   if (DestPtr->getType() != BP)
     DestPtr = Builder.CreateBitCast(DestPtr, BP, "tmp");
 
   // Get size and alignment info for this aggregate.
   std::pair<uint64_t, unsigned> TypeInfo = getContext().getTypeInfo(Ty);
+  uint64_t Size = TypeInfo.first;
+  unsigned Align = TypeInfo.second;
 
   // Don't bother emitting a zero-byte memset.
-  if (TypeInfo.first == 0)
+  if (Size == 0)
     return;
 
+  llvm::ConstantInt *SizeVal = llvm::ConstantInt::get(IntPtrTy, Size / 8);
+  llvm::ConstantInt *AlignVal = Builder.getInt32(Align / 8);
+
+  // If the type contains a pointer to data member we can't memset it to zero.
+  // Instead, create a null constant and copy it to the destination.
+  if (!CGM.getTypes().isZeroInitializable(Ty)) {
+    llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
+
+    llvm::GlobalVariable *NullVariable = 
+      new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
+                               /*isConstant=*/true, 
+                               llvm::GlobalVariable::PrivateLinkage,
+                               NullConstant, llvm::Twine());
+    llvm::Value *SrcPtr =
+      Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy());
+
+    // FIXME: variable-size types?
+
+    // Get and call the appropriate llvm.memcpy overload.
+    llvm::Constant *Memcpy =
+      CGM.getMemCpyFn(DestPtr->getType(), SrcPtr->getType(), IntPtrTy);
+    Builder.CreateCall5(Memcpy, DestPtr, SrcPtr, SizeVal, AlignVal,
+                        /*volatile*/ Builder.getFalse());
+    return;
+  } 
+  
+  // Otherwise, just memset the whole thing to zero.  This is legal
+  // because in LLVM, all default initializers (other than the ones we just
+  // handled above) are guaranteed to have a bit pattern of all zeros.
+
   // FIXME: Handle variable sized types.
   Builder.CreateCall5(CGM.getMemSetFn(BP, IntPtrTy), DestPtr,
-                 llvm::Constant::getNullValue(llvm::Type::getInt8Ty(VMContext)),
-                      // TypeInfo.first describes size in bits.
-                      llvm::ConstantInt::get(IntPtrTy, TypeInfo.first/8),
-                      llvm::ConstantInt::get(Int32Ty, TypeInfo.second/8),
-                      llvm::ConstantInt::get(llvm::Type::getInt1Ty(VMContext),
-                                             0));
+                      Builder.getInt8(0),
+                      SizeVal, AlignVal, /*volatile*/ Builder.getFalse());
 }
 
 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelStmt *L) {
@@ -585,7 +584,7 @@ llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelStmt *L) {
   if (IndirectBranch == 0)
     GetIndirectGotoBlock();
   
-  llvm::BasicBlock *BB = getJumpDestForLabel(L).Block;
+  llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
   
   // Make sure the indirect branch includes all of the address-taken blocks.
   IndirectBranch->addDestination(BB);
@@ -666,70 +665,75 @@ llvm::Value* CodeGenFunction::EmitVAListRef(const Expr* E) {
 void CodeGenFunction::PopCleanupBlocks(EHScopeStack::stable_iterator Old) {
   assert(Old.isValid());
 
-  EHScopeStack::iterator E = EHStack.find(Old);
-  while (EHStack.begin() != E)
-    PopCleanupBlock();
-}
+  while (EHStack.stable_begin() != Old) {
+    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
+
+    // As long as Old strictly encloses the scope's enclosing normal
+    // cleanup, we're going to emit another normal cleanup which
+    // fallthrough can propagate through.
+    bool FallThroughIsBranchThrough =
+      Old.strictlyEncloses(Scope.getEnclosingNormalCleanup());
 
-/// Destroys a cleanup if it was unused.
-static void DestroyCleanup(CodeGenFunction &CGF,
-                           llvm::BasicBlock *Entry,
-                           llvm::BasicBlock *Exit) {
-  assert(Entry->use_empty() && "destroying cleanup with uses!");
-  assert(Exit->getTerminator() == 0 &&
-         "exit has terminator but entry has no predecessors!");
-
-  // This doesn't always remove the entire cleanup, but it's much
-  // safer as long as we don't know what blocks belong to the cleanup.
-  // A *much* better approach if we care about this inefficiency would
-  // be to lazily emit the cleanup.
-
-  // If the exit block is distinct from the entry, give it a branch to
-  // an unreachable destination.  This preserves the well-formedness
-  // of the IR.
-  if (Entry != Exit)
-    llvm::BranchInst::Create(CGF.getUnreachableBlock(), Exit);
-
-  assert(!Entry->getParent() && "cleanup entry already positioned?");
-  // We can't just delete the entry; we have to kill any references to
-  // its instructions in other blocks.
-  for (llvm::BasicBlock::iterator I = Entry->begin(), E = Entry->end();
-         I != E; ++I)
-    if (!I->use_empty())
-      I->replaceAllUsesWith(llvm::UndefValue::get(I->getType()));
-  delete Entry;
+    PopCleanupBlock(FallThroughIsBranchThrough);
+  }
 }
 
-/// Creates a switch instruction to thread branches out of the given
-/// block (which is the exit block of a cleanup).
-static void CreateCleanupSwitch(CodeGenFunction &CGF,
-                                llvm::BasicBlock *Block) {
-  if (Block->getTerminator()) {
-    assert(isa<llvm::SwitchInst>(Block->getTerminator()) &&
-           "cleanup block already has a terminator, but it isn't a switch");
-    return;
+static llvm::BasicBlock *CreateNormalEntry(CodeGenFunction &CGF,
+                                           EHCleanupScope &Scope) {
+  assert(Scope.isNormalCleanup());
+  llvm::BasicBlock *Entry = Scope.getNormalBlock();
+  if (!Entry) {
+    Entry = CGF.createBasicBlock("cleanup");
+    Scope.setNormalBlock(Entry);
   }
+  return Entry;
+}
 
-  llvm::Value *DestCodePtr
-    = CGF.CreateTempAlloca(CGF.Builder.getInt32Ty(), "cleanup.dst");
-  CGBuilderTy Builder(Block);
-  llvm::Value *DestCode = Builder.CreateLoad(DestCodePtr, "tmp");
+static llvm::BasicBlock *CreateEHEntry(CodeGenFunction &CGF,
+                                       EHCleanupScope &Scope) {
+  assert(Scope.isEHCleanup());
+  llvm::BasicBlock *Entry = Scope.getEHBlock();
+  if (!Entry) {
+    Entry = CGF.createBasicBlock("eh.cleanup");
+    Scope.setEHBlock(Entry);
+  }
+  return Entry;
+}
 
-  // Create a switch instruction to determine where to jump next.
-  Builder.CreateSwitch(DestCode, CGF.getUnreachableBlock());
+/// Transitions the terminator of the given exit-block of a cleanup to
+/// be a cleanup switch.
+static llvm::SwitchInst *TransitionToCleanupSwitch(CodeGenFunction &CGF,
+                                                   llvm::BasicBlock *Block) {
+  // If it's a branch, turn it into a switch whose default
+  // destination is its original target.
+  llvm::TerminatorInst *Term = Block->getTerminator();
+  assert(Term && "can't transition block without terminator");
+
+  if (llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Term)) {
+    assert(Br->isUnconditional());
+    llvm::LoadInst *Load =
+      new llvm::LoadInst(CGF.getNormalCleanupDestSlot(), "cleanup.dest", Term);
+    llvm::SwitchInst *Switch =
+      llvm::SwitchInst::Create(Load, Br->getSuccessor(0), 4, Block);
+    Br->eraseFromParent();
+    return Switch;
+  } else {
+    return cast<llvm::SwitchInst>(Term);
+  }
 }
 
 /// Attempts to reduce a cleanup's entry block to a fallthrough.  This
 /// is basically llvm::MergeBlockIntoPredecessor, except
-/// simplified/optimized for the tighter constraints on cleanup
-/// blocks.
-static void SimplifyCleanupEntry(CodeGenFunction &CGF,
-                                 llvm::BasicBlock *Entry) {
+/// simplified/optimized for the tighter constraints on cleanup blocks.
+///
+/// Returns the new block, whatever it is.
+static llvm::BasicBlock *SimplifyCleanupEntry(CodeGenFunction &CGF,
+                                              llvm::BasicBlock *Entry) {
   llvm::BasicBlock *Pred = Entry->getSinglePredecessor();
-  if (!Pred) return;
+  if (!Pred) return Entry;
 
   llvm::BranchInst *Br = dyn_cast<llvm::BranchInst>(Pred->getTerminator());
-  if (!Br || Br->isConditional()) return;
+  if (!Br || Br->isConditional()) return Entry;
   assert(Br->getSuccessor(0) == Entry);
 
   // If we were previously inserting at the end of the cleanup entry
@@ -749,145 +753,44 @@ static void SimplifyCleanupEntry(CodeGenFunction &CGF,
 
   if (WasInsertBlock)
     CGF.Builder.SetInsertPoint(Pred);
-}
-
-/// Attempts to reduce an cleanup's exit switch to an unconditional
-/// branch.
-static void SimplifyCleanupExit(llvm::BasicBlock *Exit) {
-  llvm::TerminatorInst *Terminator = Exit->getTerminator();
-  assert(Terminator && "completed cleanup exit has no terminator");
-
-  llvm::SwitchInst *Switch = dyn_cast<llvm::SwitchInst>(Terminator);
-  if (!Switch) return;
-  if (Switch->getNumCases() != 2) return; // default + 1
 
-  llvm::LoadInst *Cond = cast<llvm::LoadInst>(Switch->getCondition());
-  llvm::AllocaInst *CondVar = cast<llvm::AllocaInst>(Cond->getPointerOperand());
-
-  // Replace the switch instruction with an unconditional branch.
-  llvm::BasicBlock *Dest = Switch->getSuccessor(1); // default is 0
-  Switch->eraseFromParent();
-  llvm::BranchInst::Create(Dest, Exit);
-
-  // Delete all uses of the condition variable.
-  Cond->eraseFromParent();
-  while (!CondVar->use_empty())
-    cast<llvm::StoreInst>(*CondVar->use_begin())->eraseFromParent();
-
-  // Delete the condition variable itself.
-  CondVar->eraseFromParent();
+  return Pred;
 }
 
-/// Threads a branch fixup through a cleanup block.
-static void ThreadFixupThroughCleanup(CodeGenFunction &CGF,
-                                      BranchFixup &Fixup,
-                                      llvm::BasicBlock *Entry,
-                                      llvm::BasicBlock *Exit) {
-  if (!Exit->getTerminator())
-    CreateCleanupSwitch(CGF, Exit);
-
-  // Find the switch and its destination index alloca.
-  llvm::SwitchInst *Switch = cast<llvm::SwitchInst>(Exit->getTerminator());
-  llvm::Value *DestCodePtr =
-    cast<llvm::LoadInst>(Switch->getCondition())->getPointerOperand();
-
-  // Compute the index of the new case we're adding to the switch.
-  unsigned Index = Switch->getNumCases();
-
-  const llvm::IntegerType *i32 = llvm::Type::getInt32Ty(CGF.getLLVMContext());
-  llvm::ConstantInt *IndexV = llvm::ConstantInt::get(i32, Index);
-
-  // Set the index in the origin block.
-  new llvm::StoreInst(IndexV, DestCodePtr, Fixup.Origin);
-
-  // Add a case to the switch.
-  Switch->addCase(IndexV, Fixup.Destination);
-
-  // Change the last branch to point to the cleanup entry block.
-  Fixup.LatestBranch->setSuccessor(Fixup.LatestBranchIndex, Entry);
-
-  // And finally, update the fixup.
-  Fixup.LatestBranch = Switch;
-  Fixup.LatestBranchIndex = Index;
-}
-
-/// Try to simplify both the entry and exit edges of a cleanup.
-static void SimplifyCleanupEdges(CodeGenFunction &CGF,
-                                 llvm::BasicBlock *Entry,
-                                 llvm::BasicBlock *Exit) {
-
-  // Given their current implementations, it's important to run these
-  // in this order: SimplifyCleanupEntry will delete Entry if it can
-  // be merged into its predecessor, which will then break
-  // SimplifyCleanupExit if (as is common) Entry == Exit.
-
-  SimplifyCleanupExit(Exit);
-  SimplifyCleanupEntry(CGF, Entry);  
-}
-
-static void EmitLazyCleanup(CodeGenFunction &CGF,
-                            EHScopeStack::LazyCleanup *Fn,
-                            bool ForEH) {
+static void EmitCleanup(CodeGenFunction &CGF,
+                        EHScopeStack::Cleanup *Fn,
+                        bool ForEH) {
   if (ForEH) CGF.EHStack.pushTerminate();
   Fn->Emit(CGF, ForEH);
   if (ForEH) CGF.EHStack.popTerminate();
   assert(CGF.HaveInsertPoint() && "cleanup ended with no insertion point?");
 }
 
-static void SplitAndEmitLazyCleanup(CodeGenFunction &CGF,
-                                    EHScopeStack::LazyCleanup *Fn,
-                                    bool ForEH,
-                                    llvm::BasicBlock *Entry) {
-  assert(Entry && "no entry block for cleanup");
-
-  // Remove the switch and load from the end of the entry block.
-  llvm::Instruction *Switch = &Entry->getInstList().back();
-  Entry->getInstList().remove(Switch);
-  assert(isa<llvm::SwitchInst>(Switch));
-  llvm::Instruction *Load = &Entry->getInstList().back();
-  Entry->getInstList().remove(Load);
-  assert(isa<llvm::LoadInst>(Load));
-
-  assert(Entry->getInstList().empty() &&
-         "lazy cleanup block not empty after removing load/switch pair?");
-
-  // Emit the actual cleanup at the end of the entry block.
-  CGF.Builder.SetInsertPoint(Entry);
-  EmitLazyCleanup(CGF, Fn, ForEH);
-
-  // Put the load and switch at the end of the exit block.
-  llvm::BasicBlock *Exit = CGF.Builder.GetInsertBlock();
-  Exit->getInstList().push_back(Load);
-  Exit->getInstList().push_back(Switch);
-
-  // Clean up the edges if possible.
-  SimplifyCleanupEdges(CGF, Entry, Exit);
-
-  CGF.Builder.ClearInsertionPoint();
-}
-
-static void PopLazyCleanupBlock(CodeGenFunction &CGF) {
-  assert(isa<EHLazyCleanupScope>(*CGF.EHStack.begin()) && "top not a cleanup!");
-  EHLazyCleanupScope &Scope = cast<EHLazyCleanupScope>(*CGF.EHStack.begin());
-  assert(Scope.getFixupDepth() <= CGF.EHStack.getNumBranchFixups());
+/// Pops a cleanup block.  If the block includes a normal cleanup, the
+/// current insertion point is threaded through the cleanup, as are
+/// any branch fixups on the cleanup.
+void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
+  assert(!EHStack.empty() && "cleanup stack is empty!");
+  assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
+  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
+  assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
+  assert(Scope.isActive() && "cleanup was still inactive when popped!");
 
   // Check whether we need an EH cleanup.  This is only true if we've
   // generated a lazy EH cleanup block.
-  llvm::BasicBlock *EHEntry = Scope.getEHBlock();
-  bool RequiresEHCleanup = (EHEntry != 0);
+  bool RequiresEHCleanup = Scope.hasEHBranches();
 
   // Check the three conditions which might require a normal cleanup:
 
   // - whether there are branch fix-ups through this cleanup
   unsigned FixupDepth = Scope.getFixupDepth();
-  bool HasFixups = CGF.EHStack.getNumBranchFixups() != FixupDepth;
+  bool HasFixups = EHStack.getNumBranchFixups() != FixupDepth;
 
-  // - whether control has already been threaded through this cleanup
-  llvm::BasicBlock *NormalEntry = Scope.getNormalBlock();
-  bool HasExistingBranches = (NormalEntry != 0);
+  // - whether there are branch-throughs or branch-afters
+  bool HasExistingBranches = Scope.hasBranches();
 
   // - whether there's a fallthrough
-  llvm::BasicBlock *FallthroughSource = CGF.Builder.GetInsertBlock();
+  llvm::BasicBlock *FallthroughSource = Builder.GetInsertBlock();
   bool HasFallthrough = (FallthroughSource != 0);
 
   bool RequiresNormalCleanup = false;
@@ -898,9 +801,9 @@ static void PopLazyCleanupBlock(CodeGenFunction &CGF) {
 
   // If we don't need the cleanup at all, we're done.
   if (!RequiresNormalCleanup && !RequiresEHCleanup) {
-    CGF.EHStack.popCleanup();
-    assert(CGF.EHStack.getNumBranchFixups() == 0 ||
-           CGF.EHStack.hasNormalCleanups());
+    EHStack.popCleanup(); // safe because there are no fixups
+    assert(EHStack.getNumBranchFixups() == 0 ||
+           EHStack.hasNormalCleanups());
     return;
   }
 
@@ -912,319 +815,527 @@ static void PopLazyCleanupBlock(CodeGenFunction &CGF) {
   memcpy(CleanupBuffer.data(),
          Scope.getCleanupBuffer(), Scope.getCleanupSize());
   CleanupBuffer.set_size(Scope.getCleanupSize());
-  EHScopeStack::LazyCleanup *Fn =
-    reinterpret_cast<EHScopeStack::LazyCleanup*>(CleanupBuffer.data());
+  EHScopeStack::Cleanup *Fn =
+    reinterpret_cast<EHScopeStack::Cleanup*>(CleanupBuffer.data());
+
+  // We want to emit the EH cleanup after the normal cleanup, but go
+  // ahead and do the setup for the EH cleanup while the scope is still
+  // alive.
+  llvm::BasicBlock *EHEntry = 0;
+  llvm::SmallVector<llvm::Instruction*, 2> EHInstsToAppend;
+  if (RequiresEHCleanup) {
+    EHEntry = CreateEHEntry(*this, Scope);
+
+    // Figure out the branch-through dest if necessary.
+    llvm::BasicBlock *EHBranchThroughDest = 0;
+    if (Scope.hasEHBranchThroughs()) {
+      assert(Scope.getEnclosingEHCleanup() != EHStack.stable_end());
+      EHScope &S = *EHStack.find(Scope.getEnclosingEHCleanup());
+      EHBranchThroughDest = CreateEHEntry(*this, cast<EHCleanupScope>(S));
+    }
 
-  // We're done with the scope; pop it off so we can emit the cleanups.
-  CGF.EHStack.popCleanup();
+    // If we have exactly one branch-after and no branch-throughs, we
+    // can dispatch it without a switch.
+    if (!Scope.hasEHBranchThroughs() &&
+        Scope.getNumEHBranchAfters() == 1) {
+      assert(!EHBranchThroughDest);
+
+      // TODO: remove the spurious eh.cleanup.dest stores if this edge
+      // never went through any switches.
+      llvm::BasicBlock *BranchAfterDest = Scope.getEHBranchAfterBlock(0);
+      EHInstsToAppend.push_back(llvm::BranchInst::Create(BranchAfterDest));
+    
+    // Otherwise, if we have any branch-afters, we need a switch.
+    } else if (Scope.getNumEHBranchAfters()) {
+      // The default of the switch belongs to the branch-throughs if
+      // they exist.
+      llvm::BasicBlock *Default =
+        (EHBranchThroughDest ? EHBranchThroughDest : getUnreachableBlock());
+
+      const unsigned SwitchCapacity = Scope.getNumEHBranchAfters();
+
+      llvm::LoadInst *Load =
+        new llvm::LoadInst(getEHCleanupDestSlot(), "cleanup.dest");
+      llvm::SwitchInst *Switch =
+        llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
+
+      EHInstsToAppend.push_back(Load);
+      EHInstsToAppend.push_back(Switch);
+
+      for (unsigned I = 0, E = Scope.getNumEHBranchAfters(); I != E; ++I)
+        Switch->addCase(Scope.getEHBranchAfterIndex(I),
+                        Scope.getEHBranchAfterBlock(I));
+
+    // Otherwise, we have only branch-throughs; jump to the next EH
+    // cleanup.
+    } else {
+      assert(EHBranchThroughDest);
+      EHInstsToAppend.push_back(llvm::BranchInst::Create(EHBranchThroughDest));
+    }
+  }
+
+  if (!RequiresNormalCleanup) {
+    EHStack.popCleanup();
+  } else {
+    // As a kindof crazy internal case, branch-through fall-throughs
+    // leave the insertion point set to the end of the last cleanup.
+    bool HasPrebranchedFallthrough =
+      (HasFallthrough && FallthroughSource->getTerminator());
+    assert(!HasPrebranchedFallthrough ||
+           FallthroughSource->getTerminator()->getSuccessor(0)
+             == Scope.getNormalBlock());
 
-  if (RequiresNormalCleanup) {
     // If we have a fallthrough and no other need for the cleanup,
     // emit it directly.
-    if (HasFallthrough && !HasFixups && !HasExistingBranches) {
-      EmitLazyCleanup(CGF, Fn, /*ForEH*/ false);
+    if (HasFallthrough && !HasPrebranchedFallthrough &&
+        !HasFixups && !HasExistingBranches) {
+
+      // Fixups can cause us to optimistically create a normal block,
+      // only to later have no real uses for it.  Just delete it in
+      // this case.
+      // TODO: we can potentially simplify all the uses after this.
+      if (Scope.getNormalBlock()) {
+        Scope.getNormalBlock()->replaceAllUsesWith(getUnreachableBlock());
+        delete Scope.getNormalBlock();
+      }
+
+      EHStack.popCleanup();
+
+      EmitCleanup(*this, Fn, /*ForEH*/ false);
 
     // Otherwise, the best approach is to thread everything through
     // the cleanup block and then try to clean up after ourselves.
     } else {
       // Force the entry block to exist.
-      if (!HasExistingBranches) {
-        NormalEntry = CGF.createBasicBlock("cleanup");
-        CreateCleanupSwitch(CGF, NormalEntry);
+      llvm::BasicBlock *NormalEntry = CreateNormalEntry(*this, Scope);
+
+      // If there's a fallthrough, we need to store the cleanup
+      // destination index.  For fall-throughs this is always zero.
+      if (HasFallthrough && !HasPrebranchedFallthrough)
+        Builder.CreateStore(Builder.getInt32(0), getNormalCleanupDestSlot());
+
+      // Emit the entry block.  This implicitly branches to it if we
+      // have fallthrough.  All the fixups and existing branches should
+      // already be branched to it.
+      EmitBlock(NormalEntry);
+
+      bool HasEnclosingCleanups =
+        (Scope.getEnclosingNormalCleanup() != EHStack.stable_end());
+
+      // Compute the branch-through dest if we need it:
+      //   - if there are branch-throughs threaded through the scope
+      //   - if fall-through is a branch-through
+      //   - if there are fixups that will be optimistically forwarded
+      //     to the enclosing cleanup
+      llvm::BasicBlock *BranchThroughDest = 0;
+      if (Scope.hasBranchThroughs() ||
+          (HasFallthrough && FallthroughIsBranchThrough) ||
+          (HasFixups && HasEnclosingCleanups)) {
+        assert(HasEnclosingCleanups);
+        EHScope &S = *EHStack.find(Scope.getEnclosingNormalCleanup());
+        BranchThroughDest = CreateNormalEntry(*this, cast<EHCleanupScope>(S));
       }
 
-      CGF.EmitBlock(NormalEntry);
-
-      // Thread the fallthrough edge through the (momentarily trivial)
-      // cleanup.
-      llvm::BasicBlock *FallthroughDestination = 0;
-      if (HasFallthrough) {
-        assert(isa<llvm::BranchInst>(FallthroughSource->getTerminator()));
-        FallthroughDestination = CGF.createBasicBlock("cleanup.cont");
-
-        BranchFixup Fix;
-        Fix.Destination = FallthroughDestination;
-        Fix.LatestBranch = FallthroughSource->getTerminator();
-        Fix.LatestBranchIndex = 0;
-        Fix.Origin = Fix.LatestBranch;
+      llvm::BasicBlock *FallthroughDest = 0;
+      llvm::SmallVector<llvm::Instruction*, 2> InstsToAppend;
+
+      // If there's exactly one branch-after and no other threads,
+      // we can route it without a switch.
+      if (!Scope.hasBranchThroughs() && !HasFixups && !HasFallthrough &&
+          Scope.getNumBranchAfters() == 1) {
+        assert(!BranchThroughDest);
+
+        // TODO: clean up the possibly dead stores to the cleanup dest slot.
+        llvm::BasicBlock *BranchAfter = Scope.getBranchAfterBlock(0);
+        InstsToAppend.push_back(llvm::BranchInst::Create(BranchAfter));
+
+      // Build a switch-out if we need it:
+      //   - if there are branch-afters threaded through the scope
+      //   - if fall-through is a branch-after
+      //   - if there are fixups that have nowhere left to go and
+      //     so must be immediately resolved
+      } else if (Scope.getNumBranchAfters() ||
+                 (HasFallthrough && !FallthroughIsBranchThrough) ||
+                 (HasFixups && !HasEnclosingCleanups)) {
+
+        llvm::BasicBlock *Default =
+          (BranchThroughDest ? BranchThroughDest : getUnreachableBlock());
+
+        // TODO: base this on the number of branch-afters and fixups
+        const unsigned SwitchCapacity = 10;
+
+        llvm::LoadInst *Load =
+          new llvm::LoadInst(getNormalCleanupDestSlot(), "cleanup.dest");
+        llvm::SwitchInst *Switch =
+          llvm::SwitchInst::Create(Load, Default, SwitchCapacity);
+
+        InstsToAppend.push_back(Load);
+        InstsToAppend.push_back(Switch);
+
+        // Branch-after fallthrough.
+        if (HasFallthrough && !FallthroughIsBranchThrough) {
+          FallthroughDest = createBasicBlock("cleanup.cont");
+          Switch->addCase(Builder.getInt32(0), FallthroughDest);
+        }
 
-        // Restore fixup invariant.  EmitBlock added a branch to the
-        // cleanup which we need to redirect to the destination.
-        cast<llvm::BranchInst>(Fix.LatestBranch)
-          ->setSuccessor(0, Fix.Destination);
+        for (unsigned I = 0, E = Scope.getNumBranchAfters(); I != E; ++I) {
+          Switch->addCase(Scope.getBranchAfterIndex(I),
+                          Scope.getBranchAfterBlock(I));
+        }
 
-        ThreadFixupThroughCleanup(CGF, Fix, NormalEntry, NormalEntry);
+        if (HasFixups && !HasEnclosingCleanups)
+          ResolveAllBranchFixups(Switch);
+      } else {
+        // We should always have a branch-through destination in this case.
+        assert(BranchThroughDest);
+        InstsToAppend.push_back(llvm::BranchInst::Create(BranchThroughDest));
       }
 
-      // Thread any "real" fixups we need to thread.
-      for (unsigned I = FixupDepth, E = CGF.EHStack.getNumBranchFixups();
-           I != E; ++I)
-        if (CGF.EHStack.getBranchFixup(I).Destination)
-          ThreadFixupThroughCleanup(CGF, CGF.EHStack.getBranchFixup(I),
-                                    NormalEntry, NormalEntry);
-
-      SplitAndEmitLazyCleanup(CGF, Fn, /*ForEH*/ false, NormalEntry);
-
-      if (HasFallthrough)
-        CGF.EmitBlock(FallthroughDestination);
+      // We're finally ready to pop the cleanup.
+      EHStack.popCleanup();
+      assert(EHStack.hasNormalCleanups() == HasEnclosingCleanups);
+
+      EmitCleanup(*this, Fn, /*ForEH*/ false);
+
+      // Append the prepared cleanup prologue from above.
+      llvm::BasicBlock *NormalExit = Builder.GetInsertBlock();
+      for (unsigned I = 0, E = InstsToAppend.size(); I != E; ++I)
+        NormalExit->getInstList().push_back(InstsToAppend[I]);
+
+      // Optimistically hope that any fixups will continue falling through.
+      for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
+           I < E; ++I) {
+        BranchFixup &Fixup = CGF.EHStack.getBranchFixup(I);
+        if (!Fixup.Destination) continue;
+        if (!Fixup.OptimisticBranchBlock) {
+          new llvm::StoreInst(Builder.getInt32(Fixup.DestinationIndex),
+                              getNormalCleanupDestSlot(),
+                              Fixup.InitialBranch);
+          Fixup.InitialBranch->setSuccessor(0, NormalEntry);
+        }
+        Fixup.OptimisticBranchBlock = NormalExit;
+      }
+      
+      if (FallthroughDest)
+        EmitBlock(FallthroughDest);
+      else if (!HasFallthrough)
+        Builder.ClearInsertionPoint();
+
+      // Check whether we can merge NormalEntry into a single predecessor.
+      // This might invalidate (non-IR) pointers to NormalEntry.
+      llvm::BasicBlock *NewNormalEntry =
+        SimplifyCleanupEntry(*this, NormalEntry);
+
+      // If it did invalidate those pointers, and NormalEntry was the same
+      // as NormalExit, go back and patch up the fixups.
+      if (NewNormalEntry != NormalEntry && NormalEntry == NormalExit)
+        for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
+               I < E; ++I)
+          CGF.EHStack.getBranchFixup(I).OptimisticBranchBlock = NewNormalEntry;
     }
   }
 
+  assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0);
+
   // Emit the EH cleanup if required.
   if (RequiresEHCleanup) {
-    CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
-    CGF.EmitBlock(EHEntry);
-    SplitAndEmitLazyCleanup(CGF, Fn, /*ForEH*/ true, EHEntry);
-    CGF.Builder.restoreIP(SavedIP);
-  }
-}
+    CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
 
-/// Pops a cleanup block.  If the block includes a normal cleanup, the
-/// current insertion point is threaded through the cleanup, as are
-/// any branch fixups on the cleanup.
-void CodeGenFunction::PopCleanupBlock() {
-  assert(!EHStack.empty() && "cleanup stack is empty!");
-  if (isa<EHLazyCleanupScope>(*EHStack.begin()))
-    return PopLazyCleanupBlock(*this);
+    EmitBlock(EHEntry);
+    EmitCleanup(*this, Fn, /*ForEH*/ true);
 
-  assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
-  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
-  assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
+    // Append the prepared cleanup prologue from above.
+    llvm::BasicBlock *EHExit = Builder.GetInsertBlock();
+    for (unsigned I = 0, E = EHInstsToAppend.size(); I != E; ++I)
+      EHExit->getInstList().push_back(EHInstsToAppend[I]);
 
-  // Handle the EH cleanup if (1) there is one and (2) it's different
-  // from the normal cleanup.
-  if (Scope.isEHCleanup() &&
-      Scope.getEHEntry() != Scope.getNormalEntry()) {
-    llvm::BasicBlock *EHEntry = Scope.getEHEntry();
-    llvm::BasicBlock *EHExit = Scope.getEHExit();
-    
-    if (EHEntry->use_empty()) {
-      DestroyCleanup(*this, EHEntry, EHExit);
-    } else {
-      // TODO: this isn't really the ideal location to put this EH
-      // cleanup, but lazy emission is a better solution than trying
-      // to pick a better spot.
-      CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
-      EmitBlock(EHEntry);
-      Builder.restoreIP(SavedIP);
-
-      SimplifyCleanupEdges(*this, EHEntry, EHExit);
-    }
-  }
+    Builder.restoreIP(SavedIP);
 
-  // If we only have an EH cleanup, we don't really need to do much
-  // here.  Branch fixups just naturally drop down to the enclosing
-  // cleanup scope.
-  if (!Scope.isNormalCleanup()) {
-    EHStack.popCleanup();
-    assert(EHStack.getNumBranchFixups() == 0 || EHStack.hasNormalCleanups());
-    return;
+    SimplifyCleanupEntry(*this, EHEntry);
   }
+}
 
-  // Check whether the scope has any fixups that need to be threaded.
-  unsigned FixupDepth = Scope.getFixupDepth();
-  bool HasFixups = EHStack.getNumBranchFixups() != FixupDepth;
+/// Terminate the current block by emitting a branch which might leave
+/// the current cleanup-protected scope.  The target scope may not yet
+/// be known, in which case this will require a fixup.
+///
+/// As a side-effect, this method clears the insertion point.
+void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
+  assert(Dest.getScopeDepth().encloses(EHStack.getInnermostNormalCleanup())
+         && "stale jump destination");
 
-  // Grab the entry and exit blocks.
-  llvm::BasicBlock *Entry = Scope.getNormalEntry();
-  llvm::BasicBlock *Exit = Scope.getNormalExit();
+  if (!HaveInsertPoint())
+    return;
 
-  // Check whether anything's been threaded through the cleanup already.
-  assert((Exit->getTerminator() == 0) == Entry->use_empty() &&
-         "cleanup entry/exit mismatch");
-  bool HasExistingBranches = !Entry->use_empty();
+  // Create the branch.
+  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
 
-  // Check whether we need to emit a "fallthrough" branch through the
-  // cleanup for the current insertion point.
-  llvm::BasicBlock *FallThrough = Builder.GetInsertBlock();
-  if (FallThrough && FallThrough->getTerminator())
-    FallThrough = 0;
+  // Calculate the innermost active normal cleanup.
+  EHScopeStack::stable_iterator
+    TopCleanup = EHStack.getInnermostActiveNormalCleanup();
 
-  // If *nothing* is using the cleanup, kill it.
-  if (!FallThrough && !HasFixups && !HasExistingBranches) {
-    EHStack.popCleanup();
-    DestroyCleanup(*this, Entry, Exit);
+  // If we're not in an active normal cleanup scope, or if the
+  // destination scope is within the innermost active normal cleanup
+  // scope, we don't need to worry about fixups.
+  if (TopCleanup == EHStack.stable_end() ||
+      TopCleanup.encloses(Dest.getScopeDepth())) { // works for invalid
+    Builder.ClearInsertionPoint();
     return;
   }
 
-  // Otherwise, add the block to the function.
-  EmitBlock(Entry);
+  // If we can't resolve the destination cleanup scope, just add this
+  // to the current cleanup scope as a branch fixup.
+  if (!Dest.getScopeDepth().isValid()) {
+    BranchFixup &Fixup = EHStack.addBranchFixup();
+    Fixup.Destination = Dest.getBlock();
+    Fixup.DestinationIndex = Dest.getDestIndex();
+    Fixup.InitialBranch = BI;
+    Fixup.OptimisticBranchBlock = 0;
 
-  if (FallThrough)
-    Builder.SetInsertPoint(Exit);
-  else
     Builder.ClearInsertionPoint();
-
-  // Fast case: if we don't have to add any fixups, and either
-  // we don't have a fallthrough or the cleanup wasn't previously
-  // used, then the setup above is sufficient.
-  if (!HasFixups) {
-    if (!FallThrough) {
-      assert(HasExistingBranches && "no reason for cleanup but didn't kill before");
-      EHStack.popCleanup();
-      SimplifyCleanupEdges(*this, Entry, Exit);
-      return;
-    } else if (!HasExistingBranches) {
-      assert(FallThrough && "no reason for cleanup but didn't kill before");
-      // We can't simplify the exit edge in this case because we're
-      // already inserting at the end of the exit block.
-      EHStack.popCleanup();
-      SimplifyCleanupEntry(*this, Entry);
-      return;
-    }
+    return;
   }
 
-  // Otherwise we're going to have to thread things through the cleanup.
-  llvm::SmallVector<BranchFixup*, 8> Fixups;
-
-  // Synthesize a fixup for the current insertion point.
-  BranchFixup Cur;
-  if (FallThrough) {
-    Cur.Destination = createBasicBlock("cleanup.cont");
-    Cur.LatestBranch = FallThrough->getTerminator();
-    Cur.LatestBranchIndex = 0;
-    Cur.Origin = Cur.LatestBranch;
-
-    // Restore fixup invariant.  EmitBlock added a branch to the cleanup
-    // which we need to redirect to the destination.
-    cast<llvm::BranchInst>(Cur.LatestBranch)->setSuccessor(0, Cur.Destination);
+  // Otherwise, thread through all the normal cleanups in scope.
 
-    Fixups.push_back(&Cur);
-  } else {
-    Cur.Destination = 0;
-  }
+  // Store the index at the start.
+  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
+  new llvm::StoreInst(Index, getNormalCleanupDestSlot(), BI);
 
-  // Collect any "real" fixups we need to thread.
-  for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
-        I != E; ++I)
-    if (EHStack.getBranchFixup(I).Destination)
-      Fixups.push_back(&EHStack.getBranchFixup(I));
-
-  assert(!Fixups.empty() && "no fixups, invariants broken!");
-
-  // If there's only a single fixup to thread through, do so with
-  // unconditional branches.  This only happens if there's a single
-  // branch and no fallthrough.
-  if (Fixups.size() == 1 && !HasExistingBranches) {
-    Fixups[0]->LatestBranch->setSuccessor(Fixups[0]->LatestBranchIndex, Entry);
-    llvm::BranchInst *Br =
-      llvm::BranchInst::Create(Fixups[0]->Destination, Exit);
-    Fixups[0]->LatestBranch = Br;
-    Fixups[0]->LatestBranchIndex = 0;
-
-  // Otherwise, force a switch statement and thread everything through
-  // the switch.
-  } else {
-    CreateCleanupSwitch(*this, Exit);
-    for (unsigned I = 0, E = Fixups.size(); I != E; ++I)
-      ThreadFixupThroughCleanup(*this, *Fixups[I], Entry, Exit);
+  // Adjust BI to point to the first cleanup block.
+  {
+    EHCleanupScope &Scope =
+      cast<EHCleanupScope>(*EHStack.find(TopCleanup));
+    BI->setSuccessor(0, CreateNormalEntry(*this, Scope));
   }
 
-  // Emit the fallthrough destination block if necessary.
-  if (Cur.Destination)
-    EmitBlock(Cur.Destination);
+  // Add this destination to all the scopes involved.
+  EHScopeStack::stable_iterator I = TopCleanup;
+  EHScopeStack::stable_iterator E = Dest.getScopeDepth();
+  if (E.strictlyEncloses(I)) {
+    while (true) {
+      EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
+      assert(Scope.isNormalCleanup());
+      I = Scope.getEnclosingNormalCleanup();
+
+      // If this is the last cleanup we're propagating through, tell it
+      // that there's a resolved jump moving through it.
+      if (!E.strictlyEncloses(I)) {
+        Scope.addBranchAfter(Index, Dest.getBlock());
+        break;
+      }
 
-  // We're finally done with the cleanup.
-  EHStack.popCleanup();
+      // Otherwise, tell the scope that there's a jump propoagating
+      // through it.  If this isn't new information, all the rest of
+      // the work has been done before.
+      if (!Scope.addBranchThrough(Dest.getBlock()))
+        break;
+    }
+  }
+  
+  Builder.ClearInsertionPoint();
 }
 
-void CodeGenFunction::EmitBranchThroughCleanup(JumpDest Dest) {
+void CodeGenFunction::EmitBranchThroughEHCleanup(UnwindDest Dest) {
+  // We should never get invalid scope depths for an UnwindDest; that
+  // implies that the destination wasn't set up correctly.
+  assert(Dest.getScopeDepth().isValid() && "invalid scope depth on EH dest?");
+
   if (!HaveInsertPoint())
     return;
 
   // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.Block);
+  llvm::BranchInst *BI = Builder.CreateBr(Dest.getBlock());
 
-  // If we're not in a cleanup scope, we don't need to worry about
-  // fixups.
-  if (!EHStack.hasNormalCleanups()) {
+  // Calculate the innermost active cleanup.
+  EHScopeStack::stable_iterator
+    InnermostCleanup = EHStack.getInnermostActiveEHCleanup();
+
+  // If the destination is in the same EH cleanup scope as us, we
+  // don't need to thread through anything.
+  if (InnermostCleanup.encloses(Dest.getScopeDepth())) {
     Builder.ClearInsertionPoint();
     return;
   }
+  assert(InnermostCleanup != EHStack.stable_end());
 
-  // Initialize a fixup.
-  BranchFixup Fixup;
-  Fixup.Destination = Dest.Block;
-  Fixup.Origin = BI;
-  Fixup.LatestBranch = BI;
-  Fixup.LatestBranchIndex = 0;
+  // Store the index at the start.
+  llvm::ConstantInt *Index = Builder.getInt32(Dest.getDestIndex());
+  new llvm::StoreInst(Index, getEHCleanupDestSlot(), BI);
 
-  // If we can't resolve the destination cleanup scope, just add this
-  // to the current cleanup scope.
-  if (!Dest.ScopeDepth.isValid()) {
-    EHStack.addBranchFixup() = Fixup;
-    Builder.ClearInsertionPoint();
-    return;
+  // Adjust BI to point to the first cleanup block.
+  {
+    EHCleanupScope &Scope =
+      cast<EHCleanupScope>(*EHStack.find(InnermostCleanup));
+    BI->setSuccessor(0, CreateEHEntry(*this, Scope));
   }
-
-  for (EHScopeStack::iterator I = EHStack.begin(),
-         E = EHStack.find(Dest.ScopeDepth); I != E; ++I) {
-    if (isa<EHCleanupScope>(*I)) {
-      EHCleanupScope &Scope = cast<EHCleanupScope>(*I);
-      if (Scope.isNormalCleanup())
-        ThreadFixupThroughCleanup(*this, Fixup, Scope.getNormalEntry(),
-                                  Scope.getNormalExit());
-    } else if (isa<EHLazyCleanupScope>(*I)) {
-      EHLazyCleanupScope &Scope = cast<EHLazyCleanupScope>(*I);
-      if (Scope.isNormalCleanup()) {
-        llvm::BasicBlock *Block = Scope.getNormalBlock();
-        if (!Block) {
-          Block = createBasicBlock("cleanup");
-          Scope.setNormalBlock(Block);
-        }
-        ThreadFixupThroughCleanup(*this, Fixup, Block, Block);
-      }
+  
+  // Add this destination to all the scopes involved.
+  for (EHScopeStack::stable_iterator
+         I = InnermostCleanup, E = Dest.getScopeDepth(); ; ) {
+    assert(E.strictlyEncloses(I));
+    EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(I));
+    assert(Scope.isEHCleanup());
+    I = Scope.getEnclosingEHCleanup();
+
+    // If this is the last cleanup we're propagating through, add this
+    // as a branch-after.
+    if (I == E) {
+      Scope.addEHBranchAfter(Index, Dest.getBlock());
+      break;
     }
+
+    // Otherwise, add it as a branch-through.  If this isn't new
+    // information, all the rest of the work has been done before.
+    if (!Scope.addEHBranchThrough(Dest.getBlock()))
+      break;
   }
   
   Builder.ClearInsertionPoint();
 }
 
-void CodeGenFunction::EmitBranchThroughEHCleanup(JumpDest Dest) {
-  if (!HaveInsertPoint())
-    return;
+/// All the branch fixups on the EH stack have propagated out past the
+/// outermost normal cleanup; resolve them all by adding cases to the
+/// given switch instruction.
+void CodeGenFunction::ResolveAllBranchFixups(llvm::SwitchInst *Switch) {
+  llvm::SmallPtrSet<llvm::BasicBlock*, 4> CasesAdded;
+
+  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
+    // Skip this fixup if its destination isn't set or if we've
+    // already treated it.
+    BranchFixup &Fixup = EHStack.getBranchFixup(I);
+    if (Fixup.Destination == 0) continue;
+    if (!CasesAdded.insert(Fixup.Destination)) continue;
+
+    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex),
+                    Fixup.Destination);
+  }
 
-  // Create the branch.
-  llvm::BranchInst *BI = Builder.CreateBr(Dest.Block);
+  EHStack.clearFixups();
+}
 
-  // If we're not in a cleanup scope, we don't need to worry about
-  // fixups.
-  if (!EHStack.hasEHCleanups()) {
-    Builder.ClearInsertionPoint();
-    return;
+void CodeGenFunction::ResolveBranchFixups(llvm::BasicBlock *Block) {
+  assert(Block && "resolving a null target block");
+  if (!EHStack.getNumBranchFixups()) return;
+
+  assert(EHStack.hasNormalCleanups() &&
+         "branch fixups exist with no normal cleanups on stack");
+
+  llvm::SmallPtrSet<llvm::BasicBlock*, 4> ModifiedOptimisticBlocks;
+  bool ResolvedAny = false;
+
+  for (unsigned I = 0, E = EHStack.getNumBranchFixups(); I != E; ++I) {
+    // Skip this fixup if its destination doesn't match.
+    BranchFixup &Fixup = EHStack.getBranchFixup(I);
+    if (Fixup.Destination != Block) continue;
+
+    Fixup.Destination = 0;
+    ResolvedAny = true;
+
+    // If it doesn't have an optimistic branch block, LatestBranch is
+    // already pointing to the right place.
+    llvm::BasicBlock *BranchBB = Fixup.OptimisticBranchBlock;
+    if (!BranchBB)
+      continue;
+
+    // Don't process the same optimistic branch block twice.
+    if (!ModifiedOptimisticBlocks.insert(BranchBB))
+      continue;
+
+    llvm::SwitchInst *Switch = TransitionToCleanupSwitch(*this, BranchBB);
+
+    // Add a case to the switch.
+    Switch->addCase(Builder.getInt32(Fixup.DestinationIndex), Block);
   }
 
-  // Initialize a fixup.
-  BranchFixup Fixup;
-  Fixup.Destination = Dest.Block;
-  Fixup.Origin = BI;
-  Fixup.LatestBranch = BI;
-  Fixup.LatestBranchIndex = 0;
-
-  // We should never get invalid scope depths for these: invalid scope
-  // depths only arise for as-yet-unemitted labels, and we can't do an
-  // EH-unwind to one of those.
-  assert(Dest.ScopeDepth.isValid() && "invalid scope depth on EH dest?");
-
-  for (EHScopeStack::iterator I = EHStack.begin(),
-         E = EHStack.find(Dest.ScopeDepth); I != E; ++I) {
-    if (isa<EHCleanupScope>(*I)) {
-      EHCleanupScope &Scope = cast<EHCleanupScope>(*I);
-      if (Scope.isEHCleanup())
-        ThreadFixupThroughCleanup(*this, Fixup, Scope.getEHEntry(),
-                                  Scope.getEHExit());
-    } else if (isa<EHLazyCleanupScope>(*I)) {
-      EHLazyCleanupScope &Scope = cast<EHLazyCleanupScope>(*I);
-      if (Scope.isEHCleanup()) {
-        llvm::BasicBlock *Block = Scope.getEHBlock();
-        if (!Block) {
-          Block = createBasicBlock("eh.cleanup");
-          Scope.setEHBlock(Block);
+  if (ResolvedAny)
+    EHStack.popNullFixups();
+}
+
+/// Activate a cleanup that was created in an inactivated state.
+void CodeGenFunction::ActivateCleanup(EHScopeStack::stable_iterator C) {
+  assert(C != EHStack.stable_end() && "activating bottom of stack?");
+  EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.find(C));
+  assert(!Scope.isActive() && "double activation");
+
+  // Calculate whether the cleanup was used:
+  bool Used = false;
+
+  //   - as a normal cleanup
+  if (Scope.isNormalCleanup()) {
+    bool NormalUsed = false;
+    if (Scope.getNormalBlock()) {
+      NormalUsed = true;
+    } else {
+      // Check whether any enclosed cleanups were needed.
+      for (EHScopeStack::stable_iterator
+             I = EHStack.getInnermostNormalCleanup(); I != C; ) {
+        assert(C.strictlyEncloses(I));
+        EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
+        if (S.getNormalBlock()) {
+          NormalUsed = true;
+          break;
         }
-        ThreadFixupThroughCleanup(*this, Fixup, Block, Block);
+        I = S.getEnclosingNormalCleanup();
       }
     }
+
+    if (NormalUsed)
+      Used = true;
+    else
+      Scope.setActivatedBeforeNormalUse(true);
+  }
+
+  //  - as an EH cleanup
+  if (Scope.isEHCleanup()) {
+    bool EHUsed = false;
+    if (Scope.getEHBlock()) {
+      EHUsed = true;
+    } else {
+      // Check whether any enclosed cleanups were needed.
+      for (EHScopeStack::stable_iterator
+             I = EHStack.getInnermostEHCleanup(); I != C; ) {
+        assert(C.strictlyEncloses(I));
+        EHCleanupScope &S = cast<EHCleanupScope>(*EHStack.find(I));
+        if (S.getEHBlock()) {
+          EHUsed = true;
+          break;
+        }
+        I = S.getEnclosingEHCleanup();
+      }
+    }
+
+    if (EHUsed)
+      Used = true;
+    else
+      Scope.setActivatedBeforeEHUse(true);
   }
   
-  Builder.ClearInsertionPoint();
+  llvm::AllocaInst *Var = EHCleanupScope::activeSentinel();
+  if (Used) {
+    Var = CreateTempAlloca(Builder.getInt1Ty());
+    InitTempAlloca(Var, Builder.getFalse());
+  }
+  Scope.setActiveVar(Var);
+}
+
+llvm::Value *CodeGenFunction::getNormalCleanupDestSlot() {
+  if (!NormalCleanupDest)
+    NormalCleanupDest =
+      CreateTempAlloca(Builder.getInt32Ty(), "cleanup.dest.slot");
+  return NormalCleanupDest;
+}
+
+llvm::Value *CodeGenFunction::getEHCleanupDestSlot() {
+  if (!EHCleanupDest)
+    EHCleanupDest =
+      CreateTempAlloca(Builder.getInt32Ty(), "eh.cleanup.dest.slot");
+  return EHCleanupDest;
+}
+
+void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E, 
+                                              llvm::ConstantInt *Init) {
+  assert (Init && "Invalid DeclRefExpr initializer!");
+  if (CGDebugInfo *Dbg = getDebugInfo())
+    Dbg->EmitGlobalVariable(E->getDecl(), Init, Builder);
 }
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index 5ee3db08eea0..4f0420536ad2 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -41,6 +41,7 @@ namespace llvm {
 }
 
 namespace clang {
+  class APValue;
   class ASTContext;
   class CXXDestructorDecl;
   class CXXTryStmt;
@@ -69,6 +70,7 @@ namespace CodeGen {
   class CGFunctionInfo;
   class CGRecordLayout;
   class CGBlockInfo;
+  class CGCXXABI;
 
 /// A branch fixup.  These are required when emitting a goto to a
 /// label which hasn't been emitted yet.  The goto is optimistically
@@ -77,25 +79,34 @@ namespace CodeGen {
 /// the innermost cleanup.  When a (normal) cleanup is popped, any
 /// unresolved fixups in that scope are threaded through the cleanup.
 struct BranchFixup {
-  /// The origin of the branch.  Any switch-index stores required by
-  /// cleanup threading are added before this instruction.
-  llvm::Instruction *Origin;
+  /// The block containing the terminator which needs to be modified
+  /// into a switch if this fixup is resolved into the current scope.
+  /// If null, LatestBranch points directly to the destination.
+  llvm::BasicBlock *OptimisticBranchBlock;
 
-  /// The destination of the branch.
+  /// The ultimate destination of the branch.
   ///
   /// This can be set to null to indicate that this fixup was
   /// successfully resolved.
   llvm::BasicBlock *Destination;
 
-  /// The last branch of the fixup.  It is an invariant that
-  /// LatestBranch->getSuccessor(LatestBranchIndex) == Destination.
-  ///
-  /// The branch is always either a BranchInst or a SwitchInst.
-  llvm::TerminatorInst *LatestBranch;
-  unsigned LatestBranchIndex;
+  /// The destination index value.
+  unsigned DestinationIndex;
+
+  /// The initial branch of the fixup.
+  llvm::BranchInst *InitialBranch;
 };
 
-enum CleanupKind { NormalAndEHCleanup, EHCleanup, NormalCleanup };
+enum CleanupKind {
+  EHCleanup = 0x1,
+  NormalCleanup = 0x2,
+  NormalAndEHCleanup = EHCleanup | NormalCleanup,
+
+  InactiveCleanup = 0x4,
+  InactiveEHCleanup = EHCleanup | InactiveCleanup,
+  InactiveNormalCleanup = NormalCleanup | InactiveCleanup,
+  InactiveNormalAndEHCleanup = NormalAndEHCleanup | InactiveCleanup
+};
 
 /// A stack of scopes which respond to exceptions, including cleanups
 /// and catch blocks.
@@ -117,6 +128,17 @@ public:
 
     bool isValid() const { return Size >= 0; }
 
+    /// Returns true if this scope encloses I.
+    /// Returns false if I is invalid.
+    /// This scope must be valid.
+    bool encloses(stable_iterator I) const { return Size <= I.Size; }
+
+    /// Returns true if this scope strictly encloses I: that is,
+    /// if it encloses I and is not I.
+    /// Returns false is I is invalid.
+    /// This scope must be valid.
+    bool strictlyEncloses(stable_iterator I) const { return Size < I.Size; }
+
     friend bool operator==(stable_iterator A, stable_iterator B) {
       return A.Size == B.Size;
     }
@@ -125,13 +147,14 @@ public:
     }
   };
 
-  /// A lazy cleanup.  Subclasses must be POD-like:  cleanups will
-  /// not be destructed, and they will be allocated on the cleanup
-  /// stack and freely copied and moved around.
+  /// Information for lazily generating a cleanup.  Subclasses must be
+  /// POD-like: cleanups will not be destructed, and they will be
+  /// allocated on the cleanup stack and freely copied and moved
+  /// around.
   ///
-  /// LazyCleanup implementations should generally be declared in an
+  /// Cleanup implementations should generally be declared in an
   /// anonymous namespace.
-  class LazyCleanup {
+  class Cleanup {
   public:
     // Anchor the construction vtable.  We use the destructor because
     // gcc gives an obnoxious warning if there are virtual methods
@@ -140,7 +163,7 @@ public:
     // doesn't seem to be any other way around this warning.
     //
     // This destructor will never be called.
-    virtual ~LazyCleanup();
+    virtual ~Cleanup();
 
     /// Emit the cleanup.  For normal cleanups, this is run in the
     /// same EH context as when the cleanup was pushed, i.e. the
@@ -177,6 +200,11 @@ private:
   /// The number of catches on the stack.
   unsigned CatchDepth;
 
+  /// The current EH destination index.  Reset to FirstCatchIndex
+  /// whenever the last EH cleanup is popped.
+  unsigned NextEHDestIndex;
+  enum { FirstEHDestIndex = 1 };
+
   /// The current set of branch fixups.  A branch fixup is a jump to
   /// an as-yet unemitted label, i.e. a label for which we don't yet
   /// know the EH stack depth.  Whenever we pop a cleanup, we have
@@ -198,64 +226,64 @@ private:
 
   char *allocate(size_t Size);
 
-  void popNullFixups();
-
-  void *pushLazyCleanup(CleanupKind K, size_t DataSize);
+  void *pushCleanup(CleanupKind K, size_t DataSize);
 
 public:
   EHScopeStack() : StartOfBuffer(0), EndOfBuffer(0), StartOfData(0),
                    InnermostNormalCleanup(stable_end()),
                    InnermostEHCleanup(stable_end()),
-                   CatchDepth(0) {}
+                   CatchDepth(0), NextEHDestIndex(FirstEHDestIndex) {}
   ~EHScopeStack() { delete[] StartOfBuffer; }
 
   // Variadic templates would make this not terrible.
 
   /// Push a lazily-created cleanup on the stack.
   template <class T>
-  void pushLazyCleanup(CleanupKind Kind) {
-    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
-    LazyCleanup *Obj = new(Buffer) T();
+  void pushCleanup(CleanupKind Kind) {
+    void *Buffer = pushCleanup(Kind, sizeof(T));
+    Cleanup *Obj = new(Buffer) T();
     (void) Obj;
   }
 
   /// Push a lazily-created cleanup on the stack.
   template <class T, class A0>
-  void pushLazyCleanup(CleanupKind Kind, A0 a0) {
-    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
-    LazyCleanup *Obj = new(Buffer) T(a0);
+  void pushCleanup(CleanupKind Kind, A0 a0) {
+    void *Buffer = pushCleanup(Kind, sizeof(T));
+    Cleanup *Obj = new(Buffer) T(a0);
     (void) Obj;
   }
 
   /// Push a lazily-created cleanup on the stack.
   template <class T, class A0, class A1>
-  void pushLazyCleanup(CleanupKind Kind, A0 a0, A1 a1) {
-    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
-    LazyCleanup *Obj = new(Buffer) T(a0, a1);
+  void pushCleanup(CleanupKind Kind, A0 a0, A1 a1) {
+    void *Buffer = pushCleanup(Kind, sizeof(T));
+    Cleanup *Obj = new(Buffer) T(a0, a1);
     (void) Obj;
   }
 
   /// Push a lazily-created cleanup on the stack.
   template <class T, class A0, class A1, class A2>
-  void pushLazyCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2) {
-    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
-    LazyCleanup *Obj = new(Buffer) T(a0, a1, a2);
+  void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2) {
+    void *Buffer = pushCleanup(Kind, sizeof(T));
+    Cleanup *Obj = new(Buffer) T(a0, a1, a2);
     (void) Obj;
   }
 
   /// Push a lazily-created cleanup on the stack.
   template <class T, class A0, class A1, class A2, class A3>
-  void pushLazyCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) {
-    void *Buffer = pushLazyCleanup(Kind, sizeof(T));
-    LazyCleanup *Obj = new(Buffer) T(a0, a1, a2, a3);
+  void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3) {
+    void *Buffer = pushCleanup(Kind, sizeof(T));
+    Cleanup *Obj = new(Buffer) T(a0, a1, a2, a3);
     (void) Obj;
   }
 
-  /// Push a cleanup on the stack.
-  void pushCleanup(llvm::BasicBlock *NormalEntry,
-                   llvm::BasicBlock *NormalExit,
-                   llvm::BasicBlock *EHEntry,
-                   llvm::BasicBlock *EHExit);
+  /// Push a lazily-created cleanup on the stack.
+  template <class T, class A0, class A1, class A2, class A3, class A4>
+  void pushCleanup(CleanupKind Kind, A0 a0, A1 a1, A2 a2, A3 a3, A4 a4) {
+    void *Buffer = pushCleanup(Kind, sizeof(T));
+    Cleanup *Obj = new(Buffer) T(a0, a1, a2, a3, a4);
+    (void) Obj;
+  }
 
   /// Pops a cleanup scope off the stack.  This should only be called
   /// by CodeGenFunction::PopCleanupBlock.
@@ -298,6 +326,7 @@ public:
   stable_iterator getInnermostNormalCleanup() const {
     return InnermostNormalCleanup;
   }
+  stable_iterator getInnermostActiveNormalCleanup() const; // CGException.h
 
   /// Determines whether there are any EH cleanups on the stack.
   bool hasEHCleanups() const {
@@ -309,6 +338,7 @@ public:
   stable_iterator getInnermostEHCleanup() const {
     return InnermostEHCleanup;
   }
+  stable_iterator getInnermostActiveEHCleanup() const; // CGException.h
 
   /// An unstable reference to a scope-stack depth.  Invalidated by
   /// pushes but not pops.
@@ -359,8 +389,17 @@ public:
     return BranchFixups[I];
   }
 
-  /// Mark any branch fixups leading to the given block as resolved.
-  void resolveBranchFixups(llvm::BasicBlock *Dest);
+  /// Pops lazily-removed fixups from the end of the list.  This
+  /// should only be called by procedures which have just popped a
+  /// cleanup or resolved one or more fixups.
+  void popNullFixups();
+
+  /// Clears the branch-fixups list.  This should only be called by
+  /// CodeGenFunction::ResolveAllBranchFixups.
+  void clearFixups() { BranchFixups.clear(); }
+
+  /// Gets the next EH destination index.
+  unsigned getNextEHDestIndex() { return NextEHDestIndex++; }
 };
 
 /// CodeGenFunction - This class organizes the per-function state that is used
@@ -368,17 +407,47 @@ public:
 class CodeGenFunction : public BlockFunction {
   CodeGenFunction(const CodeGenFunction&); // DO NOT IMPLEMENT
   void operator=(const CodeGenFunction&);  // DO NOT IMPLEMENT
+
+  friend class CGCXXABI;
 public:
-  /// A jump destination is a pair of a basic block and a cleanup
-  /// depth.  They are used to implement direct jumps across cleanup
-  /// scopes, e.g. goto, break, continue, and return.
+  /// A jump destination is an abstract label, branching to which may
+  /// require a jump out through normal cleanups.
   struct JumpDest {
-    JumpDest() : Block(0), ScopeDepth() {}
-    JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth)
-      : Block(Block), ScopeDepth(Depth) {}
+    JumpDest() : Block(0), ScopeDepth(), Index(0) {}
+    JumpDest(llvm::BasicBlock *Block,
+             EHScopeStack::stable_iterator Depth,
+             unsigned Index)
+      : Block(Block), ScopeDepth(Depth), Index(Index) {}
+
+    bool isValid() const { return Block != 0; }
+    llvm::BasicBlock *getBlock() const { return Block; }
+    EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
+    unsigned getDestIndex() const { return Index; }
     
+  private:
+    llvm::BasicBlock *Block;
+    EHScopeStack::stable_iterator ScopeDepth;
+    unsigned Index;
+  };
+
+  /// An unwind destination is an abstract label, branching to which
+  /// may require a jump out through EH cleanups.
+  struct UnwindDest {
+    UnwindDest() : Block(0), ScopeDepth(), Index(0) {}
+    UnwindDest(llvm::BasicBlock *Block,
+               EHScopeStack::stable_iterator Depth,
+               unsigned Index)
+      : Block(Block), ScopeDepth(Depth), Index(Index) {}
+
+    bool isValid() const { return Block != 0; }
+    llvm::BasicBlock *getBlock() const { return Block; }
+    EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }
+    unsigned getDestIndex() const { return Index; }
+
+  private:
     llvm::BasicBlock *Block;
     EHScopeStack::stable_iterator ScopeDepth;
+    unsigned Index;
   };
 
   CodeGenModule &CGM;  // Per-module state.
@@ -406,6 +475,9 @@ public:
   /// iff the function has no return value.
   llvm::Value *ReturnValue;
 
+  /// RethrowBlock - Unified rethrow block.
+  UnwindDest RethrowBlock;
+
   /// AllocaInsertPoint - This is an instruction in the entry block before which
   /// we prefer to insert allocas.
   llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;
@@ -423,6 +495,12 @@ public:
 
   EHScopeStack EHStack;
 
+  /// i32s containing the indexes of the cleanup destinations.
+  llvm::AllocaInst *NormalCleanupDest;
+  llvm::AllocaInst *EHCleanupDest;
+
+  unsigned NextCleanupDestIndex;
+
   /// The exception slot.  All landing pads write the current
   /// exception pointer into this alloca.
   llvm::Value *ExceptionSlot;
@@ -454,30 +532,17 @@ public:
   /// non-trivial destructor.
   void PushDestructorCleanup(QualType T, llvm::Value *Addr);
 
+  /// PushDestructorCleanup - Push a cleanup to call the
+  /// complete-object variant of the given destructor on the object at
+  /// the given address.
+  void PushDestructorCleanup(const CXXDestructorDecl *Dtor,
+                             llvm::Value *Addr);
+
   /// PopCleanupBlock - Will pop the cleanup entry on the stack and
   /// process all branch fixups.
-  void PopCleanupBlock();
-
-  /// CleanupBlock - RAII object that will create a cleanup block and
-  /// set the insert point to that block. When destructed, it sets the
-  /// insert point to the previous block and pushes a new cleanup
-  /// entry on the stack.
-  class CleanupBlock {
-    CodeGenFunction &CGF;
-    CGBuilderTy::InsertPoint SavedIP;
-    llvm::BasicBlock *NormalCleanupEntryBB;
-    llvm::BasicBlock *NormalCleanupExitBB;
-    llvm::BasicBlock *EHCleanupEntryBB;
-    
-  public:
-    CleanupBlock(CodeGenFunction &CGF, CleanupKind Kind);
+  void PopCleanupBlock(bool FallThroughIsBranchThrough = false);
 
-    /// If we're currently writing a normal cleanup, tie that off and
-    /// start writing an EH cleanup.
-    void beginEHCleanup();
-    
-    ~CleanupBlock();
-  };
+  void ActivateCleanup(EHScopeStack::stable_iterator Cleanup);
 
   /// \brief Enters a new scope for capturing cleanups, all of which
   /// will be executed once the scope is exited.
@@ -528,18 +593,23 @@ public:
   /// the cleanup blocks that have been added.
   void PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize);
 
+  void ResolveAllBranchFixups(llvm::SwitchInst *Switch);
+  void ResolveBranchFixups(llvm::BasicBlock *Target);
+
   /// The given basic block lies in the current EH scope, but may be a
   /// target of a potentially scope-crossing jump; get a stable handle
   /// to which we can perform this jump later.
-  JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) const {
-    return JumpDest(Target, EHStack.stable_begin());
+  JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {
+    return JumpDest(Target,
+                    EHStack.getInnermostNormalCleanup(),
+                    NextCleanupDestIndex++);
   }
 
   /// The given basic block lies in the current EH scope, but may be a
   /// target of a potentially scope-crossing jump; get a stable handle
   /// to which we can perform this jump later.
   JumpDest getJumpDestInCurrentScope(const char *Name = 0) {
-    return JumpDest(createBasicBlock(Name), EHStack.stable_begin());
+    return getJumpDestInCurrentScope(createBasicBlock(Name));
   }
 
   /// EmitBranchThroughCleanup - Emit a branch from the current insert
@@ -550,7 +620,11 @@ public:
   /// EmitBranchThroughEHCleanup - Emit a branch from the current
   /// insert block through the EH cleanup handling code (if any) and
   /// then on to \arg Dest.
-  void EmitBranchThroughEHCleanup(JumpDest Dest);
+  void EmitBranchThroughEHCleanup(UnwindDest Dest);
+
+  /// getRethrowDest - Returns the unified outermost-scope rethrow
+  /// destination.
+  UnwindDest getRethrowDest();
 
   /// BeginConditionalBranch - Should be called before a conditional part of an
   /// expression is emitted. For example, before the RHS of the expression below
@@ -608,10 +682,6 @@ private:
   /// statement range in current switch instruction.
   llvm::BasicBlock *CaseRangeBlock;
 
-  /// InvokeDest - This is the nearest exception target for calls
-  /// which can unwind, when exceptions are being used.
-  llvm::BasicBlock *InvokeDest;
-
   // VLASizeMap - This keeps track of the associated size for each VLA type.
   // We track this by the size expression rather than the type itself because
   // in certain situations, like a const qualifier applied to an VLA typedef,
@@ -661,6 +731,7 @@ private:
 public:
   CodeGenFunction(CodeGenModule &cgm);
 
+  CodeGenTypes &getTypes() const { return CGM.getTypes(); }
   ASTContext &getContext() const;
   CGDebugInfo *getDebugInfo() { return DebugInfo; }
 
@@ -668,6 +739,9 @@ public:
   /// is assigned in every landing pad.
   llvm::Value *getExceptionSlot();
 
+  llvm::Value *getNormalCleanupDestSlot();
+  llvm::Value *getEHCleanupDestSlot();
+
   llvm::BasicBlock *getUnreachableBlock() {
     if (!UnreachableBlock) {
       UnreachableBlock = createBasicBlock("unreachable");
@@ -711,15 +785,16 @@ public:
 
   llvm::Value *BuildBlockLiteralTmp(const BlockExpr *);
   llvm::Constant *BuildDescriptorBlockDecl(const BlockExpr *,
-                                           bool BlockHasCopyDispose,
-                                           CharUnits Size,
+                                           const CGBlockInfo &Info,
                                            const llvm::StructType *,
+                                           llvm::Constant *BlockVarLayout,
                                            std::vector<HelperInfo> *);
 
   llvm::Function *GenerateBlockFunction(GlobalDecl GD,
                                         const BlockExpr *BExpr,
                                         CGBlockInfo &Info,
                                         const Decl *OuterFuncDecl,
+                                        llvm::Constant *& BlockVarLayout,
                                   llvm::DenseMap<const Decl*, llvm::Value*> ldm);
 
   llvm::Value *LoadBlockStruct();
@@ -777,11 +852,11 @@ public:
   void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
 
 
-  /// EmitDtorEpilogue - Emit all code that comes at the end of class's
-  /// destructor. This is to call destructors on members and base classes in
-  /// reverse order of their construction.
-  void EmitDtorEpilogue(const CXXDestructorDecl *Dtor,
-                        CXXDtorType Type);
+  /// EnterDtorCleanups - Enter the cleanups necessary to complete the
+  /// given phase of destruction for a destructor.  The end result
+  /// should call destructors on members and base classes in reverse
+  /// order of their construction.
+  void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);
 
   /// ShouldInstrumentFunction - Return true if the current function should be
   /// instrumented with __cyg_profile_func_* calls
@@ -898,10 +973,8 @@ public:
   //                                  Helpers
   //===--------------------------------------------------------------------===//
 
-  Qualifiers MakeQualifiers(QualType T) {
-    Qualifiers Quals = getContext().getCanonicalType(T).getQualifiers();
-    Quals.setObjCGCAttr(getContext().getObjCGCAttrKind(T));
-    return Quals;
+  LValue MakeAddrLValue(llvm::Value *V, QualType T, unsigned Alignment = 0) {
+    return LValue::MakeAddr(V, T, Alignment, getContext());
   }
 
   /// CreateTempAlloca - This creates a alloca and inserts it into the entry
@@ -965,10 +1038,16 @@ public:
   void StartBlock(const char *N);
 
   /// GetAddrOfStaticLocalVar - Return the address of a static local variable.
-  llvm::Constant *GetAddrOfStaticLocalVar(const VarDecl *BVD);
+  llvm::Constant *GetAddrOfStaticLocalVar(const VarDecl *BVD) {
+    return cast<llvm::Constant>(GetAddrOfLocalVar(BVD));
+  }
 
   /// GetAddrOfLocalVar - Return the address of a local variable.
-  llvm::Value *GetAddrOfLocalVar(const VarDecl *VD);
+  llvm::Value *GetAddrOfLocalVar(const VarDecl *VD) {
+    llvm::Value *Res = LocalDeclMap[VD];
+    assert(Res && "Invalid argument to GetAddrOfLocalVar(), no decl!");
+    return Res;
+  }
 
   /// getAccessedFieldNo - Given an encoded value and a result number, return
   /// the input field number being accessed.
@@ -1025,12 +1104,14 @@ public:
   /// load of 'this' and returns address of the base class.
   llvm::Value *GetAddressOfBaseClass(llvm::Value *Value, 
                                      const CXXRecordDecl *Derived,
-                                     const CXXBaseSpecifierArray &BasePath, 
+                                     CastExpr::path_const_iterator PathBegin,
+                                     CastExpr::path_const_iterator PathEnd,
                                      bool NullCheckValue);
 
   llvm::Value *GetAddressOfDerivedClass(llvm::Value *Value,
                                         const CXXRecordDecl *Derived,
-                                        const CXXBaseSpecifierArray &BasePath,
+                                        CastExpr::path_const_iterator PathBegin,
+                                        CastExpr::path_const_iterator PathEnd,
                                         bool NullCheckValue);
 
   llvm::Value *GetVirtualBaseClassOffset(llvm::Value *This,
@@ -1049,13 +1130,15 @@ public:
                                   const ConstantArrayType *ArrayTy,
                                   llvm::Value *ArrayPtr,
                                   CallExpr::const_arg_iterator ArgBeg,
-                                  CallExpr::const_arg_iterator ArgEnd);
+                                  CallExpr::const_arg_iterator ArgEnd,
+                                  bool ZeroInitialization = false);
   
   void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,
                                   llvm::Value *NumElements,
                                   llvm::Value *ArrayPtr,
                                   CallExpr::const_arg_iterator ArgBeg,
-                                  CallExpr::const_arg_iterator ArgEnd);
+                                  CallExpr::const_arg_iterator ArgEnd,
+                                  bool ZeroInitialization = false);
 
   void EmitCXXAggrDestructorCall(const CXXDestructorDecl *D,
                                  const ArrayType *Array,
@@ -1224,13 +1307,13 @@ public:
   /// care to appropriately convert from the memory representation to
   /// the LLVM value representation.
   llvm::Value *EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
-                                QualType Ty);
+                                unsigned Alignment, QualType Ty);
 
   /// EmitStoreOfScalar - Store a scalar value to an address, taking
   /// care to appropriately convert from the memory representation to
   /// the LLVM value representation.
   void EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
-                         bool Volatile, QualType Ty);
+                         bool Volatile, unsigned Alignment, QualType Ty);
 
   /// EmitLoadOfLValue - Given an expression that represents a value lvalue,
   /// this method emits the address of the lvalue, then loads the result as an
@@ -1270,7 +1353,6 @@ public:
   LValue EmitDeclRefLValue(const DeclRefExpr *E);
   LValue EmitStringLiteralLValue(const StringLiteral *E);
   LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);
-  LValue EmitPredefinedFunctionName(unsigned Type);
   LValue EmitPredefinedLValue(const PredefinedExpr *E);
   LValue EmitUnaryOpLValue(const UnaryOperator *E);
   LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E);
@@ -1319,7 +1401,7 @@ public:
   LValue EmitStmtExprLValue(const StmtExpr *E);
   LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);
   LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);
-  
+  void   EmitDeclRefExprDbgValue(const DeclRefExpr *E, llvm::ConstantInt *Init);
   //===--------------------------------------------------------------------===//
   //                         Scalar Expression Emission
   //===--------------------------------------------------------------------===//
@@ -1386,7 +1468,8 @@ public:
                             llvm::SmallVectorImpl<llvm::Value*> &O,
                             const char *name, bool splat = false,
                             unsigned shift = 0, bool rightshift = false);
-  llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);
+  llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,
+                             bool widen = false);
   llvm::Value *EmitNeonShiftVector(llvm::Value *V, const llvm::Type *Ty,
                                    bool negateForRightShift);
   
@@ -1542,7 +1625,7 @@ public:
 
   /// getTrapBB - Create a basic block that will call the trap intrinsic.  We'll
   /// generate a branch around the created basic block as necessary.
-  llvm::BasicBlock* getTrapBB();
+  llvm::BasicBlock *getTrapBB();
   
   /// EmitCallArg - Emit a single call argument.
   RValue EmitCallArg(const Expr *E, QualType ArgType);
@@ -1575,6 +1658,11 @@ private:
                             const TargetInfo::ConstraintInfo &Info,
                             const Expr *InputExpr, std::string &ConstraintStr);
 
+  llvm::Value* EmitAsmInputLValue(const AsmStmt &S,
+                                  const TargetInfo::ConstraintInfo &Info,
+                                  LValue InputValue, QualType InputType,
+                                  std::string &ConstraintStr);
+
   /// EmitCallArgs - Emit call arguments for a function.
   /// The CallArgTypeInfo parameter is used for iterating over the known
   /// argument types of the function being called.
@@ -1622,7 +1710,36 @@ private:
   void EmitDeclMetadata();
 };
 
-
+/// CGBlockInfo - Information to generate a block literal.
+class CGBlockInfo {
+public:
+  /// Name - The name of the block, kindof.
+  const char *Name;
+    
+  /// DeclRefs - Variables from parent scopes that have been
+  /// imported into this block.
+  llvm::SmallVector<const BlockDeclRefExpr *, 8> DeclRefs;
+    
+  /// InnerBlocks - This block and the blocks it encloses.
+  llvm::SmallPtrSet<const DeclContext *, 4> InnerBlocks;
+    
+  /// CXXThisRef - Non-null if 'this' was required somewhere, in
+  /// which case this is that expression.
+  const CXXThisExpr *CXXThisRef;
+    
+  /// NeedsObjCSelf - True if something in this block has an implicit
+  /// reference to 'self'.
+  bool NeedsObjCSelf;
+    
+  /// These are initialized by GenerateBlockFunction.
+  bool BlockHasCopyDispose;
+  CharUnits BlockSize;
+  CharUnits BlockAlign;
+  llvm::SmallVector<const Expr*, 8> BlockLayout;
+    
+  CGBlockInfo(const char *Name);
+};
+  
 }  // end namespace CodeGen
 }  // end namespace clang
 
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index bf606a616582..d125b370a07a 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -15,6 +15,7 @@
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
 #include "CGCall.h"
+#include "CGCXXABI.h"
 #include "CGObjCRuntime.h"
 #include "Mangle.h"
 #include "TargetInfo.h"
@@ -41,6 +42,17 @@
 using namespace clang;
 using namespace CodeGen;
 
+static CGCXXABI &createCXXABI(CodeGenModule &CGM) {
+  switch (CGM.getContext().Target.getCXXABI()) {
+  case CXXABI_ARM: return *CreateARMCXXABI(CGM);
+  case CXXABI_Itanium: return *CreateItaniumCXXABI(CGM);
+  case CXXABI_Microsoft: return *CreateMicrosoftCXXABI(CGM);
+  }
+
+  llvm_unreachable("invalid C++ ABI kind");
+  return *CreateItaniumCXXABI(CGM);
+}
+
 
 CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
                              llvm::Module &M, const llvm::TargetData &TD,
@@ -48,11 +60,15 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
   : BlockModule(C, M, TD, Types, *this), Context(C),
     Features(C.getLangOptions()), CodeGenOpts(CGO), TheModule(M),
     TheTargetData(TD), TheTargetCodeGenInfo(0), Diags(diags),
-    Types(C, M, TD, getTargetCodeGenInfo().getABIInfo()),
-    VTables(*this), Runtime(0), ABI(0),
-    CFConstantStringClassRef(0),
-    NSConstantStringClassRef(0),
-    VMContext(M.getContext()) {
+    ABI(createCXXABI(*this)), 
+    Types(C, M, TD, getTargetCodeGenInfo().getABIInfo(), ABI),
+    VTables(*this), Runtime(0),
+    CFConstantStringClassRef(0), NSConstantStringClassRef(0),
+    VMContext(M.getContext()),
+    NSConcreteGlobalBlockDecl(0), NSConcreteStackBlockDecl(0),
+    NSConcreteGlobalBlock(0), NSConcreteStackBlock(0),
+    BlockObjectAssignDecl(0), BlockObjectDisposeDecl(0),
+    BlockObjectAssign(0), BlockObjectDispose(0){
 
   if (!Features.ObjC1)
     Runtime = 0;
@@ -63,17 +79,13 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
   else
     Runtime = CreateMacObjCRuntime(*this);
 
-  if (!Features.CPlusPlus)
-    ABI = 0;
-  else createCXXABI();
-
   // If debug info generation is enabled, create the CGDebugInfo object.
   DebugInfo = CodeGenOpts.DebugInfo ? new CGDebugInfo(*this) : 0;
 }
 
 CodeGenModule::~CodeGenModule() {
   delete Runtime;
-  delete ABI;
+  delete &ABI;
   delete DebugInfo;
 }
 
@@ -86,13 +98,6 @@ void CodeGenModule::createObjCRuntime() {
     Runtime = CreateMacObjCRuntime(*this);
 }
 
-void CodeGenModule::createCXXABI() {
-  if (Context.Target.getCXXABI() == "microsoft")
-    ABI = CreateMicrosoftCXXABI(*this);
-  else
-    ABI = CreateItaniumCXXABI(*this);
-}
-
 void CodeGenModule::Release() {
   EmitDeferred();
   EmitCXXGlobalInitFunc();
@@ -141,17 +146,17 @@ void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type,
 LangOptions::VisibilityMode
 CodeGenModule::getDeclVisibilityMode(const Decl *D) const {
   if (const VarDecl *VD = dyn_cast<VarDecl>(D))
-    if (VD->getStorageClass() == VarDecl::PrivateExtern)
+    if (VD->getStorageClass() == SC_PrivateExtern)
       return LangOptions::Hidden;
 
   if (const VisibilityAttr *attr = D->getAttr<VisibilityAttr>()) {
     switch (attr->getVisibility()) {
     default: assert(0 && "Unknown visibility!");
-    case VisibilityAttr::DefaultVisibility:
+    case VisibilityAttr::Default:
       return LangOptions::Default;
-    case VisibilityAttr::HiddenVisibility:
+    case VisibilityAttr::Hidden:
       return LangOptions::Hidden;
-    case VisibilityAttr::ProtectedVisibility:
+    case VisibilityAttr::Protected:
       return LangOptions::Protected;
     }
   }
@@ -187,9 +192,11 @@ CodeGenModule::getDeclVisibilityMode(const Decl *D) const {
           return LangOptions::Hidden;
   }
            
-  // This decl should have the same visibility as its parent.
+  // If this decl is contained in a class, it should have the same visibility
+  // as the parent class.
   if (const DeclContext *DC = D->getDeclContext()) 
-    return getDeclVisibilityMode(cast<Decl>(DC));
+    if (DC->isRecord())
+      return getDeclVisibilityMode(cast<Decl>(DC));
 
   return getLangOptions().getVisibilityMode();
 }
@@ -213,6 +220,67 @@ void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV,
   }
 }
 
+/// Set the symbol visibility of type information (vtable and RTTI)
+/// associated with the given type.
+void CodeGenModule::setTypeVisibility(llvm::GlobalValue *GV,
+                                      const CXXRecordDecl *RD,
+                                      bool IsForRTTI) const {
+  setGlobalVisibility(GV, RD);
+
+  if (!CodeGenOpts.HiddenWeakVTables)
+    return;
+
+  // We want to drop the visibility to hidden for weak type symbols.
+  // This isn't possible if there might be unresolved references
+  // elsewhere that rely on this symbol being visible.
+
+  // This should be kept roughly in sync with setThunkVisibility
+  // in CGVTables.cpp.
+
+  // Preconditions.
+  if (GV->getLinkage() != llvm::GlobalVariable::WeakODRLinkage ||
+      GV->getVisibility() != llvm::GlobalVariable::DefaultVisibility)
+    return;
+
+  // Don't override an explicit visibility attribute.
+  if (RD->hasAttr<VisibilityAttr>())
+    return;
+
+  switch (RD->getTemplateSpecializationKind()) {
+  // We have to disable the optimization if this is an EI definition
+  // because there might be EI declarations in other shared objects.
+  case TSK_ExplicitInstantiationDefinition:
+  case TSK_ExplicitInstantiationDeclaration:
+    return;
+
+  // Every use of a non-template class's type information has to emit it.
+  case TSK_Undeclared:
+    break;
+
+  // In theory, implicit instantiations can ignore the possibility of
+  // an explicit instantiation declaration because there necessarily
+  // must be an EI definition somewhere with default visibility.  In
+  // practice, it's possible to have an explicit instantiation for
+  // an arbitrary template class, and linkers aren't necessarily able
+  // to deal with mixed-visibility symbols.
+  case TSK_ExplicitSpecialization:
+  case TSK_ImplicitInstantiation:
+    if (!CodeGenOpts.HiddenWeakTemplateVTables)
+      return;
+    break;
+  }
+
+  // If there's a key function, there may be translation units
+  // that don't have the key function's definition.  But ignore
+  // this if we're emitting RTTI under -fno-rtti.
+  if (!IsForRTTI || Features.RTTI)
+    if (Context.getKeyFunction(RD))
+      return;
+
+  // Otherwise, drop the visibility to hidden.
+  GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
+}
+
 llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
 
@@ -220,7 +288,7 @@ llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   if (!Str.empty())
     return Str;
 
-  if (!getMangleContext().shouldMangleDeclName(ND)) {
+  if (!getCXXABI().getMangleContext().shouldMangleDeclName(ND)) {
     IdentifierInfo *II = ND->getIdentifier();
     assert(II && "Attempt to mangle unnamed decl.");
 
@@ -230,13 +298,13 @@ llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   
   llvm::SmallString<256> Buffer;
   if (const CXXConstructorDecl *D = dyn_cast<CXXConstructorDecl>(ND))
-    getMangleContext().mangleCXXCtor(D, GD.getCtorType(), Buffer);
+    getCXXABI().getMangleContext().mangleCXXCtor(D, GD.getCtorType(), Buffer);
   else if (const CXXDestructorDecl *D = dyn_cast<CXXDestructorDecl>(ND))
-    getMangleContext().mangleCXXDtor(D, GD.getDtorType(), Buffer);
+    getCXXABI().getMangleContext().mangleCXXDtor(D, GD.getDtorType(), Buffer);
   else if (const BlockDecl *BD = dyn_cast<BlockDecl>(ND))
-    getMangleContext().mangleBlock(GD, BD, Buffer);
+    getCXXABI().getMangleContext().mangleBlock(GD, BD, Buffer);
   else
-    getMangleContext().mangleName(ND, Buffer);
+    getCXXABI().getMangleContext().mangleName(ND, Buffer);
 
   // Allocate space for the mangled name.
   size_t Length = Buffer.size();
@@ -250,7 +318,7 @@ llvm::StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
 
 void CodeGenModule::getMangledName(GlobalDecl GD, MangleBuffer &Buffer,
                                    const BlockDecl *BD) {
-  getMangleContext().mangleBlock(GD, BD, Buffer.getBuffer());
+  getCXXABI().getMangleContext().mangleBlock(GD, BD, Buffer.getBuffer());
 }
 
 llvm::GlobalValue *CodeGenModule::GetGlobalValue(llvm::StringRef Name) {
@@ -319,68 +387,9 @@ void CodeGenModule::EmitAnnotations() {
   gv->setSection("llvm.metadata");
 }
 
-static CodeGenModule::GVALinkage
-GetLinkageForFunction(ASTContext &Context, const FunctionDecl *FD,
-                      const LangOptions &Features) {
-  CodeGenModule::GVALinkage External = CodeGenModule::GVA_StrongExternal;
-
-  Linkage L = FD->getLinkage();
-  if (L == ExternalLinkage && Context.getLangOptions().CPlusPlus &&
-      FD->getType()->getLinkage() == UniqueExternalLinkage)
-    L = UniqueExternalLinkage;
-  
-  switch (L) {
-  case NoLinkage:
-  case InternalLinkage:
-  case UniqueExternalLinkage:
-    return CodeGenModule::GVA_Internal;
-    
-  case ExternalLinkage:
-    switch (FD->getTemplateSpecializationKind()) {
-    case TSK_Undeclared:
-    case TSK_ExplicitSpecialization:
-      External = CodeGenModule::GVA_StrongExternal;
-      break;
-
-    case TSK_ExplicitInstantiationDefinition:
-      return CodeGenModule::GVA_ExplicitTemplateInstantiation;
-
-    case TSK_ExplicitInstantiationDeclaration:
-    case TSK_ImplicitInstantiation:
-      External = CodeGenModule::GVA_TemplateInstantiation;
-      break;
-    }
-  }
-
-  if (!FD->isInlined())
-    return External;
-    
-  if (!Features.CPlusPlus || FD->hasAttr<GNUInlineAttr>()) {
-    // GNU or C99 inline semantics. Determine whether this symbol should be
-    // externally visible.
-    if (FD->isInlineDefinitionExternallyVisible())
-      return External;
-
-    // C99 inline semantics, where the symbol is not externally visible.
-    return CodeGenModule::GVA_C99Inline;
-  }
-
-  // C++0x [temp.explicit]p9:
-  //   [ Note: The intent is that an inline function that is the subject of 
-  //   an explicit instantiation declaration will still be implicitly 
-  //   instantiated when used so that the body can be considered for 
-  //   inlining, but that no out-of-line copy of the inline function would be
-  //   generated in the translation unit. -- end note ]
-  if (FD->getTemplateSpecializationKind() 
-                                       == TSK_ExplicitInstantiationDeclaration)
-    return CodeGenModule::GVA_C99Inline;
-
-  return CodeGenModule::GVA_CXXInline;
-}
-
 llvm::GlobalValue::LinkageTypes
 CodeGenModule::getFunctionLinkage(const FunctionDecl *D) {
-  GVALinkage Linkage = GetLinkageForFunction(getContext(), D, Features);
+  GVALinkage Linkage = getContext().GetGVALinkageForFunction(D);
 
   if (Linkage == GVA_Internal)
     return llvm::Function::InternalLinkage;
@@ -454,12 +463,10 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
   else if (Features.getStackProtectorMode() == LangOptions::SSPReq)
     F->addFnAttr(llvm::Attribute::StackProtectReq);
   
-  if (const AlignedAttr *AA = D->getAttr<AlignedAttr>()) {
-    unsigned width = Context.Target.getCharWidth();
-    F->setAlignment(AA->getAlignment() / width);
-    while ((AA = AA->getNext<AlignedAttr>()))
-      F->setAlignment(std::max(F->getAlignment(), AA->getAlignment() / width));
-  }
+  unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
+  if (alignment)
+    F->setAlignment(alignment);
+
   // C++ ABI requires 2-byte alignment for member functions.
   if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D))
     F->setAlignment(2);
@@ -638,102 +645,12 @@ llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
   return llvm::ConstantStruct::get(VMContext, Fields, 4, false);
 }
 
-static CodeGenModule::GVALinkage
-GetLinkageForVariable(ASTContext &Context, const VarDecl *VD) {
-  // If this is a static data member, compute the kind of template
-  // specialization. Otherwise, this variable is not part of a
-  // template.
-  TemplateSpecializationKind TSK = TSK_Undeclared;
-  if (VD->isStaticDataMember())
-    TSK = VD->getTemplateSpecializationKind();
-
-  Linkage L = VD->getLinkage();
-  if (L == ExternalLinkage && Context.getLangOptions().CPlusPlus &&
-      VD->getType()->getLinkage() == UniqueExternalLinkage)
-    L = UniqueExternalLinkage;
-
-  switch (L) {
-  case NoLinkage:
-  case InternalLinkage:
-  case UniqueExternalLinkage:
-    return CodeGenModule::GVA_Internal;
-
-  case ExternalLinkage:
-    switch (TSK) {
-    case TSK_Undeclared:
-    case TSK_ExplicitSpecialization:
-      return CodeGenModule::GVA_StrongExternal;
-
-    case TSK_ExplicitInstantiationDeclaration:
-      llvm_unreachable("Variable should not be instantiated");
-      // Fall through to treat this like any other instantiation.
-        
-    case TSK_ExplicitInstantiationDefinition:
-      return CodeGenModule::GVA_ExplicitTemplateInstantiation;
-
-    case TSK_ImplicitInstantiation:
-      return CodeGenModule::GVA_TemplateInstantiation;      
-    }
-  }
-
-  return CodeGenModule::GVA_StrongExternal;
-}
-
 bool CodeGenModule::MayDeferGeneration(const ValueDecl *Global) {
-  // Never defer when EmitAllDecls is specified or the decl has
-  // attribute used.
-  if (Features.EmitAllDecls || Global->hasAttr<UsedAttr>())
-    return false;
-
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Global)) {
-    // Constructors and destructors should never be deferred.
-    if (FD->hasAttr<ConstructorAttr>() ||
-        FD->hasAttr<DestructorAttr>())
-      return false;
-
-    // The key function for a class must never be deferred.
-    if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Global)) {
-      const CXXRecordDecl *RD = MD->getParent();
-      if (MD->isOutOfLine() && RD->isDynamicClass()) {
-        const CXXMethodDecl *KeyFunction = getContext().getKeyFunction(RD);
-        if (KeyFunction && 
-            KeyFunction->getCanonicalDecl() == MD->getCanonicalDecl())
-          return false;
-      }
-    }
-
-    GVALinkage Linkage = GetLinkageForFunction(getContext(), FD, Features);
-
-    // static, static inline, always_inline, and extern inline functions can
-    // always be deferred.  Normal inline functions can be deferred in C99/C++.
-    // Implicit template instantiations can also be deferred in C++.
-    if (Linkage == GVA_Internal || Linkage == GVA_C99Inline ||
-        Linkage == GVA_CXXInline || Linkage == GVA_TemplateInstantiation)
-      return true;
+  // Never defer when EmitAllDecls is specified.
+  if (Features.EmitAllDecls)
     return false;
-  }
 
-  const VarDecl *VD = cast<VarDecl>(Global);
-  assert(VD->isFileVarDecl() && "Invalid decl");
-
-  // We never want to defer structs that have non-trivial constructors or 
-  // destructors.
-  
-  // FIXME: Handle references.
-  if (const RecordType *RT = VD->getType()->getAs<RecordType>()) {
-    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
-      if (!RD->hasTrivialConstructor() || !RD->hasTrivialDestructor())
-        return false;
-    }
-  }
-      
-  GVALinkage L = GetLinkageForVariable(getContext(), VD);
-  if (L == GVA_Internal || L == GVA_TemplateInstantiation) {
-    if (!(VD->getInit() && VD->getInit()->HasSideEffects(Context)))
-      return true;
-  }
-
-  return false;
+  return !getContext().DeclMustBeEmitted(Global);
 }
 
 llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
@@ -774,6 +691,15 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
 
   // Ignore declarations, they will be emitted on their first use.
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Global)) {
+    if (FD->getIdentifier()) {
+      llvm::StringRef Name = FD->getName();
+      if (Name == "_Block_object_assign") {
+        BlockObjectAssignDecl = FD;
+      } else if (Name == "_Block_object_dispose") {
+        BlockObjectDisposeDecl = FD;
+      }
+    }
+
     // Forward declarations are emitted lazily on first use.
     if (!FD->isThisDeclarationADefinition())
       return;
@@ -781,6 +707,16 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     const VarDecl *VD = cast<VarDecl>(Global);
     assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
 
+    if (VD->getIdentifier()) {
+      llvm::StringRef Name = VD->getName();
+      if (Name == "_NSConcreteGlobalBlock") {
+        NSConcreteGlobalBlockDecl = VD;
+      } else if (Name == "_NSConcreteStackBlock") {
+        NSConcreteStackBlockDecl = VD;
+      }
+    }
+
+
     if (VD->isThisDeclarationADefinition() != VarDecl::Definition)
       return;
   }
@@ -792,6 +728,14 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     EmitGlobalDefinition(GD);
     return;
   }
+
+  // If we're deferring emission of a C++ variable with an
+  // initializer, remember the order in which it appeared in the file.
+  if (getLangOptions().CPlusPlus && isa<VarDecl>(Global) &&
+      cast<VarDecl>(Global)->hasInit()) {
+    DelayedCXXInitPosition[Global] = CXXGlobalInits.size();
+    CXXGlobalInits.push_back(0);
+  }
   
   // If the value has already been used, add it directly to the
   // DeferredDeclsToEmit list.
@@ -816,7 +760,8 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD) {
   if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
     // At -O0, don't generate IR for functions with available_externally 
     // linkage.
-    if (CodeGenOpts.OptimizationLevel == 0 &&
+    if (CodeGenOpts.OptimizationLevel == 0 && 
+        !Function->hasAttr<AlwaysInlineAttr>() &&
         getFunctionLinkage(Function) 
                                   == llvm::Function::AvailableExternallyLinkage)
       return;
@@ -1021,7 +966,7 @@ CodeGenModule::GetOrCreateLLVMGlobal(llvm::StringRef MangledName,
     GV->setConstant(DeclIsConstantGlobal(Context, D));
 
     // FIXME: Merge with other attribute handling code.
-    if (D->getStorageClass() == VarDecl::PrivateExtern)
+    if (D->getStorageClass() == SC_PrivateExtern)
       GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
 
     if (D->hasAttr<WeakAttr>() ||
@@ -1174,6 +1119,11 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
         ErrorUnsupported(D, "static initializer");
         Init = llvm::UndefValue::get(getTypes().ConvertType(T));
       }
+    } else {
+      // We don't need an initializer, so remove the entry for the delayed
+      // initializer position (just in case this entry was delayed).
+      if (getLangOptions().CPlusPlus)
+        DelayedCXXInitPosition.erase(D);
     }
   }
 
@@ -1235,7 +1185,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
   GV->setAlignment(getContext().getDeclAlign(D).getQuantity());
 
   // Set the llvm linkage type as appropriate.
-  GVALinkage Linkage = GetLinkageForVariable(getContext(), D);
+  GVALinkage Linkage = getContext().GetGVALinkageForVariable(D);
   if (Linkage == GVA_Internal)
     GV->setLinkage(llvm::Function::InternalLinkage);
   else if (D->hasAttr<DLLImportAttr>())
@@ -1254,7 +1204,8 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
     GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);   
   else if (!getLangOptions().CPlusPlus && !CodeGenOpts.NoCommon &&
            !D->hasExternalStorage() && !D->getInit() &&
-           !D->getAttr<SectionAttr>()) {
+           !D->getAttr<SectionAttr>() && !D->isThreadSpecified()) {
+    // Thread local vars aren't considered common linkage.
     GV->setLinkage(llvm::GlobalVariable::CommonLinkage);
     // common vars aren't constant even if declared const.
     GV->setConstant(false);
@@ -1293,6 +1244,7 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
     // TODO: Do invokes ever occur in C code?  If so, we should handle them too.
     llvm::Value::use_iterator I = UI++; // Increment before the CI is erased.
     llvm::CallInst *CI = dyn_cast<llvm::CallInst>(*I);
+    if (!CI) continue; // FIXME: when we allow Invoke, just do CallSite CS(*I)
     llvm::CallSite CS(CI);
     if (!CI || !CS.isCallee(I)) continue;
 
@@ -1343,7 +1295,7 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
 void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD) {
   const FunctionDecl *D = cast<FunctionDecl>(GD.getDecl());
   const llvm::FunctionType *Ty = getTypes().GetFunctionType(GD);
-  getMangleContext().mangleInitDiscriminator();
+  getCXXABI().getMangleContext().mangleInitDiscriminator();
   // Get or create the prototype for the function.
   llvm::Constant *Entry = GetAddrOfFunction(GD, Ty);
 
@@ -1528,18 +1480,18 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
                          bool TargetIsLSB,
                          bool &IsUTF16,
                          unsigned &StringLength) {
-  unsigned NumBytes = Literal->getByteLength();
+  llvm::StringRef String = Literal->getString();
+  unsigned NumBytes = String.size();
 
   // Check for simple case.
   if (!Literal->containsNonAsciiOrNull()) {
     StringLength = NumBytes;
-    return Map.GetOrCreateValue(llvm::StringRef(Literal->getStrData(),
-                                                StringLength));
+    return Map.GetOrCreateValue(String);
   }
 
   // Otherwise, convert the UTF8 literals into a byte string.
   llvm::SmallVector<UTF16, 128> ToBuf(NumBytes);
-  const UTF8 *FromPtr = (UTF8 *)Literal->getStrData();
+  const UTF8 *FromPtr = (UTF8 *)String.data();
   UTF16 *ToPtr = &ToBuf[0];
 
   ConversionResult Result = ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes,
@@ -1552,8 +1504,7 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
     // this duplicate code.
     assert(Result == sourceIllegal && "UTF-8 to UTF-16 conversion failed");
     StringLength = NumBytes;
-    return Map.GetOrCreateValue(llvm::StringRef(Literal->getStrData(),
-                                                StringLength));
+    return Map.GetOrCreateValue(String);
   }
 
   // ConvertUTF8toUTF16 returns the length in ToPtr.
@@ -1753,20 +1704,17 @@ CodeGenModule::GetAddrOfConstantNSString(const StringLiteral *Literal) {
 /// GetStringForStringLiteral - Return the appropriate bytes for a
 /// string literal, properly padded to match the literal type.
 std::string CodeGenModule::GetStringForStringLiteral(const StringLiteral *E) {
-  const char *StrData = E->getStrData();
-  unsigned Len = E->getByteLength();
-
   const ConstantArrayType *CAT =
     getContext().getAsConstantArrayType(E->getType());
   assert(CAT && "String isn't pointer or array!");
 
   // Resize the string to the right size.
-  std::string Str(StrData, StrData+Len);
   uint64_t RealLen = CAT->getSize().getZExtValue();
 
   if (E->isWide())
     RealLen *= getContext().Target.getWCharWidth()/8;
 
+  std::string Str = E->getString().str();
   Str.resize(RealLen, '\0');
 
   return Str;
@@ -1894,7 +1842,7 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
                                                   D->getLocation(),
                                                   D->getLocation(), cxxSelector,
                                                   getContext().VoidTy, 0, 
-                                                  DC, true, false, true,
+                                                  DC, true, false, true, false,
                                                   ObjCMethodDecl::Required);
   D->addInstanceMethod(DTORMethod);
   CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, DTORMethod, false);
@@ -1906,7 +1854,7 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
                                                 D->getLocation(),
                                                 D->getLocation(), cxxSelector,
                                                 getContext().getObjCIdType(), 0, 
-                                                DC, true, false, true,
+                                                DC, true, false, true, false,
                                                 ObjCMethodDecl::Required);
   D->addInstanceMethod(CTORMethod);
   CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, CTORMethod, true);
@@ -1993,9 +1941,16 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
   // Forward declarations, no (immediate) code generation.
   case Decl::ObjCClass:
   case Decl::ObjCForwardProtocol:
-  case Decl::ObjCCategory:
   case Decl::ObjCInterface:
     break;
+  
+    case Decl::ObjCCategory: {
+      ObjCCategoryDecl *CD = cast<ObjCCategoryDecl>(D);
+      if (CD->IsClassExtension() && CD->hasSynthBitfield())
+        Context.ResetObjCLayout(CD->getClassInterface());
+      break;
+    }
+      
 
   case Decl::ObjCProtocol:
     Runtime->GenerateProtocol(cast<ObjCProtocolDecl>(D));
@@ -2009,6 +1964,8 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
 
   case Decl::ObjCImplementation: {
     ObjCImplementationDecl *OMD = cast<ObjCImplementationDecl>(D);
+    if (Features.ObjCNonFragileABI2 && OMD->hasSynthBitfield())
+      Context.ResetObjCLayout(OMD->getClassInterface());
     EmitObjCPropertyImplementations(OMD);
     EmitObjCIvarInitializations(OMD);
     Runtime->GenerateClass(OMD);
@@ -2118,3 +2075,88 @@ void CodeGenFunction::EmitDeclMetadata() {
     }
   }
 }
+
+///@name Custom Runtime Function Interfaces
+///@{
+//
+// FIXME: These can be eliminated once we can have clients just get the required
+// AST nodes from the builtin tables.
+
+llvm::Constant *CodeGenModule::getBlockObjectDispose() {
+  if (BlockObjectDispose)
+    return BlockObjectDispose;
+
+  // If we saw an explicit decl, use that.
+  if (BlockObjectDisposeDecl) {
+    return BlockObjectDispose = GetAddrOfFunction(
+      BlockObjectDisposeDecl,
+      getTypes().GetFunctionType(BlockObjectDisposeDecl));
+  }
+
+  // Otherwise construct the function by hand.
+  const llvm::FunctionType *FTy;
+  std::vector<const llvm::Type*> ArgTys;
+  const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
+  ArgTys.push_back(PtrToInt8Ty);
+  ArgTys.push_back(llvm::Type::getInt32Ty(VMContext));
+  FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
+  return BlockObjectDispose =
+    CreateRuntimeFunction(FTy, "_Block_object_dispose");
+}
+
+llvm::Constant *CodeGenModule::getBlockObjectAssign() {
+  if (BlockObjectAssign)
+    return BlockObjectAssign;
+
+  // If we saw an explicit decl, use that.
+  if (BlockObjectAssignDecl) {
+    return BlockObjectAssign = GetAddrOfFunction(
+      BlockObjectAssignDecl,
+      getTypes().GetFunctionType(BlockObjectAssignDecl));
+  }
+
+  // Otherwise construct the function by hand.
+  const llvm::FunctionType *FTy;
+  std::vector<const llvm::Type*> ArgTys;
+  const llvm::Type *ResultType = llvm::Type::getVoidTy(VMContext);
+  ArgTys.push_back(PtrToInt8Ty);
+  ArgTys.push_back(PtrToInt8Ty);
+  ArgTys.push_back(llvm::Type::getInt32Ty(VMContext));
+  FTy = llvm::FunctionType::get(ResultType, ArgTys, false);
+  return BlockObjectAssign =
+    CreateRuntimeFunction(FTy, "_Block_object_assign");
+}
+
+llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
+  if (NSConcreteGlobalBlock)
+    return NSConcreteGlobalBlock;
+
+  // If we saw an explicit decl, use that.
+  if (NSConcreteGlobalBlockDecl) {
+    return NSConcreteGlobalBlock = GetAddrOfGlobalVar(
+      NSConcreteGlobalBlockDecl,
+      getTypes().ConvertType(NSConcreteGlobalBlockDecl->getType()));
+  }
+
+  // Otherwise construct the variable by hand.
+  return NSConcreteGlobalBlock = CreateRuntimeVariable(
+    PtrToInt8Ty, "_NSConcreteGlobalBlock");
+}
+
+llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
+  if (NSConcreteStackBlock)
+    return NSConcreteStackBlock;
+
+  // If we saw an explicit decl, use that.
+  if (NSConcreteStackBlockDecl) {
+    return NSConcreteStackBlock = GetAddrOfGlobalVar(
+      NSConcreteStackBlockDecl,
+      getTypes().ConvertType(NSConcreteStackBlockDecl->getType()));
+  }
+
+  // Otherwise construct the variable by hand.
+  return NSConcreteStackBlock = CreateRuntimeVariable(
+    PtrToInt8Ty, "_NSConcreteStackBlock");
+}
+
+///@}
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index 27f15fc018cd..cabff9e1cad5 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -22,7 +22,6 @@
 #include "CGCall.h"
 #include "CGCXX.h"
 #include "CGVTables.h"
-#include "CGCXXABI.h"
 #include "CodeGenTypes.h"
 #include "GlobalDecl.h"
 #include "Mangle.h"
@@ -71,6 +70,7 @@ namespace clang {
 namespace CodeGen {
 
   class CodeGenFunction;
+  class CGCXXABI;
   class CGDebugInfo;
   class CGObjCRuntime;
   class MangleBuffer;
@@ -109,6 +109,7 @@ class CodeGenModule : public BlockModule {
   const llvm::TargetData &TheTargetData;
   mutable const TargetCodeGenInfo *TheTargetCodeGenInfo;
   Diagnostic &Diags;
+  CGCXXABI &ABI;
   CodeGenTypes Types;
 
   /// VTables - Holds information about C++ vtables.
@@ -116,7 +117,6 @@ class CodeGenModule : public BlockModule {
   friend class CodeGenVTables;
 
   CGObjCRuntime* Runtime;
-  CXXABI* ABI;
   CGDebugInfo* DebugInfo;
 
   // WeakRefReferences - A set of references that have only been seen via
@@ -162,6 +162,12 @@ class CodeGenModule : public BlockModule {
   /// CXXGlobalInits - Global variables with initializers that need to run
   /// before main.
   std::vector<llvm::Constant*> CXXGlobalInits;
+
+  /// When a C++ decl with an initializer is deferred, null is
+  /// appended to CXXGlobalInits, and the index of that null is placed
+  /// here so that the initializer will be performed in the correct
+  /// order.
+  llvm::DenseMap<const Decl*, unsigned> DelayedCXXInitPosition;
   
   /// - Global variables with initializers whose order of initialization
   /// is set by init_priority attribute.
@@ -183,10 +189,23 @@ class CodeGenModule : public BlockModule {
 
   /// Lazily create the Objective-C runtime
   void createObjCRuntime();
-  /// Lazily create the C++ ABI
-  void createCXXABI();
 
   llvm::LLVMContext &VMContext;
+
+  /// @name Cache for Blocks Runtime Globals
+  /// @{
+
+  const VarDecl *NSConcreteGlobalBlockDecl;
+  const VarDecl *NSConcreteStackBlockDecl;
+  llvm::Constant *NSConcreteGlobalBlock;
+  llvm::Constant *NSConcreteStackBlock;
+
+  const FunctionDecl *BlockObjectAssignDecl;
+  const FunctionDecl *BlockObjectDisposeDecl;
+  llvm::Constant *BlockObjectAssign;
+  llvm::Constant *BlockObjectDispose;
+
+  /// @}
 public:
   CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts,
                 llvm::Module &M, const llvm::TargetData &TD, Diagnostic &Diags);
@@ -207,15 +226,8 @@ public:
   /// been configured.
   bool hasObjCRuntime() { return !!Runtime; }
 
-  /// getCXXABI() - Return a reference to the configured
-  /// C++ ABI.
-  CXXABI &getCXXABI() {
-    if (!ABI) createCXXABI();
-    return *ABI;
-  }
-
-  /// hasCXXABI() - Return true iff a C++ ABI has been configured.
-  bool hasCXXABI() { return !!ABI; }
+  /// getCXXABI() - Return a reference to the configured C++ ABI.
+  CGCXXABI &getCXXABI() { return ABI; }
 
   llvm::Value *getStaticLocalDeclAddress(const VarDecl *VD) {
     return StaticLocalDeclMap[VD];
@@ -231,15 +243,11 @@ public:
   const LangOptions &getLangOptions() const { return Features; }
   llvm::Module &getModule() const { return TheModule; }
   CodeGenTypes &getTypes() { return Types; }
-  MangleContext &getMangleContext() {
-    if (!ABI) createCXXABI();
-    return ABI->getMangleContext();
-  }
   CodeGenVTables &getVTables() { return VTables; }
   Diagnostic &getDiags() const { return Diags; }
   const llvm::TargetData &getTargetData() const { return TheTargetData; }
   llvm::LLVMContext &getLLVMContext() { return VMContext; }
-  const TargetCodeGenInfo &getTargetCodeGenInfo() const;
+  const TargetCodeGenInfo &getTargetCodeGenInfo();
   bool isTargetDarwin() const;
 
   /// getDeclVisibilityMode - Compute the visibility of the decl \arg D.
@@ -249,6 +257,11 @@ public:
   /// GlobalValue.
   void setGlobalVisibility(llvm::GlobalValue *GV, const Decl *D) const;
 
+  /// setTypeVisibility - Set the visibility for the given global
+  /// value which holds information about a type.
+  void setTypeVisibility(llvm::GlobalValue *GV, const CXXRecordDecl *D,
+                         bool IsForRTTI) const;
+
   llvm::Constant *GetAddrOfGlobal(GlobalDecl GD) {
     if (isa<CXXConstructorDecl>(GD.getDecl()))
       return GetAddrOfCXXConstructor(cast<CXXConstructorDecl>(GD.getDecl()),
@@ -289,7 +302,8 @@ public:
   /// a class. Returns null if the offset is 0. 
   llvm::Constant *
   GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
-                               const CXXBaseSpecifierArray &BasePath);
+                               CastExpr::path_const_iterator PathBegin,
+                               CastExpr::path_const_iterator PathEnd);
   
   /// GetStringForStringLiteral - Return the appropriate bytes for a string
   /// literal, properly padded to match the literal type. If only the address of
@@ -344,10 +358,6 @@ public:
   llvm::GlobalValue *GetAddrOfCXXDestructor(const CXXDestructorDecl *D,
                                             CXXDtorType Type);
 
-  // GetCXXMemberFunctionPointerValue - Given a method declaration, return the
-  // integer used in a member function pointer to refer to that value.
-  llvm::Constant *GetCXXMemberFunctionPointerValue(const CXXMethodDecl *MD);
-
   /// getBuiltinLibFunction - Given a builtin id for a function like
   /// "__builtin_fabsf", return a Function* for "fabsf".
   llvm::Value *getBuiltinLibFunction(const FunctionDecl *FD,
@@ -392,6 +402,16 @@ public:
   llvm::Constant *CreateRuntimeVariable(const llvm::Type *Ty,
                                         llvm::StringRef Name);
 
+  ///@name Custom Blocks Runtime Interfaces
+  ///@{
+
+  llvm::Constant *getNSConcreteGlobalBlock();
+  llvm::Constant *getNSConcreteStackBlock();
+  llvm::Constant *getBlockObjectAssign();
+  llvm::Constant *getBlockObjectDispose();
+
+  ///@}
+
   void UpdateCompletedType(const TagDecl *TD) {
     // Make sure that this type is translated.
     Types.UpdateCompletedType(TD);
@@ -411,8 +431,6 @@ public:
   llvm::Constant *EmitAnnotateAttr(llvm::GlobalValue *GV,
                                    const AnnotateAttr *AA, unsigned LineNo);
 
-  llvm::Constant *EmitPointerToDataMember(const FieldDecl *FD);
-
   /// ErrorUnsupported - Print out an error that codegen doesn't support the
   /// specified stmt yet.
   /// \param OmitOnError - If true, then this error should only be emitted if no
@@ -473,15 +491,6 @@ public:
 
   void EmitVTable(CXXRecordDecl *Class, bool DefinitionRequired);
 
-  enum GVALinkage {
-    GVA_Internal,
-    GVA_C99Inline,
-    GVA_CXXInline,
-    GVA_StrongExternal,
-    GVA_TemplateInstantiation,
-    GVA_ExplicitTemplateInstantiation
-  };
-
   llvm::GlobalVariable::LinkageTypes
   getFunctionLinkage(const FunctionDecl *FD);
 
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index d469b906fca1..5ab65c5779b6 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -13,6 +13,7 @@
 
 #include "CodeGenTypes.h"
 #include "CGCall.h"
+#include "CGCXXABI.h"
 #include "CGRecordLayout.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
@@ -26,9 +27,10 @@ using namespace clang;
 using namespace CodeGen;
 
 CodeGenTypes::CodeGenTypes(ASTContext &Ctx, llvm::Module& M,
-                           const llvm::TargetData &TD, const ABIInfo &Info)
+                           const llvm::TargetData &TD, const ABIInfo &Info,
+                           CGCXXABI &CXXABI)
   : Context(Ctx), Target(Ctx.Target), TheModule(M), TheTargetData(TD),
-    TheABIInfo(Info) {
+    TheABIInfo(Info), TheCXXABI(CXXABI) {
 }
 
 CodeGenTypes::~CodeGenTypes() {
@@ -400,17 +402,7 @@ const llvm::Type *CodeGenTypes::ConvertNewType(QualType T) {
   }
 
   case Type::MemberPointer: {
-    // FIXME: This is ABI dependent. We use the Itanium C++ ABI.
-    // http://www.codesourcery.com/public/cxx-abi/abi.html#member-pointers
-    // If we ever want to support other ABIs this needs to be abstracted.
-
-    QualType ETy = cast<MemberPointerType>(Ty).getPointeeType();
-    const llvm::Type *PtrDiffTy =
-        ConvertTypeRecursive(Context.getPointerDiffType());
-    if (ETy->isFunctionType())
-      return llvm::StructType::get(TheModule.getContext(), PtrDiffTy, PtrDiffTy,
-                                   NULL);
-    return PtrDiffTy;
+    return getCXXABI().ConvertMemberPointerType(cast<MemberPointerType>(&Ty));
   }
   }
 
@@ -491,31 +483,34 @@ CodeGenTypes::getCGRecordLayout(const RecordDecl *TD) const {
   return *Layout;
 }
 
-bool CodeGenTypes::ContainsPointerToDataMember(QualType T) {
+bool CodeGenTypes::isZeroInitializable(QualType T) {
   // No need to check for member pointers when not compiling C++.
   if (!Context.getLangOptions().CPlusPlus)
-    return false;
+    return true;
   
   T = Context.getBaseElementType(T);
   
+  // Records are non-zero-initializable if they contain any
+  // non-zero-initializable subobjects.
   if (const RecordType *RT = T->getAs<RecordType>()) {
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-    
-    return ContainsPointerToDataMember(RD);
+    return isZeroInitializable(RD);
   }
-  
+
+  // We have to ask the ABI about member pointers.
   if (const MemberPointerType *MPT = T->getAs<MemberPointerType>())
-    return !MPT->getPointeeType()->isFunctionType();
+    return getCXXABI().isZeroInitializable(MPT);
   
-  return false;
+  // Everything else is okay.
+  return true;
 }
 
-bool CodeGenTypes::ContainsPointerToDataMember(const CXXRecordDecl *RD) {
+bool CodeGenTypes::isZeroInitializable(const CXXRecordDecl *RD) {
   
   // FIXME: It would be better if there was a way to explicitly compute the
   // record layout instead of converting to a type.
   ConvertTagDeclType(RD);
   
   const CGRecordLayout &Layout = getCGRecordLayout(RD);
-  return Layout.containsPointerToDataMember();
+  return Layout.isZeroInitializable();
 }
diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h
index c7f48e6c9dd4..1fc2153fcadb 100644
--- a/lib/CodeGen/CodeGenTypes.h
+++ b/lib/CodeGen/CodeGenTypes.h
@@ -14,13 +14,12 @@
 #ifndef CLANG_CODEGEN_CODEGENTYPES_H
 #define CLANG_CODEGEN_CODEGENTYPES_H
 
+#include "CGCall.h"
+#include "GlobalDecl.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/DenseMap.h"
 #include <vector>
 
-#include "CGCall.h"
-#include "GlobalDecl.h"
-
 namespace llvm {
   class FunctionType;
   class Module;
@@ -51,6 +50,7 @@ namespace clang {
   typedef CanQual<Type> CanQualType;
 
 namespace CodeGen {
+  class CGCXXABI;
   class CGRecordLayout;
 
 /// CodeGenTypes - This class organizes the cross-module state that is used
@@ -61,6 +61,7 @@ class CodeGenTypes {
   llvm::Module& TheModule;
   const llvm::TargetData& TheTargetData;
   const ABIInfo& TheABIInfo;
+  CGCXXABI &TheCXXABI;
 
   llvm::SmallVector<std::pair<QualType,
                               llvm::OpaqueType *>, 8>  PointersToResolve;
@@ -102,13 +103,14 @@ private:
 
 public:
   CodeGenTypes(ASTContext &Ctx, llvm::Module &M, const llvm::TargetData &TD,
-               const ABIInfo &Info);
+               const ABIInfo &Info, CGCXXABI &CXXABI);
   ~CodeGenTypes();
 
   const llvm::TargetData &getTargetData() const { return TheTargetData; }
   const TargetInfo &getTarget() const { return Target; }
   ASTContext &getContext() const { return Context; }
   const ABIInfo &getABIInfo() const { return TheABIInfo; }
+  CGCXXABI &getCXXABI() const { return TheCXXABI; }
   llvm::LLVMContext &getLLVMContext() { return TheModule.getContext(); }
 
   /// ConvertType - Convert type T into a llvm::Type.
@@ -139,7 +141,7 @@ public:
   /// GetFunctionTypeForVTable - Get the LLVM function type for use in a vtable,
   /// given a CXXMethodDecl. If the method to has an incomplete return type, 
   /// and/or incomplete argument types, this will return the opaque type.
-  const llvm::Type *GetFunctionTypeForVTable(const CXXMethodDecl *MD);
+  const llvm::Type *GetFunctionTypeForVTable(GlobalDecl GD);
                                                      
   const CGRecordLayout &getCGRecordLayout(const RecordDecl*) const;
 
@@ -169,7 +171,9 @@ public:
   const CGFunctionInfo &getFunctionInfo(CanQual<FunctionNoProtoType> Ty,
                                         bool IsRecursive = false);
 
-  // getFunctionInfo - Get the function info for a member function.
+  /// getFunctionInfo - Get the function info for a member function of
+  /// the given type.  This is used for calls through member function
+  /// pointers.
   const CGFunctionInfo &getFunctionInfo(const CXXRecordDecl *RD,
                                         const FunctionProtoType *FTP);
   
@@ -205,13 +209,13 @@ public:  // These are internal details of CGT that shouldn't be used externally.
   void GetExpandedTypes(QualType Ty, std::vector<const llvm::Type*> &ArgTys,
                         bool IsRecursive);
   
-  /// ContainsPointerToDataMember - Return whether the given type contains a
-  /// pointer to a data member.
-  bool ContainsPointerToDataMember(QualType T);
+  /// IsZeroInitializable - Return whether a type can be
+  /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer.
+  bool isZeroInitializable(QualType T);
   
-  /// ContainsPointerToDataMember - Return whether the record decl contains a
-  /// pointer to a data member.
-  bool ContainsPointerToDataMember(const CXXRecordDecl *RD);
+  /// IsZeroInitializable - Return whether a record type can be
+  /// zero-initialized (in the C++ sense) with an LLVM zeroinitializer.
+  bool isZeroInitializable(const CXXRecordDecl *RD);
 };
 
 }  // end namespace CodeGen
diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp
index 98db75ea2b46..eefc530ccf18 100644
--- a/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/lib/CodeGen/ItaniumCXXABI.cpp
@@ -12,28 +12,1012 @@
 // documented at:
 //  http://www.codesourcery.com/public/cxx-abi/abi.html
 //  http://www.codesourcery.com/public/cxx-abi/abi-eh.html
+//
+// It also supports the closely-related ARM ABI, documented at:
+// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0041c/IHI0041C_cppabi.pdf
+//
 //===----------------------------------------------------------------------===//
 
 #include "CGCXXABI.h"
+#include "CGRecordLayout.h"
+#include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "Mangle.h"
+#include <clang/AST/Type.h>
+#include <llvm/Target/TargetData.h>
+#include <llvm/Value.h>
 
 using namespace clang;
+using namespace CodeGen;
 
 namespace {
-class ItaniumCXXABI : public CodeGen::CXXABI {
+class ItaniumCXXABI : public CodeGen::CGCXXABI {
+private:
+  const llvm::IntegerType *PtrDiffTy;
+protected:
   CodeGen::MangleContext MangleCtx;
+  bool IsARM;
+
+  // It's a little silly for us to cache this.
+  const llvm::IntegerType *getPtrDiffTy() {
+    if (!PtrDiffTy) {
+      QualType T = getContext().getPointerDiffType();
+      const llvm::Type *Ty = CGM.getTypes().ConvertTypeRecursive(T);
+      PtrDiffTy = cast<llvm::IntegerType>(Ty);
+    }
+    return PtrDiffTy;
+  }
+
+  bool NeedsArrayCookie(QualType ElementType);
+
 public:
-  ItaniumCXXABI(CodeGen::CodeGenModule &CGM) :
-    MangleCtx(CGM.getContext(), CGM.getDiags()) { }
+  ItaniumCXXABI(CodeGen::CodeGenModule &CGM, bool IsARM = false) :
+    CGCXXABI(CGM), PtrDiffTy(0), MangleCtx(getContext(), CGM.getDiags()),
+    IsARM(IsARM) { }
 
   CodeGen::MangleContext &getMangleContext() {
     return MangleCtx;
   }
+
+  bool isZeroInitializable(const MemberPointerType *MPT);
+
+  const llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT);
+
+  llvm::Value *EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
+                                               llvm::Value *&This,
+                                               llvm::Value *MemFnPtr,
+                                               const MemberPointerType *MPT);
+
+  llvm::Value *EmitMemberDataPointerAddress(CodeGenFunction &CGF,
+                                            llvm::Value *Base,
+                                            llvm::Value *MemPtr,
+                                            const MemberPointerType *MPT);
+
+  llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
+                                           const CastExpr *E,
+                                           llvm::Value *Src);
+
+  llvm::Constant *EmitMemberPointerConversion(llvm::Constant *C,
+                                              const CastExpr *E);
+
+  llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT);
+
+  llvm::Constant *EmitMemberPointer(const CXXMethodDecl *MD);
+  llvm::Constant *EmitMemberPointer(const FieldDecl *FD);
+
+  llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
+                                           llvm::Value *L,
+                                           llvm::Value *R,
+                                           const MemberPointerType *MPT,
+                                           bool Inequality);
+
+  llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
+                                          llvm::Value *Addr,
+                                          const MemberPointerType *MPT);
+
+  void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                 CXXCtorType T,
+                                 CanQualType &ResTy,
+                                 llvm::SmallVectorImpl<CanQualType> &ArgTys);
+
+  void BuildDestructorSignature(const CXXDestructorDecl *Dtor,
+                                CXXDtorType T,
+                                CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys);
+
+  void BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                   QualType &ResTy,
+                                   FunctionArgList &Params);
+
+  void EmitInstanceFunctionProlog(CodeGenFunction &CGF);
+
+  CharUnits GetArrayCookieSize(QualType ElementType);
+  llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
+                                     llvm::Value *NewPtr,
+                                     llvm::Value *NumElements,
+                                     QualType ElementType);
+  void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                       QualType ElementType, llvm::Value *&NumElements,
+                       llvm::Value *&AllocPtr, CharUnits &CookieSize);
+};
+
+class ARMCXXABI : public ItaniumCXXABI {
+public:
+  ARMCXXABI(CodeGen::CodeGenModule &CGM) : ItaniumCXXABI(CGM, /*ARM*/ true) {}
+
+  void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                 CXXCtorType T,
+                                 CanQualType &ResTy,
+                                 llvm::SmallVectorImpl<CanQualType> &ArgTys);
+
+  void BuildDestructorSignature(const CXXDestructorDecl *Dtor,
+                                CXXDtorType T,
+                                CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys);
+
+  void BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                   QualType &ResTy,
+                                   FunctionArgList &Params);
+
+  void EmitInstanceFunctionProlog(CodeGenFunction &CGF);
+
+  void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV, QualType ResTy);
+
+  CharUnits GetArrayCookieSize(QualType ElementType);
+  llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
+                                     llvm::Value *NewPtr,
+                                     llvm::Value *NumElements,
+                                     QualType ElementType);
+  void ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *Ptr,
+                       QualType ElementType, llvm::Value *&NumElements,
+                       llvm::Value *&AllocPtr, CharUnits &CookieSize);
+
+private:
+  /// \brief Returns true if the given instance method is one of the
+  /// kinds that the ARM ABI says returns 'this'.
+  static bool HasThisReturn(GlobalDecl GD) {
+    const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
+    return ((isa<CXXDestructorDecl>(MD) && GD.getDtorType() != Dtor_Deleting) ||
+            (isa<CXXConstructorDecl>(MD)));
+  }
 };
 }
 
-CodeGen::CXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
+CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
   return new ItaniumCXXABI(CGM);
 }
 
+CodeGen::CGCXXABI *CodeGen::CreateARMCXXABI(CodeGenModule &CGM) {
+  return new ARMCXXABI(CGM);
+}
+
+const llvm::Type *
+ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
+  if (MPT->isMemberDataPointer())
+    return getPtrDiffTy();
+  else
+    return llvm::StructType::get(CGM.getLLVMContext(),
+                                 getPtrDiffTy(), getPtrDiffTy(), NULL);
+}
+
+/// In the Itanium and ARM ABIs, method pointers have the form:
+///   struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
+///
+/// In the Itanium ABI:
+///  - method pointers are virtual if (memptr.ptr & 1) is nonzero
+///  - the this-adjustment is (memptr.adj)
+///  - the virtual offset is (memptr.ptr - 1)
+///
+/// In the ARM ABI:
+///  - method pointers are virtual if (memptr.adj & 1) is nonzero
+///  - the this-adjustment is (memptr.adj >> 1)
+///  - the virtual offset is (memptr.ptr)
+/// ARM uses 'adj' for the virtual flag because Thumb functions
+/// may be only single-byte aligned.
+///
+/// If the member is virtual, the adjusted 'this' pointer points
+/// to a vtable pointer from which the virtual offset is applied.
+///
+/// If the member is non-virtual, memptr.ptr is the address of
+/// the function to call.
+llvm::Value *
+ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
+                                               llvm::Value *&This,
+                                               llvm::Value *MemFnPtr,
+                                               const MemberPointerType *MPT) {
+  CGBuilderTy &Builder = CGF.Builder;
+
+  const FunctionProtoType *FPT = 
+    MPT->getPointeeType()->getAs<FunctionProtoType>();
+  const CXXRecordDecl *RD = 
+    cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
+
+  const llvm::FunctionType *FTy = 
+    CGM.getTypes().GetFunctionType(CGM.getTypes().getFunctionInfo(RD, FPT),
+                                   FPT->isVariadic());
+
+  const llvm::IntegerType *ptrdiff = getPtrDiffTy();
+  llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(ptrdiff, 1);
+
+  llvm::BasicBlock *FnVirtual = CGF.createBasicBlock("memptr.virtual");
+  llvm::BasicBlock *FnNonVirtual = CGF.createBasicBlock("memptr.nonvirtual");
+  llvm::BasicBlock *FnEnd = CGF.createBasicBlock("memptr.end");
+
+  // Extract memptr.adj, which is in the second field.
+  llvm::Value *RawAdj = Builder.CreateExtractValue(MemFnPtr, 1, "memptr.adj");
+
+  // Compute the true adjustment.
+  llvm::Value *Adj = RawAdj;
+  if (IsARM)
+    Adj = Builder.CreateAShr(Adj, ptrdiff_1, "memptr.adj.shifted");
+
+  // Apply the adjustment and cast back to the original struct type
+  // for consistency.
+  llvm::Value *Ptr = Builder.CreateBitCast(This, Builder.getInt8PtrTy());
+  Ptr = Builder.CreateInBoundsGEP(Ptr, Adj);
+  This = Builder.CreateBitCast(Ptr, This->getType(), "this.adjusted");
+  
+  // Load the function pointer.
+  llvm::Value *FnAsInt = Builder.CreateExtractValue(MemFnPtr, 0, "memptr.ptr");
+  
+  // If the LSB in the function pointer is 1, the function pointer points to
+  // a virtual function.
+  llvm::Value *IsVirtual;
+  if (IsARM)
+    IsVirtual = Builder.CreateAnd(RawAdj, ptrdiff_1);
+  else
+    IsVirtual = Builder.CreateAnd(FnAsInt, ptrdiff_1);
+  IsVirtual = Builder.CreateIsNotNull(IsVirtual, "memptr.isvirtual");
+  Builder.CreateCondBr(IsVirtual, FnVirtual, FnNonVirtual);
+
+  // In the virtual path, the adjustment left 'This' pointing to the
+  // vtable of the correct base subobject.  The "function pointer" is an
+  // offset within the vtable (+1 for the virtual flag on non-ARM).
+  CGF.EmitBlock(FnVirtual);
+
+  // Cast the adjusted this to a pointer to vtable pointer and load.
+  const llvm::Type *VTableTy = Builder.getInt8PtrTy();
+  llvm::Value *VTable = Builder.CreateBitCast(This, VTableTy->getPointerTo());
+  VTable = Builder.CreateLoad(VTable, "memptr.vtable");
+
+  // Apply the offset.
+  llvm::Value *VTableOffset = FnAsInt;
+  if (!IsARM) VTableOffset = Builder.CreateSub(VTableOffset, ptrdiff_1);
+  VTable = Builder.CreateGEP(VTable, VTableOffset);
+
+  // Load the virtual function to call.
+  VTable = Builder.CreateBitCast(VTable, FTy->getPointerTo()->getPointerTo());
+  llvm::Value *VirtualFn = Builder.CreateLoad(VTable, "memptr.virtualfn");
+  CGF.EmitBranch(FnEnd);
+
+  // In the non-virtual path, the function pointer is actually a
+  // function pointer.
+  CGF.EmitBlock(FnNonVirtual);
+  llvm::Value *NonVirtualFn =
+    Builder.CreateIntToPtr(FnAsInt, FTy->getPointerTo(), "memptr.nonvirtualfn");
+  
+  // We're done.
+  CGF.EmitBlock(FnEnd);
+  llvm::PHINode *Callee = Builder.CreatePHI(FTy->getPointerTo());
+  Callee->reserveOperandSpace(2);
+  Callee->addIncoming(VirtualFn, FnVirtual);
+  Callee->addIncoming(NonVirtualFn, FnNonVirtual);
+  return Callee;
+}
+
+/// Compute an l-value by applying the given pointer-to-member to a
+/// base object.
+llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
+                                                         llvm::Value *Base,
+                                                         llvm::Value *MemPtr,
+                                           const MemberPointerType *MPT) {
+  assert(MemPtr->getType() == getPtrDiffTy());
+
+  CGBuilderTy &Builder = CGF.Builder;
+
+  unsigned AS = cast<llvm::PointerType>(Base->getType())->getAddressSpace();
+
+  // Cast to char*.
+  Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS));
+
+  // Apply the offset, which we assume is non-null.
+  llvm::Value *Addr = Builder.CreateInBoundsGEP(Base, MemPtr, "memptr.offset");
+
+  // Cast the address to the appropriate pointer type, adopting the
+  // address space of the base pointer.
+  const llvm::Type *PType
+    = CGF.ConvertTypeForMem(MPT->getPointeeType())->getPointerTo(AS);
+  return Builder.CreateBitCast(Addr, PType);
+}
+
+/// Perform a derived-to-base or base-to-derived member pointer conversion.
+///
+/// Obligatory offset/adjustment diagram:
+///         <-- offset -->          <-- adjustment -->
+///   |--------------------------|----------------------|--------------------|
+///   ^Derived address point     ^Base address point    ^Member address point
+///
+/// So when converting a base member pointer to a derived member pointer,
+/// we add the offset to the adjustment because the address point has
+/// decreased;  and conversely, when converting a derived MP to a base MP
+/// we subtract the offset from the adjustment because the address point
+/// has increased.
+///
+/// The standard forbids (at compile time) conversion to and from
+/// virtual bases, which is why we don't have to consider them here.
+///
+/// The standard forbids (at run time) casting a derived MP to a base
+/// MP when the derived MP does not point to a member of the base.
+/// This is why -1 is a reasonable choice for null data member
+/// pointers.
+llvm::Value *
+ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
+                                           const CastExpr *E,
+                                           llvm::Value *Src) {
+  assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
+         E->getCastKind() == CK_BaseToDerivedMemberPointer);
+
+  if (isa<llvm::Constant>(Src))
+    return EmitMemberPointerConversion(cast<llvm::Constant>(Src), E);
+
+  CGBuilderTy &Builder = CGF.Builder;
+
+  const MemberPointerType *SrcTy =
+    E->getSubExpr()->getType()->getAs<MemberPointerType>();
+  const MemberPointerType *DestTy = E->getType()->getAs<MemberPointerType>();
+
+  const CXXRecordDecl *SrcDecl = SrcTy->getClass()->getAsCXXRecordDecl();
+  const CXXRecordDecl *DestDecl = DestTy->getClass()->getAsCXXRecordDecl();
+
+  bool DerivedToBase =
+    E->getCastKind() == CK_DerivedToBaseMemberPointer;
+
+  const CXXRecordDecl *BaseDecl, *DerivedDecl;
+  if (DerivedToBase)
+    DerivedDecl = SrcDecl, BaseDecl = DestDecl;
+  else
+    BaseDecl = SrcDecl, DerivedDecl = DestDecl;
+
+  llvm::Constant *Adj = 
+    CGF.CGM.GetNonVirtualBaseClassOffset(DerivedDecl,
+                                         E->path_begin(),
+                                         E->path_end());
+  if (!Adj) return Src;
+
+  // For member data pointers, this is just a matter of adding the
+  // offset if the source is non-null.
+  if (SrcTy->isMemberDataPointer()) {
+    llvm::Value *Dst;
+    if (DerivedToBase)
+      Dst = Builder.CreateNSWSub(Src, Adj, "adj");
+    else
+      Dst = Builder.CreateNSWAdd(Src, Adj, "adj");
+
+    // Null check.
+    llvm::Value *Null = llvm::Constant::getAllOnesValue(Src->getType());
+    llvm::Value *IsNull = Builder.CreateICmpEQ(Src, Null, "memptr.isnull");
+    return Builder.CreateSelect(IsNull, Src, Dst);
+  }
+
+  // The this-adjustment is left-shifted by 1 on ARM.
+  if (IsARM) {
+    uint64_t Offset = cast<llvm::ConstantInt>(Adj)->getZExtValue();
+    Offset <<= 1;
+    Adj = llvm::ConstantInt::get(Adj->getType(), Offset);
+  }
+
+  llvm::Value *SrcAdj = Builder.CreateExtractValue(Src, 1, "src.adj");
+  llvm::Value *DstAdj;
+  if (DerivedToBase)
+    DstAdj = Builder.CreateNSWSub(SrcAdj, Adj, "adj");
+  else
+    DstAdj = Builder.CreateNSWAdd(SrcAdj, Adj, "adj");
+
+  return Builder.CreateInsertValue(Src, DstAdj, 1);
+}
+
+llvm::Constant *
+ItaniumCXXABI::EmitMemberPointerConversion(llvm::Constant *C,
+                                           const CastExpr *E) {
+  const MemberPointerType *SrcTy = 
+    E->getSubExpr()->getType()->getAs<MemberPointerType>();
+  const MemberPointerType *DestTy = 
+    E->getType()->getAs<MemberPointerType>();
+
+  bool DerivedToBase =
+    E->getCastKind() == CK_DerivedToBaseMemberPointer;
+
+  const CXXRecordDecl *DerivedDecl;
+  if (DerivedToBase)
+    DerivedDecl = SrcTy->getClass()->getAsCXXRecordDecl();
+  else
+    DerivedDecl = DestTy->getClass()->getAsCXXRecordDecl();
+
+  // Calculate the offset to the base class.
+  llvm::Constant *Offset = 
+    CGM.GetNonVirtualBaseClassOffset(DerivedDecl,
+                                     E->path_begin(),
+                                     E->path_end());
+  // If there's no offset, we're done.
+  if (!Offset) return C;
+
+  // If the source is a member data pointer, we have to do a null
+  // check and then add the offset.  In the common case, we can fold
+  // away the offset.
+  if (SrcTy->isMemberDataPointer()) {
+    assert(C->getType() == getPtrDiffTy());
+
+    // If it's a constant int, just create a new constant int.
+    if (llvm::ConstantInt *CI = dyn_cast<llvm::ConstantInt>(C)) {
+      int64_t Src = CI->getSExtValue();
+
+      // Null converts to null.
+      if (Src == -1) return CI;
+
+      // Otherwise, just add the offset.
+      int64_t OffsetV = cast<llvm::ConstantInt>(Offset)->getSExtValue();
+      int64_t Dst = (DerivedToBase ? Src - OffsetV : Src + OffsetV);
+      return llvm::ConstantInt::get(CI->getType(), Dst, /*signed*/ true);
+    }
+
+    // Otherwise, we have to form a constant select expression.
+    llvm::Constant *Null = llvm::Constant::getAllOnesValue(C->getType());
+
+    llvm::Constant *IsNull =
+      llvm::ConstantExpr::getICmp(llvm::ICmpInst::ICMP_EQ, C, Null);
+
+    llvm::Constant *Dst;
+    if (DerivedToBase)
+      Dst = llvm::ConstantExpr::getNSWSub(C, Offset);
+    else
+      Dst = llvm::ConstantExpr::getNSWAdd(C, Offset);
+
+    return llvm::ConstantExpr::getSelect(IsNull, Null, Dst);
+  }
+
+  // The this-adjustment is left-shifted by 1 on ARM.
+  if (IsARM) {
+    int64_t OffsetV = cast<llvm::ConstantInt>(Offset)->getSExtValue();
+    OffsetV <<= 1;
+    Offset = llvm::ConstantInt::get(Offset->getType(), OffsetV);
+  }
+
+  llvm::ConstantStruct *CS = cast<llvm::ConstantStruct>(C);
+
+  llvm::Constant *Values[2] = { CS->getOperand(0), 0 };
+  if (DerivedToBase)
+    Values[1] = llvm::ConstantExpr::getSub(CS->getOperand(1), Offset);
+  else
+    Values[1] = llvm::ConstantExpr::getAdd(CS->getOperand(1), Offset);
+
+  return llvm::ConstantStruct::get(CGM.getLLVMContext(), Values, 2,
+                                   /*Packed=*/false);
+}        
+
+
+llvm::Constant *
+ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
+  const llvm::Type *ptrdiff_t = getPtrDiffTy();
+
+  // Itanium C++ ABI 2.3:
+  //   A NULL pointer is represented as -1.
+  if (MPT->isMemberDataPointer()) 
+    return llvm::ConstantInt::get(ptrdiff_t, -1ULL, /*isSigned=*/true);
+
+  llvm::Constant *Zero = llvm::ConstantInt::get(ptrdiff_t, 0);
+  llvm::Constant *Values[2] = { Zero, Zero };
+  return llvm::ConstantStruct::get(CGM.getLLVMContext(), Values, 2,
+                                   /*Packed=*/false);
+}
+
+llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const FieldDecl *FD) {
+  // Itanium C++ ABI 2.3:
+  //   A pointer to data member is an offset from the base address of
+  //   the class object containing it, represented as a ptrdiff_t
+
+  QualType ClassType = getContext().getTypeDeclType(FD->getParent());
+  const llvm::StructType *ClassLTy =
+    cast<llvm::StructType>(CGM.getTypes().ConvertType(ClassType));
+
+  const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(FD->getParent());
+  unsigned FieldNo = RL.getLLVMFieldNo(FD);
+  uint64_t Offset = 
+    CGM.getTargetData().getStructLayout(ClassLTy)->getElementOffset(FieldNo);
+
+  return llvm::ConstantInt::get(getPtrDiffTy(), Offset);
+}
+
+llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const CXXMethodDecl *MD) {
+  assert(MD->isInstance() && "Member function must not be static!");
+  MD = MD->getCanonicalDecl();
+
+  CodeGenTypes &Types = CGM.getTypes();
+  const llvm::Type *ptrdiff_t = getPtrDiffTy();
+
+  // Get the function pointer (or index if this is a virtual function).
+  llvm::Constant *MemPtr[2];
+  if (MD->isVirtual()) {
+    uint64_t Index = CGM.getVTables().getMethodVTableIndex(MD);
+
+    // FIXME: We shouldn't use / 8 here.
+    uint64_t PointerWidthInBytes =
+      getContext().Target.getPointerWidth(0) / 8;
+    uint64_t VTableOffset = (Index * PointerWidthInBytes);
+
+    if (IsARM) {
+      // ARM C++ ABI 3.2.1:
+      //   This ABI specifies that adj contains twice the this
+      //   adjustment, plus 1 if the member function is virtual. The
+      //   least significant bit of adj then makes exactly the same
+      //   discrimination as the least significant bit of ptr does for
+      //   Itanium.
+      MemPtr[0] = llvm::ConstantInt::get(ptrdiff_t, VTableOffset);
+      MemPtr[1] = llvm::ConstantInt::get(ptrdiff_t, 1);
+    } else {
+      // Itanium C++ ABI 2.3:
+      //   For a virtual function, [the pointer field] is 1 plus the
+      //   virtual table offset (in bytes) of the function,
+      //   represented as a ptrdiff_t.
+      MemPtr[0] = llvm::ConstantInt::get(ptrdiff_t, VTableOffset + 1);
+      MemPtr[1] = llvm::ConstantInt::get(ptrdiff_t, 0);
+    }
+  } else {
+    const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+    const llvm::Type *Ty;
+    // Check whether the function has a computable LLVM signature.
+    if (!CodeGenTypes::VerifyFuncTypeComplete(FPT)) {
+      // The function has a computable LLVM signature; use the correct type.
+      Ty = Types.GetFunctionType(Types.getFunctionInfo(MD), FPT->isVariadic());
+    } else {
+      // Use an arbitrary non-function type to tell GetAddrOfFunction that the
+      // function type is incomplete.
+      Ty = ptrdiff_t;
+    }
+
+    llvm::Constant *Addr = CGM.GetAddrOfFunction(MD, Ty);
+    MemPtr[0] = llvm::ConstantExpr::getPtrToInt(Addr, ptrdiff_t);
+    MemPtr[1] = llvm::ConstantInt::get(ptrdiff_t, 0);
+  }
+  
+  return llvm::ConstantStruct::get(CGM.getLLVMContext(),
+                                   MemPtr, 2, /*Packed=*/false);
+}
+
+/// The comparison algorithm is pretty easy: the member pointers are
+/// the same if they're either bitwise identical *or* both null.
+///
+/// ARM is different here only because null-ness is more complicated.
+llvm::Value *
+ItaniumCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
+                                           llvm::Value *L,
+                                           llvm::Value *R,
+                                           const MemberPointerType *MPT,
+                                           bool Inequality) {
+  CGBuilderTy &Builder = CGF.Builder;
+
+  llvm::ICmpInst::Predicate Eq;
+  llvm::Instruction::BinaryOps And, Or;
+  if (Inequality) {
+    Eq = llvm::ICmpInst::ICMP_NE;
+    And = llvm::Instruction::Or;
+    Or = llvm::Instruction::And;
+  } else {
+    Eq = llvm::ICmpInst::ICMP_EQ;
+    And = llvm::Instruction::And;
+    Or = llvm::Instruction::Or;
+  }
+
+  // Member data pointers are easy because there's a unique null
+  // value, so it just comes down to bitwise equality.
+  if (MPT->isMemberDataPointer())
+    return Builder.CreateICmp(Eq, L, R);
+
+  // For member function pointers, the tautologies are more complex.
+  // The Itanium tautology is:
+  //   (L == R) <==> (L.ptr == R.ptr && (L.ptr == 0 || L.adj == R.adj))
+  // The ARM tautology is:
+  //   (L == R) <==> (L.ptr == R.ptr &&
+  //                  (L.adj == R.adj ||
+  //                   (L.ptr == 0 && ((L.adj|R.adj) & 1) == 0)))
+  // The inequality tautologies have exactly the same structure, except
+  // applying De Morgan's laws.
+  
+  llvm::Value *LPtr = Builder.CreateExtractValue(L, 0, "lhs.memptr.ptr");
+  llvm::Value *RPtr = Builder.CreateExtractValue(R, 0, "rhs.memptr.ptr");
+
+  // This condition tests whether L.ptr == R.ptr.  This must always be
+  // true for equality to hold.
+  llvm::Value *PtrEq = Builder.CreateICmp(Eq, LPtr, RPtr, "cmp.ptr");
+
+  // This condition, together with the assumption that L.ptr == R.ptr,
+  // tests whether the pointers are both null.  ARM imposes an extra
+  // condition.
+  llvm::Value *Zero = llvm::Constant::getNullValue(LPtr->getType());
+  llvm::Value *EqZero = Builder.CreateICmp(Eq, LPtr, Zero, "cmp.ptr.null");
+
+  // This condition tests whether L.adj == R.adj.  If this isn't
+  // true, the pointers are unequal unless they're both null.
+  llvm::Value *LAdj = Builder.CreateExtractValue(L, 1, "lhs.memptr.adj");
+  llvm::Value *RAdj = Builder.CreateExtractValue(R, 1, "rhs.memptr.adj");
+  llvm::Value *AdjEq = Builder.CreateICmp(Eq, LAdj, RAdj, "cmp.adj");
+
+  // Null member function pointers on ARM clear the low bit of Adj,
+  // so the zero condition has to check that neither low bit is set.
+  if (IsARM) {
+    llvm::Value *One = llvm::ConstantInt::get(LPtr->getType(), 1);
+
+    // Compute (l.adj | r.adj) & 1 and test it against zero.
+    llvm::Value *OrAdj = Builder.CreateOr(LAdj, RAdj, "or.adj");
+    llvm::Value *OrAdjAnd1 = Builder.CreateAnd(OrAdj, One);
+    llvm::Value *OrAdjAnd1EqZero = Builder.CreateICmp(Eq, OrAdjAnd1, Zero,
+                                                      "cmp.or.adj");
+    EqZero = Builder.CreateBinOp(And, EqZero, OrAdjAnd1EqZero);
+  }
+
+  // Tie together all our conditions.
+  llvm::Value *Result = Builder.CreateBinOp(Or, EqZero, AdjEq);
+  Result = Builder.CreateBinOp(And, PtrEq, Result,
+                               Inequality ? "memptr.ne" : "memptr.eq");
+  return Result;
+}
+
+llvm::Value *
+ItaniumCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
+                                          llvm::Value *MemPtr,
+                                          const MemberPointerType *MPT) {
+  CGBuilderTy &Builder = CGF.Builder;
+
+  /// For member data pointers, this is just a check against -1.
+  if (MPT->isMemberDataPointer()) {
+    assert(MemPtr->getType() == getPtrDiffTy());
+    llvm::Value *NegativeOne =
+      llvm::Constant::getAllOnesValue(MemPtr->getType());
+    return Builder.CreateICmpNE(MemPtr, NegativeOne, "memptr.tobool");
+  }
+  
+  // In Itanium, a member function pointer is null if 'ptr' is null.
+  llvm::Value *Ptr = Builder.CreateExtractValue(MemPtr, 0, "memptr.ptr");
+
+  llvm::Constant *Zero = llvm::ConstantInt::get(Ptr->getType(), 0);
+  llvm::Value *Result = Builder.CreateICmpNE(Ptr, Zero, "memptr.tobool");
+
+  // In ARM, it's that, plus the low bit of 'adj' must be zero.
+  if (IsARM) {
+    llvm::Constant *One = llvm::ConstantInt::get(Ptr->getType(), 1);
+    llvm::Value *Adj = Builder.CreateExtractValue(MemPtr, 1, "memptr.adj");
+    llvm::Value *VirtualBit = Builder.CreateAnd(Adj, One, "memptr.virtualbit");
+    llvm::Value *IsNotVirtual = Builder.CreateICmpEQ(VirtualBit, Zero,
+                                                     "memptr.notvirtual");
+    Result = Builder.CreateAnd(Result, IsNotVirtual);
+  }
+
+  return Result;
+}
+
+/// The Itanium ABI requires non-zero initialization only for data
+/// member pointers, for which '0' is a valid offset.
+bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
+  return MPT->getPointeeType()->isFunctionType();
+}
+
+/// The generic ABI passes 'this', plus a VTT if it's initializing a
+/// base subobject.
+void ItaniumCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                              CXXCtorType Type,
+                                              CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+  ASTContext &Context = getContext();
+
+  // 'this' is already there.
+
+  // Check if we need to add a VTT parameter (which has type void **).
+  if (Type == Ctor_Base && Ctor->getParent()->getNumVBases() != 0)
+    ArgTys.push_back(Context.getPointerType(Context.VoidPtrTy));
+}
+
+/// The ARM ABI does the same as the Itanium ABI, but returns 'this'.
+void ARMCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                          CXXCtorType Type,
+                                          CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+  ItaniumCXXABI::BuildConstructorSignature(Ctor, Type, ResTy, ArgTys);
+  ResTy = ArgTys[0];
+}
+
+/// The generic ABI passes 'this', plus a VTT if it's destroying a
+/// base subobject.
+void ItaniumCXXABI::BuildDestructorSignature(const CXXDestructorDecl *Dtor,
+                                             CXXDtorType Type,
+                                             CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+  ASTContext &Context = getContext();
+
+  // 'this' is already there.
+
+  // Check if we need to add a VTT parameter (which has type void **).
+  if (Type == Dtor_Base && Dtor->getParent()->getNumVBases() != 0)
+    ArgTys.push_back(Context.getPointerType(Context.VoidPtrTy));
+}
+
+/// The ARM ABI does the same as the Itanium ABI, but returns 'this'
+/// for non-deleting destructors.
+void ARMCXXABI::BuildDestructorSignature(const CXXDestructorDecl *Dtor,
+                                         CXXDtorType Type,
+                                         CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+  ItaniumCXXABI::BuildDestructorSignature(Dtor, Type, ResTy, ArgTys);
+
+  if (Type != Dtor_Deleting)
+    ResTy = ArgTys[0];
+}
+
+void ItaniumCXXABI::BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                                QualType &ResTy,
+                                                FunctionArgList &Params) {
+  /// Create the 'this' variable.
+  BuildThisParam(CGF, Params);
+
+  const CXXMethodDecl *MD = cast<CXXMethodDecl>(CGF.CurGD.getDecl());
+  assert(MD->isInstance());
+
+  // Check if we need a VTT parameter as well.
+  if (CodeGenVTables::needsVTTParameter(CGF.CurGD)) {
+    ASTContext &Context = getContext();
+
+    // FIXME: avoid the fake decl
+    QualType T = Context.getPointerType(Context.VoidPtrTy);
+    ImplicitParamDecl *VTTDecl
+      = ImplicitParamDecl::Create(Context, 0, MD->getLocation(),
+                                  &Context.Idents.get("vtt"), T);
+    Params.push_back(std::make_pair(VTTDecl, VTTDecl->getType()));
+    getVTTDecl(CGF) = VTTDecl;
+  }
+}
+
+void ARMCXXABI::BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                            QualType &ResTy,
+                                            FunctionArgList &Params) {
+  ItaniumCXXABI::BuildInstanceFunctionParams(CGF, ResTy, Params);
+
+  // Return 'this' from certain constructors and destructors.
+  if (HasThisReturn(CGF.CurGD))
+    ResTy = Params[0].second;
+}
+
+void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
+  /// Initialize the 'this' slot.
+  EmitThisParam(CGF);
+
+  /// Initialize the 'vtt' slot if needed.
+  if (getVTTDecl(CGF)) {
+    getVTTValue(CGF)
+      = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(getVTTDecl(CGF)),
+                               "vtt");
+  }
+}
+
+void ARMCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
+  ItaniumCXXABI::EmitInstanceFunctionProlog(CGF);
+
+  /// Initialize the return slot to 'this' at the start of the
+  /// function.
+  if (HasThisReturn(CGF.CurGD))
+    CGF.Builder.CreateStore(CGF.LoadCXXThis(), CGF.ReturnValue);
+}
+
+void ARMCXXABI::EmitReturnFromThunk(CodeGenFunction &CGF,
+                                    RValue RV, QualType ResultType) {
+  if (!isa<CXXDestructorDecl>(CGF.CurGD.getDecl()))
+    return ItaniumCXXABI::EmitReturnFromThunk(CGF, RV, ResultType);
+
+  // Destructor thunks in the ARM ABI have indeterminate results.
+  const llvm::Type *T =
+    cast<llvm::PointerType>(CGF.ReturnValue->getType())->getElementType();
+  RValue Undef = RValue::get(llvm::UndefValue::get(T));
+  return ItaniumCXXABI::EmitReturnFromThunk(CGF, Undef, ResultType);
+}
+
+/************************** Array allocation cookies **************************/
+
+bool ItaniumCXXABI::NeedsArrayCookie(QualType ElementType) {
+  ElementType = getContext().getBaseElementType(ElementType);
+  const RecordType *RT = ElementType->getAs<RecordType>();
+  if (!RT) return false;
+  
+  const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+
+  // If the class has a non-trivial destructor, it always needs a cookie.
+  if (!RD->hasTrivialDestructor()) return true;
+
+  // If the class's usual deallocation function takes two arguments,
+  // it needs a cookie.  Otherwise we don't need a cookie.
+  const CXXMethodDecl *UsualDeallocationFunction = 0;
+
+  // Usual deallocation functions of this form are always found on the
+  // class.
+  //
+  // FIXME: what exactly is this code supposed to do if there's an
+  // ambiguity?  That's possible with using declarations.
+  DeclarationName OpName =
+    getContext().DeclarationNames.getCXXOperatorName(OO_Array_Delete);
+  DeclContext::lookup_const_iterator Op, OpEnd;
+  for (llvm::tie(Op, OpEnd) = RD->lookup(OpName); Op != OpEnd; ++Op) {
+    const CXXMethodDecl *Delete =
+      cast<CXXMethodDecl>((*Op)->getUnderlyingDecl());
+
+    if (Delete->isUsualDeallocationFunction()) {
+      UsualDeallocationFunction = Delete;
+      break;
+    }
+  }
+    
+  // No usual deallocation function, we don't need a cookie.
+  if (!UsualDeallocationFunction)
+    return false;
+    
+  // The usual deallocation function doesn't take a size_t argument,
+  // so we don't need a cookie.
+  if (UsualDeallocationFunction->getNumParams() == 1)
+    return false;
+        
+  assert(UsualDeallocationFunction->getNumParams() == 2 && 
+         "Unexpected deallocation function type!");
+  return true;
+}
+
+CharUnits ItaniumCXXABI::GetArrayCookieSize(QualType ElementType) {
+  if (!NeedsArrayCookie(ElementType))
+    return CharUnits::Zero();
+  
+  // Padding is the maximum of sizeof(size_t) and alignof(ElementType)
+  ASTContext &Ctx = getContext();
+  return std::max(Ctx.getTypeSizeInChars(Ctx.getSizeType()),
+                  Ctx.getTypeAlignInChars(ElementType));
+}
+
+llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+                                                  llvm::Value *NewPtr,
+                                                  llvm::Value *NumElements,
+                                                  QualType ElementType) {
+  assert(NeedsArrayCookie(ElementType));
+
+  unsigned AS = cast<llvm::PointerType>(NewPtr->getType())->getAddressSpace();
+
+  ASTContext &Ctx = getContext();
+  QualType SizeTy = Ctx.getSizeType();
+  CharUnits SizeSize = Ctx.getTypeSizeInChars(SizeTy);
+
+  // The size of the cookie.
+  CharUnits CookieSize =
+    std::max(SizeSize, Ctx.getTypeAlignInChars(ElementType));
+
+  // Compute an offset to the cookie.
+  llvm::Value *CookiePtr = NewPtr;
+  CharUnits CookieOffset = CookieSize - SizeSize;
+  if (!CookieOffset.isZero())
+    CookiePtr = CGF.Builder.CreateConstInBoundsGEP1_64(CookiePtr,
+                                                 CookieOffset.getQuantity());
+
+  // Write the number of elements into the appropriate slot.
+  llvm::Value *NumElementsPtr
+    = CGF.Builder.CreateBitCast(CookiePtr,
+                                CGF.ConvertType(SizeTy)->getPointerTo(AS));
+  CGF.Builder.CreateStore(NumElements, NumElementsPtr);
+
+  // Finally, compute a pointer to the actual data buffer by skipping
+  // over the cookie completely.
+  return CGF.Builder.CreateConstInBoundsGEP1_64(NewPtr,
+                                                CookieSize.getQuantity());  
+}
+
+void ItaniumCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
+                                    llvm::Value *Ptr,
+                                    QualType ElementType,
+                                    llvm::Value *&NumElements,
+                                    llvm::Value *&AllocPtr,
+                                    CharUnits &CookieSize) {
+  // Derive a char* in the same address space as the pointer.
+  unsigned AS = cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
+  const llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
+
+  // If we don't need an array cookie, bail out early.
+  if (!NeedsArrayCookie(ElementType)) {
+    AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
+    NumElements = 0;
+    CookieSize = CharUnits::Zero();
+    return;
+  }
+
+  QualType SizeTy = getContext().getSizeType();
+  CharUnits SizeSize = getContext().getTypeSizeInChars(SizeTy);
+  const llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
+  
+  CookieSize
+    = std::max(SizeSize, getContext().getTypeAlignInChars(ElementType));
+
+  CharUnits NumElementsOffset = CookieSize - SizeSize;
+
+  // Compute the allocated pointer.
+  AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
+  AllocPtr = CGF.Builder.CreateConstInBoundsGEP1_64(AllocPtr,
+                                                    -CookieSize.getQuantity());
+
+  llvm::Value *NumElementsPtr = AllocPtr;
+  if (!NumElementsOffset.isZero())
+    NumElementsPtr =
+      CGF.Builder.CreateConstInBoundsGEP1_64(NumElementsPtr,
+                                             NumElementsOffset.getQuantity());
+  NumElementsPtr = 
+    CGF.Builder.CreateBitCast(NumElementsPtr, SizeLTy->getPointerTo(AS));
+  NumElements = CGF.Builder.CreateLoad(NumElementsPtr);
+}
+
+CharUnits ARMCXXABI::GetArrayCookieSize(QualType ElementType) {
+  if (!NeedsArrayCookie(ElementType))
+    return CharUnits::Zero();
+
+  // On ARM, the cookie is always:
+  //   struct array_cookie {
+  //     std::size_t element_size; // element_size != 0
+  //     std::size_t element_count;
+  //   };
+  // TODO: what should we do if the allocated type actually wants
+  // greater alignment?
+  return getContext().getTypeSizeInChars(getContext().getSizeType()) * 2;
+}
+
+llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
+                                              llvm::Value *NewPtr,
+                                              llvm::Value *NumElements,
+                                              QualType ElementType) {
+  assert(NeedsArrayCookie(ElementType));
+
+  // NewPtr is a char*.
+
+  unsigned AS = cast<llvm::PointerType>(NewPtr->getType())->getAddressSpace();
+
+  ASTContext &Ctx = getContext();
+  CharUnits SizeSize = Ctx.getTypeSizeInChars(Ctx.getSizeType());
+  const llvm::IntegerType *SizeTy =
+    cast<llvm::IntegerType>(CGF.ConvertType(Ctx.getSizeType()));
+
+  // The cookie is always at the start of the buffer.
+  llvm::Value *CookiePtr = NewPtr;
+
+  // The first element is the element size.
+  CookiePtr = CGF.Builder.CreateBitCast(CookiePtr, SizeTy->getPointerTo(AS));
+  llvm::Value *ElementSize = llvm::ConstantInt::get(SizeTy,
+                          Ctx.getTypeSizeInChars(ElementType).getQuantity());
+  CGF.Builder.CreateStore(ElementSize, CookiePtr);
+
+  // The second element is the element count.
+  CookiePtr = CGF.Builder.CreateConstInBoundsGEP1_32(CookiePtr, 1);
+  CGF.Builder.CreateStore(NumElements, CookiePtr);
+
+  // Finally, compute a pointer to the actual data buffer by skipping
+  // over the cookie completely.
+  CharUnits CookieSize = 2 * SizeSize;
+  return CGF.Builder.CreateConstInBoundsGEP1_64(NewPtr,
+                                                CookieSize.getQuantity());
+}
+
+void ARMCXXABI::ReadArrayCookie(CodeGenFunction &CGF,
+                                llvm::Value *Ptr,
+                                QualType ElementType,
+                                llvm::Value *&NumElements,
+                                llvm::Value *&AllocPtr,
+                                CharUnits &CookieSize) {
+  // Derive a char* in the same address space as the pointer.
+  unsigned AS = cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
+  const llvm::Type *CharPtrTy = CGF.Builder.getInt8Ty()->getPointerTo(AS);
+
+  // If we don't need an array cookie, bail out early.
+  if (!NeedsArrayCookie(ElementType)) {
+    AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
+    NumElements = 0;
+    CookieSize = CharUnits::Zero();
+    return;
+  }
+
+  QualType SizeTy = getContext().getSizeType();
+  CharUnits SizeSize = getContext().getTypeSizeInChars(SizeTy);
+  const llvm::Type *SizeLTy = CGF.ConvertType(SizeTy);
+  
+  // The cookie size is always 2 * sizeof(size_t).
+  CookieSize = 2 * SizeSize;
+  CharUnits NumElementsOffset = CookieSize - SizeSize;
+
+  // The allocated pointer is the input ptr, minus that amount.
+  AllocPtr = CGF.Builder.CreateBitCast(Ptr, CharPtrTy);
+  AllocPtr = CGF.Builder.CreateConstInBoundsGEP1_64(AllocPtr,
+                                               -CookieSize.getQuantity());
+
+  // The number of elements is at offset sizeof(size_t) relative to that.
+  llvm::Value *NumElementsPtr
+    = CGF.Builder.CreateConstInBoundsGEP1_64(AllocPtr,
+                                             SizeSize.getQuantity());
+  NumElementsPtr = 
+    CGF.Builder.CreateBitCast(NumElementsPtr, SizeLTy->getPointerTo(AS));
+  NumElements = CGF.Builder.CreateLoad(NumElementsPtr);
+}
+
diff --git a/lib/CodeGen/Makefile b/lib/CodeGen/Makefile
index 4b93524267ae..6032dffec1a5 100644
--- a/lib/CodeGen/Makefile
+++ b/lib/CodeGen/Makefile
@@ -14,7 +14,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangCodeGen
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/CodeGen/Mangle.cpp b/lib/CodeGen/Mangle.cpp
index 30ee541c00fa..e1988743b7f9 100644
--- a/lib/CodeGen/Mangle.cpp
+++ b/lib/CodeGen/Mangle.cpp
@@ -241,11 +241,11 @@ private:
                         NestedNameSpecifier *Qualifier,
                         DeclarationName Name,
                         unsigned KnownArity);
-  void mangleCalledExpression(const Expr *E, unsigned KnownArity);
-  void mangleExpression(const Expr *E);
+  void mangleExpression(const Expr *E, unsigned Arity = UnknownArity);
   void mangleCXXCtorType(CXXCtorType T);
   void mangleCXXDtorType(CXXDtorType T);
 
+  void mangleTemplateArgs(const ExplicitTemplateArgumentList &TemplateArgs);
   void mangleTemplateArgs(TemplateName Template,
                           const TemplateArgument *TemplateArgs,
                           unsigned NumTemplateArgs);  
@@ -304,6 +304,10 @@ bool MangleContext::shouldMangleDeclName(const NamedDecl *D) {
       return false;
   }
 
+  // Class members are always mangled.
+  if (D->getDeclContext()->isRecord())
+    return true;
+
   // C functions and "main" are not mangled.
   if ((FD && FD->isMain()) || isInCLinkageSpecification(D))
     return false;
@@ -695,7 +699,11 @@ void CXXNameMangler::mangleUnqualifiedName(const NamedDecl *ND,
       //   a program to refer to the anonymous union, and there is therefore no
       //   need to mangle its name.
       const FieldDecl *FD = FindFirstNamedDataMember(RD);
-      assert(FD && "Didn't find a named data member!");
+
+      // It's actually possible for various reasons for us to get here
+      // with an empty anonymous struct / union.  Fortunately, it
+      // doesn't really matter what name we generate.
+      if (!FD) break;
       assert(FD->getIdentifier() && "Data member name isn't an identifier!");
       
       mangleSourceName(FD->getIdentifier());
@@ -1016,24 +1024,21 @@ CXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO, unsigned Arity) {
   //              ::= da        # delete[]
   case OO_Array_Delete: Out << "da"; break;
   //              ::= ps        # + (unary)
-  //              ::= pl        # +
+  //              ::= pl        # + (binary or unknown)
   case OO_Plus:
-    assert((Arity == 1 || Arity == 2) && "Invalid arity!");
     Out << (Arity == 1? "ps" : "pl"); break;
   //              ::= ng        # - (unary)
-  //              ::= mi        # -
+  //              ::= mi        # - (binary or unknown)
   case OO_Minus:
-    assert((Arity == 1 || Arity == 2) && "Invalid arity!");
     Out << (Arity == 1? "ng" : "mi"); break;
   //              ::= ad        # & (unary)
-  //              ::= an        # &
+  //              ::= an        # & (binary or unknown)
   case OO_Amp:
-    assert((Arity == 1 || Arity == 2) && "Invalid arity!");
     Out << (Arity == 1? "ad" : "an"); break;
   //              ::= de        # * (unary)
-  //              ::= ml        # *
+  //              ::= ml        # * (binary or unknown)
   case OO_Star:
-    assert((Arity == 1 || Arity == 2) && "Invalid arity!");
+    // Use binary when unknown.
     Out << (Arity == 1? "de" : "ml"); break;
   //              ::= co        # ~
   case OO_Tilde: Out << "co"; break;
@@ -1275,7 +1280,7 @@ void CXXNameMangler::mangleBareFunctionType(const FunctionType *T,
     mangleType(Proto->getResultType());
 
   if (Proto->getNumArgs() == 0 && !Proto->isVariadic()) {
-    //   <builtin-type> ::= v	# void
+    //   <builtin-type> ::= v   # void
     Out << 'v';
     return;
   }
@@ -1536,25 +1541,6 @@ void CXXNameMangler::mangleIntegerLiteral(QualType T,
 
 }
 
-void CXXNameMangler::mangleCalledExpression(const Expr *E, unsigned Arity) {
-  if (E->getType() != getASTContext().OverloadTy)
-    mangleExpression(E);
-  // propagate arity to dependent overloads?
-
-  llvm::PointerIntPair<OverloadExpr*,1> R
-    = OverloadExpr::find(const_cast<Expr*>(E));
-  if (R.getInt())
-    Out << "an"; // &
-  const OverloadExpr *Ovl = R.getPointer();
-  if (const UnresolvedMemberExpr *ME = dyn_cast<UnresolvedMemberExpr>(Ovl)) {
-    mangleMemberExpr(ME->getBase(), ME->isArrow(), ME->getQualifier(),
-                     ME->getMemberName(), Arity);
-    return;
-  }
-
-  mangleUnresolvedName(Ovl->getQualifier(), Ovl->getName(), Arity);
-}
-
 /// Mangles a member expression.  Implicit accesses are not handled,
 /// but that should be okay, because you shouldn't be able to
 /// make an implicit access in a function template declaration.
@@ -1570,14 +1556,14 @@ void CXXNameMangler::mangleMemberExpr(const Expr *Base,
   mangleUnresolvedName(Qualifier, Member, Arity);
 }
 
-void CXXNameMangler::mangleExpression(const Expr *E) {
+void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity) {
   // <expression> ::= <unary operator-name> <expression>
   //              ::= <binary operator-name> <expression> <expression>
   //              ::= <trinary operator-name> <expression> <expression> <expression>
-  //              ::= cl <expression>* E	     # call
+  //              ::= cl <expression>* E             # call
   //              ::= cv <type> expression           # conversion with one argument
   //              ::= cv <type> _ <expression>* E # conversion with a different number of arguments
-  //              ::= st <type>		             # sizeof (a type)
+  //              ::= st <type>                      # sizeof (a type)
   //              ::= at <type>                      # alignof (a type)
   //              ::= <template-param>
   //              ::= <function-param>
@@ -1644,14 +1630,14 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
   }
 
   case Expr::CXXDefaultArgExprClass:
-    mangleExpression(cast<CXXDefaultArgExpr>(E)->getExpr());
+    mangleExpression(cast<CXXDefaultArgExpr>(E)->getExpr(), Arity);
     break;
 
   case Expr::CXXMemberCallExprClass: // fallthrough
   case Expr::CallExprClass: {
     const CallExpr *CE = cast<CallExpr>(E);
     Out << "cl";
-    mangleCalledExpression(CE->getCallee(), CE->getNumArgs());
+    mangleExpression(CE->getCallee(), CE->getNumArgs());
     for (unsigned I = 0, N = CE->getNumArgs(); I != N; ++I)
       mangleExpression(CE->getArg(I));
     Out << 'E';
@@ -1682,7 +1668,7 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     const MemberExpr *ME = cast<MemberExpr>(E);
     mangleMemberExpr(ME->getBase(), ME->isArrow(),
                      ME->getQualifier(), ME->getMemberDecl()->getDeclName(),
-                     UnknownArity);
+                     Arity);
     break;
   }
 
@@ -1690,7 +1676,9 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     const UnresolvedMemberExpr *ME = cast<UnresolvedMemberExpr>(E);
     mangleMemberExpr(ME->getBase(), ME->isArrow(),
                      ME->getQualifier(), ME->getMemberName(),
-                     UnknownArity);
+                     Arity);
+    if (ME->hasExplicitTemplateArgs())
+      mangleTemplateArgs(ME->getExplicitTemplateArgs());
     break;
   }
 
@@ -1699,7 +1687,9 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
       = cast<CXXDependentScopeMemberExpr>(E);
     mangleMemberExpr(ME->getBase(), ME->isArrow(),
                      ME->getQualifier(), ME->getMember(),
-                     UnknownArity);
+                     Arity);
+    if (ME->hasExplicitTemplateArgs())
+      mangleTemplateArgs(ME->getExplicitTemplateArgs());
     break;
   }
 
@@ -1708,7 +1698,9 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     // using something as close as possible to the original lookup
     // expression.
     const UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(E);
-    mangleUnresolvedName(ULE->getQualifier(), ULE->getName(), UnknownArity);
+    mangleUnresolvedName(ULE->getQualifier(), ULE->getName(), Arity);
+    if (ULE->hasExplicitTemplateArgs())
+      mangleTemplateArgs(ULE->getExplicitTemplateArgs());
     break;
   }
 
@@ -1821,13 +1813,13 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     const ConditionalOperator *CO = cast<ConditionalOperator>(E);
     mangleOperatorName(OO_Conditional, /*Arity=*/3);
     mangleExpression(CO->getCond());
-    mangleExpression(CO->getLHS());
-    mangleExpression(CO->getRHS());
+    mangleExpression(CO->getLHS(), Arity);
+    mangleExpression(CO->getRHS(), Arity);
     break;
   }
 
   case Expr::ImplicitCastExprClass: {
-    mangleExpression(cast<ImplicitCastExpr>(E)->getSubExpr());
+    mangleExpression(cast<ImplicitCastExpr>(E)->getSubExpr(), Arity);
     break;
   }
 
@@ -1855,7 +1847,7 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
   }
 
   case Expr::ParenExprClass:
-    mangleExpression(cast<ParenExpr>(E)->getSubExpr());
+    mangleExpression(cast<ParenExpr>(E)->getSubExpr(), Arity);
     break;
 
   case Expr::DeclRefExprClass: {
@@ -1869,6 +1861,12 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
       Out << 'E';
       break;
 
+    case Decl::EnumConstant: {
+      const EnumConstantDecl *ED = cast<EnumConstantDecl>(D);
+      mangleIntegerLiteral(ED->getType(), ED->getInitVal());
+      break;
+    }
+
     case Decl::NonTypeTemplateParm: {
       const NonTypeTemplateParmDecl *PD = cast<NonTypeTemplateParmDecl>(D);
       mangleTemplateParameter(PD->getIndex());
@@ -1897,27 +1895,23 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
     }
     assert(QTy && "Qualifier was not type!");
 
-    // ::= sr <type> <unqualified-name>                   # dependent name
+    // ::= sr <type> <unqualified-name>                  # dependent name
+    // ::= sr <type> <unqualified-name> <template-args>  # dependent template-id
     Out << "sr";
     mangleType(QualType(QTy, 0));
-
-    assert(DRE->getDeclName().getNameKind() == DeclarationName::Identifier &&
-           "Unhandled decl name kind!");
-    mangleSourceName(DRE->getDeclName().getAsIdentifierInfo());
+    mangleUnqualifiedName(0, DRE->getDeclName(), Arity);
+    if (DRE->hasExplicitTemplateArgs())
+      mangleTemplateArgs(DRE->getExplicitTemplateArgs());
 
     break;
   }
 
-  case Expr::CXXBindReferenceExprClass:
-    mangleExpression(cast<CXXBindReferenceExpr>(E)->getSubExpr());
-    break;
-
   case Expr::CXXBindTemporaryExprClass:
     mangleExpression(cast<CXXBindTemporaryExpr>(E)->getSubExpr());
     break;
 
   case Expr::CXXExprWithTemporariesClass:
-    mangleExpression(cast<CXXExprWithTemporaries>(E)->getSubExpr());
+    mangleExpression(cast<CXXExprWithTemporaries>(E)->getSubExpr(), Arity);
     break;
 
   case Expr::FloatingLiteralClass: {
@@ -1974,13 +1968,10 @@ void CXXNameMangler::mangleExpression(const Expr *E) {
   }
 
   case Expr::StringLiteralClass: {
-    // Proposal from David Vandervoorde, 2010.06.30.
-    // I've sent a comment off asking whether this needs to also
-    // represent the length of the string.
+    // Revised proposal from David Vandervoorde, 2010.07.15.
     Out << 'L';
-    const ConstantArrayType *T = cast<ConstantArrayType>(E->getType());
-    QualType CharTy = T->getElementType().getUnqualifiedType();
-    mangleType(CharTy);
+    assert(isa<ConstantArrayType>(E->getType()));
+    mangleType(E->getType());
     Out << 'E';
     break;
   }
@@ -2035,6 +2026,15 @@ void CXXNameMangler::mangleCXXDtorType(CXXDtorType T) {
   }
 }
 
+void CXXNameMangler::mangleTemplateArgs(
+                          const ExplicitTemplateArgumentList &TemplateArgs) {
+  // <template-args> ::= I <template-arg>+ E
+  Out << 'I';
+  for (unsigned I = 0, E = TemplateArgs.NumTemplateArgs; I != E; ++I)
+    mangleTemplateArg(0, TemplateArgs.getTemplateArgs()[I].getArgument());
+  Out << 'E';
+}
+
 void CXXNameMangler::mangleTemplateArgs(TemplateName Template,
                                         const TemplateArgument *TemplateArgs,
                                         unsigned NumTemplateArgs) {
diff --git a/lib/CodeGen/MicrosoftCXXABI.cpp b/lib/CodeGen/MicrosoftCXXABI.cpp
index da0fdb616d6c..9407335e3283 100644
--- a/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -41,8 +41,6 @@ public:
   MicrosoftCXXNameMangler(MangleContext &C, llvm::SmallVectorImpl<char> &Res)
   : Context(C), Out(Res) { }
 
-  llvm::raw_svector_ostream &getStream() { return Out; }
-
   void mangle(const NamedDecl *D, llvm::StringRef Prefix = "?");
   void mangleName(const NamedDecl *ND);
   void mangleFunctionEncoding(const FunctionDecl *FD);
@@ -110,15 +108,43 @@ public:
                              llvm::SmallVectorImpl<char> &);
 };
 
-class MicrosoftCXXABI : public CXXABI {
+class MicrosoftCXXABI : public CGCXXABI {
   MicrosoftMangleContext MangleCtx;
 public:
   MicrosoftCXXABI(CodeGenModule &CGM)
-   : MangleCtx(CGM.getContext(), CGM.getDiags()) {}
+    : CGCXXABI(CGM), MangleCtx(CGM.getContext(), CGM.getDiags()) {}
 
   MicrosoftMangleContext &getMangleContext() {
     return MangleCtx;
   }
+
+  void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                 CXXCtorType Type,
+                                 CanQualType &ResTy,
+                                 llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+    // 'this' is already in place
+    // TODO: 'for base' flag
+  }  
+
+  void BuildDestructorSignature(const CXXDestructorDecl *Ctor,
+                                CXXDtorType Type,
+                                CanQualType &ResTy,
+                                llvm::SmallVectorImpl<CanQualType> &ArgTys) {
+    // 'this' is already in place
+    // TODO: 'for base' flag
+  }
+
+  void BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                   QualType &ResTy,
+                                   FunctionArgList &Params) {
+    BuildThisParam(CGF, Params);
+    // TODO: 'for base' flag
+  }
+
+  void EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
+    EmitThisParam(CGF);
+    // TODO: 'for base' flag
+  }
 };
 
 }
@@ -893,6 +919,7 @@ void MicrosoftCXXNameMangler::mangleCallingConvention(const FunctionType *T) {
   switch (T->getCallConv()) {
     case CC_Default:
     case CC_C: Out << 'A'; break;
+    case CC_X86Pascal: Out << 'C'; break;
     case CC_X86ThisCall: Out << 'E'; break;
     case CC_X86StdCall: Out << 'G'; break;
     case CC_X86FastCall: Out << 'I'; break;
@@ -1185,7 +1212,7 @@ void MicrosoftMangleContext::mangleCXXDtor(const CXXDestructorDecl *D,
   assert(false && "Can't yet mangle destructors!");
 }
 
-CXXABI *clang::CodeGen::CreateMicrosoftCXXABI(CodeGenModule &CGM) {
+CGCXXABI *clang::CodeGen::CreateMicrosoftCXXABI(CodeGenModule &CGM) {
   return new MicrosoftCXXABI(CGM);
 }
 
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp
index c65f20371552..4d221d4e657e 100644
--- a/lib/CodeGen/TargetInfo.cpp
+++ b/lib/CodeGen/TargetInfo.cpp
@@ -36,14 +36,36 @@ static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
   }
 }
 
+static bool isAggregateTypeForABI(QualType T) {
+  return CodeGenFunction::hasAggregateLLVMType(T) ||
+         T->isMemberFunctionPointerType();
+}
+
 ABIInfo::~ABIInfo() {}
 
+ASTContext &ABIInfo::getContext() const {
+  return CGT.getContext();
+}
+
+llvm::LLVMContext &ABIInfo::getVMContext() const {
+  return CGT.getLLVMContext();
+}
+
+const llvm::TargetData &ABIInfo::getTargetData() const {
+  return CGT.getTargetData();
+}
+
+
 void ABIArgInfo::dump() const {
   llvm::raw_ostream &OS = llvm::errs();
   OS << "(ABIArgInfo Kind=";
   switch (TheKind) {
   case Direct:
-    OS << "Direct";
+    OS << "Direct Type=";
+    if (const llvm::Type *Ty = getCoerceToType())
+      Ty->print(OS);
+    else
+      OS << "null";
     break;
   case Extend:
     OS << "Extend";
@@ -51,10 +73,6 @@ void ABIArgInfo::dump() const {
   case Ignore:
     OS << "Ignore";
     break;
-  case Coerce:
-    OS << "Coerce Type=";
-    getCoerceToType()->print(OS);
-    break;
   case Indirect:
     OS << "Indirect Align=" << getIndirectAlign()
        << " Byal=" << getIndirectByVal();
@@ -129,7 +147,7 @@ static bool hasNonTrivialDestructorOrCopyConstructor(const RecordType *RT) {
   const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
   if (!RD)
     return false;
-  
+
   return !RD->hasTrivialDestructor() || !RD->hasTrivialCopyConstructor();
 }
 
@@ -162,7 +180,7 @@ static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
     return 0;
 
   const Type *Found = 0;
-  
+
   // If this is a C++ record, check the bases first.
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
     for (CXXRecordDecl::base_class_const_iterator i = CXXRD->bases_begin(),
@@ -205,7 +223,7 @@ static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
       FT = AT->getElementType();
     }
 
-    if (!CodeGenFunction::hasAggregateLLVMType(FT)) {
+    if (!isAggregateTypeForABI(FT)) {
       Found = FT.getTypePtr();
     } else {
       Found = isSingleElementStruct(FT, Context);
@@ -272,23 +290,17 @@ namespace {
 /// self-consistent and sensible LLVM IR generation, but does not
 /// conform to any particular ABI.
 class DefaultABIInfo : public ABIInfo {
-  ABIArgInfo classifyReturnType(QualType RetTy,
-                                ASTContext &Context,
-                                llvm::LLVMContext &VMContext) const;
-
-  ABIArgInfo classifyArgumentType(QualType RetTy,
-                                  ASTContext &Context,
-                                  llvm::LLVMContext &VMContext) const;
-
-  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                           llvm::LLVMContext &VMContext,
-                           const llvm::Type *const *PrefTypes,
-                           unsigned NumPrefTypes) const {
-    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context,
-                                            VMContext);
+public:
+  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+
+  virtual void computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
     for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
          it != ie; ++it)
-      it->info = classifyArgumentType(it->type, Context, VMContext);
+      it->info = classifyArgumentType(it->type);
   }
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -297,7 +309,8 @@ class DefaultABIInfo : public ABIInfo {
 
 class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  DefaultTargetCodeGenInfo():TargetCodeGenInfo(new DefaultABIInfo()) {}
+  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {}
 };
 
 llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -305,10 +318,8 @@ llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return 0;
 }
 
-ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty,
-                                                ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
-  if (CodeGenFunction::hasAggregateLLVMType(Ty))
+ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
+  if (isAggregateTypeForABI(Ty))
     return ABIArgInfo::getIndirect(0);
 
   // Treat an enum type as its underlying type.
@@ -322,10 +333,9 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty,
 //===----------------------------------------------------------------------===//
 // X86-32 ABI Implementation
 //===----------------------------------------------------------------------===//
-  
+
 /// X86_32ABIInfo - The X86-32 ABI information.
 class X86_32ABIInfo : public ABIInfo {
-  ASTContext &Context;
   bool IsDarwinVectorABI;
   bool IsSmallStructInRegABI;
 
@@ -337,41 +347,31 @@ class X86_32ABIInfo : public ABIInfo {
 
   /// getIndirectResult - Give a source type \arg Ty, return a suitable result
   /// such that the argument will be passed in memory.
-  ABIArgInfo getIndirectResult(QualType Ty, ASTContext &Context,
-                               bool ByVal = true) const;
+  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal = true) const;
 
 public:
-  ABIArgInfo classifyReturnType(QualType RetTy,
-                                ASTContext &Context,
-                                llvm::LLVMContext &VMContext) const;
-
-  ABIArgInfo classifyArgumentType(QualType RetTy,
-                                  ASTContext &Context,
-                                  llvm::LLVMContext &VMContext) const;
-
-  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                           llvm::LLVMContext &VMContext,
-                           const llvm::Type *const *PrefTypes,
-                           unsigned NumPrefTypes) const {
-    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context,
-                                            VMContext);
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+
+  virtual void computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
     for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
          it != ie; ++it)
-      it->info = classifyArgumentType(it->type, Context, VMContext);
+      it->info = classifyArgumentType(it->type);
   }
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                  CodeGenFunction &CGF) const;
 
-  X86_32ABIInfo(ASTContext &Context, bool d, bool p)
-    : ABIInfo(), Context(Context), IsDarwinVectorABI(d),
-      IsSmallStructInRegABI(p) {}
+  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool d, bool p)
+    : ABIInfo(CGT), IsDarwinVectorABI(d), IsSmallStructInRegABI(p) {}
 };
 
 class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  X86_32TargetCodeGenInfo(ASTContext &Context, bool d, bool p)
-    :TargetCodeGenInfo(new X86_32ABIInfo(Context, d, p)) {}
+  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool d, bool p)
+    :TargetCodeGenInfo(new X86_32ABIInfo(CGT, d, p)) {}
 
   void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &CGM) const;
@@ -443,153 +443,172 @@ bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
   return true;
 }
 
-ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
-                                            ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
-  if (RetTy->isVoidType()) {
+ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
-  } else if (const VectorType *VT = RetTy->getAs<VectorType>()) {
+
+  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
     // On Darwin, some vectors are returned in registers.
     if (IsDarwinVectorABI) {
-      uint64_t Size = Context.getTypeSize(RetTy);
+      uint64_t Size = getContext().getTypeSize(RetTy);
 
       // 128-bit vectors are a special case; they are returned in
       // registers and we need to make sure to pick a type the LLVM
       // backend will like.
       if (Size == 128)
-        return ABIArgInfo::getCoerce(llvm::VectorType::get(
-                  llvm::Type::getInt64Ty(VMContext), 2));
+        return ABIArgInfo::getDirect(llvm::VectorType::get(
+                  llvm::Type::getInt64Ty(getVMContext()), 2));
 
       // Always return in register if it fits in a general purpose
       // register, or if it is 64 bits and has a single element.
       if ((Size == 8 || Size == 16 || Size == 32) ||
           (Size == 64 && VT->getNumElements() == 1))
-        return ABIArgInfo::getCoerce(llvm::IntegerType::get(VMContext, Size));
+        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
+                                                            Size));
 
       return ABIArgInfo::getIndirect(0);
     }
 
     return ABIArgInfo::getDirect();
-  } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+  }
+
+  if (isAggregateTypeForABI(RetTy)) {
     if (const RecordType *RT = RetTy->getAs<RecordType>()) {
       // Structures with either a non-trivial destructor or a non-trivial
       // copy constructor are always indirect.
       if (hasNonTrivialDestructorOrCopyConstructor(RT))
         return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
-      
+
       // Structures with flexible arrays are always indirect.
       if (RT->getDecl()->hasFlexibleArrayMember())
         return ABIArgInfo::getIndirect(0);
     }
-    
+
     // If specified, structs and unions are always indirect.
     if (!IsSmallStructInRegABI && !RetTy->isAnyComplexType())
       return ABIArgInfo::getIndirect(0);
 
     // Classify "single element" structs as their element type.
-    if (const Type *SeltTy = isSingleElementStruct(RetTy, Context)) {
+    if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext())) {
       if (const BuiltinType *BT = SeltTy->getAs<BuiltinType>()) {
         if (BT->isIntegerType()) {
           // We need to use the size of the structure, padding
           // bit-fields can adjust that to be larger than the single
           // element type.
-          uint64_t Size = Context.getTypeSize(RetTy);
-          return ABIArgInfo::getCoerce(
-            llvm::IntegerType::get(VMContext, (unsigned) Size));
-        } else if (BT->getKind() == BuiltinType::Float) {
-          assert(Context.getTypeSize(RetTy) == Context.getTypeSize(SeltTy) &&
+          uint64_t Size = getContext().getTypeSize(RetTy);
+          return ABIArgInfo::getDirect(
+            llvm::IntegerType::get(getVMContext(), (unsigned)Size));
+        }
+
+        if (BT->getKind() == BuiltinType::Float) {
+          assert(getContext().getTypeSize(RetTy) ==
+                 getContext().getTypeSize(SeltTy) &&
                  "Unexpect single element structure size!");
-          return ABIArgInfo::getCoerce(llvm::Type::getFloatTy(VMContext));
-        } else if (BT->getKind() == BuiltinType::Double) {
-          assert(Context.getTypeSize(RetTy) == Context.getTypeSize(SeltTy) &&
+          return ABIArgInfo::getDirect(llvm::Type::getFloatTy(getVMContext()));
+        }
+
+        if (BT->getKind() == BuiltinType::Double) {
+          assert(getContext().getTypeSize(RetTy) ==
+                 getContext().getTypeSize(SeltTy) &&
                  "Unexpect single element structure size!");
-          return ABIArgInfo::getCoerce(llvm::Type::getDoubleTy(VMContext));
+          return ABIArgInfo::getDirect(llvm::Type::getDoubleTy(getVMContext()));
         }
       } else if (SeltTy->isPointerType()) {
         // FIXME: It would be really nice if this could come out as the proper
         // pointer type.
-        const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(VMContext);
-        return ABIArgInfo::getCoerce(PtrTy);
+        const llvm::Type *PtrTy = llvm::Type::getInt8PtrTy(getVMContext());
+        return ABIArgInfo::getDirect(PtrTy);
       } else if (SeltTy->isVectorType()) {
         // 64- and 128-bit vectors are never returned in a
         // register when inside a structure.
-        uint64_t Size = Context.getTypeSize(RetTy);
+        uint64_t Size = getContext().getTypeSize(RetTy);
         if (Size == 64 || Size == 128)
           return ABIArgInfo::getIndirect(0);
 
-        return classifyReturnType(QualType(SeltTy, 0), Context, VMContext);
+        return classifyReturnType(QualType(SeltTy, 0));
       }
     }
 
     // Small structures which are register sized are generally returned
     // in a register.
-    if (X86_32ABIInfo::shouldReturnTypeInRegister(RetTy, Context)) {
-      uint64_t Size = Context.getTypeSize(RetTy);
-      return ABIArgInfo::getCoerce(llvm::IntegerType::get(VMContext, Size));
+    if (X86_32ABIInfo::shouldReturnTypeInRegister(RetTy, getContext())) {
+      uint64_t Size = getContext().getTypeSize(RetTy);
+      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
     }
 
     return ABIArgInfo::getIndirect(0);
-  } else {
-    // Treat an enum type as its underlying type.
-    if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-      RetTy = EnumTy->getDecl()->getIntegerType();
-
-    return (RetTy->isPromotableIntegerType() ?
-            ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
   }
+
+  // Treat an enum type as its underlying type.
+  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+    RetTy = EnumTy->getDecl()->getIntegerType();
+
+  return (RetTy->isPromotableIntegerType() ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
-ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty,
-                                            ASTContext &Context,
-                                            bool ByVal) const {
+ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal) const {
   if (!ByVal)
     return ABIArgInfo::getIndirect(0, false);
 
   // Compute the byval alignment. We trust the back-end to honor the
   // minimum ABI alignment for byval, to make cleaner IR.
   const unsigned MinABIAlign = 4;
-  unsigned Align = Context.getTypeAlign(Ty) / 8;
+  unsigned Align = getContext().getTypeAlign(Ty) / 8;
   if (Align > MinABIAlign)
     return ABIArgInfo::getIndirect(Align);
   return ABIArgInfo::getIndirect(0);
 }
 
-ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
-                                               ASTContext &Context,
-                                           llvm::LLVMContext &VMContext) const {
+ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
   // FIXME: Set alignment on indirect arguments.
-  if (CodeGenFunction::hasAggregateLLVMType(Ty)) {
+  if (isAggregateTypeForABI(Ty)) {
     // Structures with flexible arrays are always indirect.
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
       // Structures with either a non-trivial destructor or a non-trivial
       // copy constructor are always indirect.
       if (hasNonTrivialDestructorOrCopyConstructor(RT))
-        return getIndirectResult(Ty, Context, /*ByVal=*/false);
+        return getIndirectResult(Ty, /*ByVal=*/false);
 
       if (RT->getDecl()->hasFlexibleArrayMember())
-        return getIndirectResult(Ty, Context);
+        return getIndirectResult(Ty);
     }
 
     // Ignore empty structs.
-    if (Ty->isStructureType() && Context.getTypeSize(Ty) == 0)
+    if (Ty->isStructureType() && getContext().getTypeSize(Ty) == 0)
       return ABIArgInfo::getIgnore();
 
     // Expand small (<= 128-bit) record types when we know that the stack layout
     // of those arguments will match the struct. This is important because the
     // LLVM backend isn't smart enough to remove byval, which inhibits many
     // optimizations.
-    if (Context.getTypeSize(Ty) <= 4*32 &&
-        canExpandIndirectArgument(Ty, Context))
+    if (getContext().getTypeSize(Ty) <= 4*32 &&
+        canExpandIndirectArgument(Ty, getContext()))
       return ABIArgInfo::getExpand();
 
-    return getIndirectResult(Ty, Context);
-  } else {
-    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+    return getIndirectResult(Ty);
+  }
 
-    return (Ty->isPromotableIntegerType() ?
-            ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+  if (const VectorType *VT = Ty->getAs<VectorType>()) {
+    // On Darwin, some vectors are passed in memory, we handle this by passing
+    // it as an i8/i16/i32/i64.
+    if (IsDarwinVectorABI) {
+      uint64_t Size = getContext().getTypeSize(Ty);
+      if ((Size == 8 || Size == 16 || Size == 32) ||
+          (Size == 64 && VT->getNumElements() == 1))
+        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
+                                                            Size));
+    }
+    
+    return ABIArgInfo::getDirect();
   }
+  
+  
+  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+    Ty = EnumTy->getDecl()->getIntegerType();
+
+  return (Ty->isPromotableIntegerType() ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
 llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -637,7 +656,7 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
 
   const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
   llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
-    
+
   // 0-7 are the eight integer registers;  the order is different
   //   on Darwin (for EH), but the range is the same.
   // 8 is %eip.
@@ -649,7 +668,7 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
     // platforms with 8-byte alignment for that type.
     llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
     AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
-      
+
   } else {
     // 9 is %eflags, which doesn't get a size on Darwin for some
     // reason.
@@ -673,9 +692,6 @@ bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
 namespace {
 /// X86_64ABIInfo - The X86_64 ABI information.
 class X86_64ABIInfo : public ABIInfo {
-  ASTContext &Context;
-  const llvm::TargetData &TD;
-  
   enum Class {
     Integer = 0,
     SSE,
@@ -721,17 +737,13 @@ class X86_64ABIInfo : public ABIInfo {
   /// also be ComplexX87.
   void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi) const;
 
-  /// getCoerceResult - Given a source type \arg Ty and an LLVM type
-  /// to coerce to, chose the best way to pass Ty in the same place
-  /// that \arg CoerceTo would be passed, but while keeping the
-  /// emitted code as simple as possible.
-  ///
-  /// FIXME: Note, this should be cleaned up to just take an enumeration of all
-  /// the ways we might want to pass things, instead of constructing an LLVM
-  /// type. This makes this code more explicit, and it makes it clearer that we
-  /// are also doing this for correctness in the case of passing scalar types.
-  ABIArgInfo getCoerceResult(QualType Ty,
-                             const llvm::Type *CoerceTo) const;
+  const llvm::Type *Get16ByteVectorType(QualType Ty) const;
+  const llvm::Type *GetSSETypeAtOffset(const llvm::Type *IRType,
+                                       unsigned IROffset, QualType SourceTy,
+                                       unsigned SourceOffset) const;
+  const llvm::Type *GetINTEGERTypeAtOffset(const llvm::Type *IRType,
+                                           unsigned IROffset, QualType SourceTy,
+                                           unsigned SourceOffset) const;
 
   /// getIndirectResult - Give a source type \arg Ty, return a suitable result
   /// such that the argument will be returned in memory.
@@ -741,23 +753,24 @@ class X86_64ABIInfo : public ABIInfo {
   /// such that the argument will be passed in memory.
   ABIArgInfo getIndirectResult(QualType Ty) const;
 
-  ABIArgInfo classifyReturnType(QualType RetTy,
-                                llvm::LLVMContext &VMContext) const;
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
 
-  ABIArgInfo classifyArgumentType(QualType Ty,
-                                  llvm::LLVMContext &VMContext,
-                                  unsigned &neededInt,
-                                  unsigned &neededSSE,
-                                  const llvm::Type *PrefType) const;
+  ABIArgInfo classifyArgumentType(QualType Ty, unsigned &neededInt,
+                                  unsigned &neededSSE) const;
 
 public:
-  X86_64ABIInfo(ASTContext &Ctx, const llvm::TargetData &td)
-    : Context(Ctx), TD(td) {}
+  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
+
+  virtual void computeInfo(CGFunctionInfo &FI) const;
+
+  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                 CodeGenFunction &CGF) const;
+};
 
-  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                           llvm::LLVMContext &VMContext,
-                           const llvm::Type *const *PrefTypes,
-                           unsigned NumPrefTypes) const;
+/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
+class WinX86_64ABIInfo : public X86_64ABIInfo {
+public:
+  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT) : X86_64ABIInfo(CGT) {}
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                  CodeGenFunction &CGF) const;
@@ -765,8 +778,33 @@ public:
 
 class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  X86_64TargetCodeGenInfo(ASTContext &Ctx, const llvm::TargetData &TD)
-    : TargetCodeGenInfo(new X86_64ABIInfo(Ctx, TD)) {}
+  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new X86_64ABIInfo(CGT)) {}
+
+  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const {
+    return 7;
+  }
+
+  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
+                               llvm::Value *Address) const {
+    CodeGen::CGBuilderTy &Builder = CGF.Builder;
+    llvm::LLVMContext &Context = CGF.getLLVMContext();
+
+    const llvm::IntegerType *i8 = llvm::Type::getInt8Ty(Context);
+    llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
+
+    // 0-15 are the 16 integer registers.
+    // 16 is %rip.
+    AssignToArrayRange(Builder, Address, Eight8, 0, 16);
+
+    return false;
+  }
+};
+
+class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
+public:
+  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new WinX86_64ABIInfo(CGT)) {}
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const {
     return 7;
@@ -865,18 +903,18 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
     return;
   }
-  
+
   if (const EnumType *ET = Ty->getAs<EnumType>()) {
     // Classify the underlying integer type.
     classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi);
     return;
   }
-  
+
   if (Ty->hasPointerRepresentation()) {
     Current = Integer;
     return;
   }
-  
+
   if (Ty->isMemberPointerType()) {
     if (Ty->isMemberFunctionPointerType())
       Lo = Hi = Integer;
@@ -884,9 +922,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
       Current = Integer;
     return;
   }
-  
+
   if (const VectorType *VT = Ty->getAs<VectorType>()) {
-    uint64_t Size = Context.getTypeSize(VT);
+    uint64_t Size = getContext().getTypeSize(VT);
     if (Size == 32) {
       // gcc passes all <4 x char>, <2 x short>, <1 x int>, <1 x
       // float> as integer.
@@ -904,7 +942,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
         return;
 
       // gcc passes <1 x long long> as INTEGER.
-      if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::LongLong))
+      if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::LongLong) ||
+          VT->getElementType()->isSpecificBuiltinType(BuiltinType::ULongLong) ||
+          VT->getElementType()->isSpecificBuiltinType(BuiltinType::Long) ||
+          VT->getElementType()->isSpecificBuiltinType(BuiltinType::ULong))
         Current = Integer;
       else
         Current = SSE;
@@ -919,37 +960,37 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     }
     return;
   }
-  
+
   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    QualType ET = Context.getCanonicalType(CT->getElementType());
+    QualType ET = getContext().getCanonicalType(CT->getElementType());
 
-    uint64_t Size = Context.getTypeSize(Ty);
+    uint64_t Size = getContext().getTypeSize(Ty);
     if (ET->isIntegralOrEnumerationType()) {
       if (Size <= 64)
         Current = Integer;
       else if (Size <= 128)
         Lo = Hi = Integer;
-    } else if (ET == Context.FloatTy)
+    } else if (ET == getContext().FloatTy)
       Current = SSE;
-    else if (ET == Context.DoubleTy)
+    else if (ET == getContext().DoubleTy)
       Lo = Hi = SSE;
-    else if (ET == Context.LongDoubleTy)
+    else if (ET == getContext().LongDoubleTy)
       Current = ComplexX87;
 
     // If this complex type crosses an eightbyte boundary then it
     // should be split.
     uint64_t EB_Real = (OffsetBase) / 64;
-    uint64_t EB_Imag = (OffsetBase + Context.getTypeSize(ET)) / 64;
+    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
     if (Hi == NoClass && EB_Real != EB_Imag)
       Hi = Lo;
-    
+
     return;
   }
-  
-  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
+
+  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
     // Arrays are treated like structures.
 
-    uint64_t Size = Context.getTypeSize(Ty);
+    uint64_t Size = getContext().getTypeSize(Ty);
 
     // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
     // than two eightbytes, ..., it has class MEMORY.
@@ -960,13 +1001,13 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     // fields, it has class MEMORY.
     //
     // Only need to check alignment of array base.
-    if (OffsetBase % Context.getTypeAlign(AT->getElementType()))
+    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
       return;
 
     // Otherwise implement simplified merge. We could be smarter about
     // this, but it isn't worth it and would be harder to verify.
     Current = NoClass;
-    uint64_t EltSize = Context.getTypeSize(AT->getElementType());
+    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
     uint64_t ArraySize = AT->getSize().getZExtValue();
     for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
       Class FieldLo, FieldHi;
@@ -983,9 +1024,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
     return;
   }
-  
+
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    uint64_t Size = Context.getTypeSize(Ty);
+    uint64_t Size = getContext().getTypeSize(Ty);
 
     // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
     // than two eightbytes, ..., it has class MEMORY.
@@ -1004,7 +1045,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     if (RD->hasFlexibleArrayMember())
       return;
 
-    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
 
     // Reset Lo class, this will be recomputed.
     Current = NoClass;
@@ -1031,10 +1072,6 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
         if (Lo == Memory || Hi == Memory)
           break;
       }
-
-      // If this record has no fields but isn't empty, classify as INTEGER.
-      if (RD->field_empty() && Size)
-        Current = Integer;
     }
 
     // Classify the fields one at a time, merging the results.
@@ -1048,7 +1085,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
       // fields, it has class MEMORY.
       //
       // Note, skip this test for bit-fields, see below.
-      if (!BitField && Offset % Context.getTypeAlign(i->getType())) {
+      if (!BitField && Offset % getContext().getTypeAlign(i->getType())) {
         Lo = Memory;
         return;
       }
@@ -1070,7 +1107,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
           continue;
 
         uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
-        uint64_t Size = i->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
+        uint64_t Size =
+          i->getBitWidth()->EvaluateAsInt(getContext()).getZExtValue();
 
         uint64_t EB_Lo = Offset / 64;
         uint64_t EB_Hi = (Offset + Size - 1) / 64;
@@ -1110,52 +1148,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
   }
 }
 
-ABIArgInfo X86_64ABIInfo::getCoerceResult(QualType Ty,
-                                          const llvm::Type *CoerceTo) const {
-  if (CoerceTo->isIntegerTy(64) || isa<llvm::PointerType>(CoerceTo)) {
-    // Integer and pointer types will end up in a general purpose
-    // register.
-
-    // Treat an enum type as its underlying type.
-    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
-
-    if (Ty->isIntegralOrEnumerationType() || Ty->hasPointerRepresentation())
-      return (Ty->isPromotableIntegerType() ?
-              ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
-    
-    // If this is a 8/16/32-bit structure that is passed as an int64, then it
-    // will be passed in the low 8/16/32-bits of a 64-bit GPR, which is the same
-    // as how an i8/i16/i32 is passed.  Coerce to a i8/i16/i32 instead of a i64.
-    switch (Context.getTypeSizeInChars(Ty).getQuantity()) {
-    default: break;
-    case 1: CoerceTo = llvm::Type::getInt8Ty(CoerceTo->getContext()); break;
-    case 2: CoerceTo = llvm::Type::getInt16Ty(CoerceTo->getContext()); break;
-    case 4: CoerceTo = llvm::Type::getInt32Ty(CoerceTo->getContext()); break;
-    }
-    
-  } else if (CoerceTo->isDoubleTy()) {
-    assert(Ty.isCanonical() && "should always have a canonical type here");
-    assert(!Ty.hasQualifiers() && "should never have a qualified type here");
-
-    // Float and double end up in a single SSE reg.
-    if (Ty == Context.FloatTy || Ty == Context.DoubleTy)
-      return ABIArgInfo::getDirect();
-
-    // If this is a 32-bit structure that is passed as a double, then it will be
-    // passed in the low 32-bits of the XMM register, which is the same as how a
-    // float is passed.  Coerce to a float instead of a double.
-    if (Context.getTypeSizeInChars(Ty).getQuantity() == 4)
-      CoerceTo = llvm::Type::getFloatTy(CoerceTo->getContext());
-  }
-
-  return ABIArgInfo::getCoerce(CoerceTo);
-}
-
 ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
   // If this is a scalar LLVM value then assume LLVM will pass it in the right
   // place naturally.
-  if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
+  if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
@@ -1170,7 +1166,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
 ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty) const {
   // If this is a scalar LLVM value then assume LLVM will pass it in the right
   // place naturally.
-  if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
+  if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
@@ -1185,14 +1181,297 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty) const {
   // Compute the byval alignment. We trust the back-end to honor the
   // minimum ABI alignment for byval, to make cleaner IR.
   const unsigned MinABIAlign = 8;
-  unsigned Align = Context.getTypeAlign(Ty) / 8;
+  unsigned Align = getContext().getTypeAlign(Ty) / 8;
   if (Align > MinABIAlign)
     return ABIArgInfo::getIndirect(Align);
   return ABIArgInfo::getIndirect(0);
 }
 
+/// Get16ByteVectorType - The ABI specifies that a value should be passed in an
+/// full vector XMM register.  Pick an LLVM IR type that will be passed as a
+/// vector register.
+const llvm::Type *X86_64ABIInfo::Get16ByteVectorType(QualType Ty) const {
+  const llvm::Type *IRType = CGT.ConvertTypeRecursive(Ty);
+
+  // Wrapper structs that just contain vectors are passed just like vectors,
+  // strip them off if present.
+  const llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType);
+  while (STy && STy->getNumElements() == 1) {
+    IRType = STy->getElementType(0);
+    STy = dyn_cast<llvm::StructType>(IRType);
+  }
+
+  // If the preferred type is a 16-byte vector, prefer to pass it.
+  if (const llvm::VectorType *VT = dyn_cast<llvm::VectorType>(IRType)){
+    const llvm::Type *EltTy = VT->getElementType();
+    if (VT->getBitWidth() == 128 &&
+        (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
+         EltTy->isIntegerTy(8) || EltTy->isIntegerTy(16) ||
+         EltTy->isIntegerTy(32) || EltTy->isIntegerTy(64) ||
+         EltTy->isIntegerTy(128)))
+      return VT;
+  }
+
+  return llvm::VectorType::get(llvm::Type::getDoubleTy(getVMContext()), 2);
+}
+
+/// BitsContainNoUserData - Return true if the specified [start,end) bit range
+/// is known to either be off the end of the specified type or being in
+/// alignment padding.  The user type specified is known to be at most 128 bits
+/// in size, and have passed through X86_64ABIInfo::classify with a successful
+/// classification that put one of the two halves in the INTEGER class.
+///
+/// It is conservatively correct to return false.
+static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
+                                  unsigned EndBit, ASTContext &Context) {
+  // If the bytes being queried are off the end of the type, there is no user
+  // data hiding here.  This handles analysis of builtins, vectors and other
+  // types that don't contain interesting padding.
+  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
+  if (TySize <= StartBit)
+    return true;
+
+  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
+    unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
+    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
+
+    // Check each element to see if the element overlaps with the queried range.
+    for (unsigned i = 0; i != NumElts; ++i) {
+      // If the element is after the span we care about, then we're done..
+      unsigned EltOffset = i*EltSize;
+      if (EltOffset >= EndBit) break;
+
+      unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
+      if (!BitsContainNoUserData(AT->getElementType(), EltStart,
+                                 EndBit-EltOffset, Context))
+        return false;
+    }
+    // If it overlaps no elements, then it is safe to process as padding.
+    return true;
+  }
+
+  if (const RecordType *RT = Ty->getAs<RecordType>()) {
+    const RecordDecl *RD = RT->getDecl();
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+    // If this is a C++ record, check the bases first.
+    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+      for (CXXRecordDecl::base_class_const_iterator i = CXXRD->bases_begin(),
+           e = CXXRD->bases_end(); i != e; ++i) {
+        assert(!i->isVirtual() && !i->getType()->isDependentType() &&
+               "Unexpected base class!");
+        const CXXRecordDecl *Base =
+          cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
+
+        // If the base is after the span we care about, ignore it.
+        unsigned BaseOffset = (unsigned)Layout.getBaseClassOffset(Base);
+        if (BaseOffset >= EndBit) continue;
+
+        unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
+        if (!BitsContainNoUserData(i->getType(), BaseStart,
+                                   EndBit-BaseOffset, Context))
+          return false;
+      }
+    }
+
+    // Verify that no field has data that overlaps the region of interest.  Yes
+    // this could be sped up a lot by being smarter about queried fields,
+    // however we're only looking at structs up to 16 bytes, so we don't care
+    // much.
+    unsigned idx = 0;
+    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+         i != e; ++i, ++idx) {
+      unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
+
+      // If we found a field after the region we care about, then we're done.
+      if (FieldOffset >= EndBit) break;
+
+      unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
+      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
+                                 Context))
+        return false;
+    }
+
+    // If nothing in this record overlapped the area of interest, then we're
+    // clean.
+    return true;
+  }
+
+  return false;
+}
+
+/// ContainsFloatAtOffset - Return true if the specified LLVM IR type has a
+/// float member at the specified offset.  For example, {int,{float}} has a
+/// float at offset 4.  It is conservatively correct for this routine to return
+/// false.
+static bool ContainsFloatAtOffset(const llvm::Type *IRType, unsigned IROffset,
+                                  const llvm::TargetData &TD) {
+  // Base case if we find a float.
+  if (IROffset == 0 && IRType->isFloatTy())
+    return true;
+
+  // If this is a struct, recurse into the field at the specified offset.
+  if (const llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
+    const llvm::StructLayout *SL = TD.getStructLayout(STy);
+    unsigned Elt = SL->getElementContainingOffset(IROffset);
+    IROffset -= SL->getElementOffset(Elt);
+    return ContainsFloatAtOffset(STy->getElementType(Elt), IROffset, TD);
+  }
+
+  // If this is an array, recurse into the field at the specified offset.
+  if (const llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
+    const llvm::Type *EltTy = ATy->getElementType();
+    unsigned EltSize = TD.getTypeAllocSize(EltTy);
+    IROffset -= IROffset/EltSize*EltSize;
+    return ContainsFloatAtOffset(EltTy, IROffset, TD);
+  }
+
+  return false;
+}
+
+
+/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
+/// low 8 bytes of an XMM register, corresponding to the SSE class.
+const llvm::Type *X86_64ABIInfo::
+GetSSETypeAtOffset(const llvm::Type *IRType, unsigned IROffset,
+                   QualType SourceTy, unsigned SourceOffset) const {
+  // The only three choices we have are either double, <2 x float>, or float. We
+  // pass as float if the last 4 bytes is just padding.  This happens for
+  // structs that contain 3 floats.
+  if (BitsContainNoUserData(SourceTy, SourceOffset*8+32,
+                            SourceOffset*8+64, getContext()))
+    return llvm::Type::getFloatTy(getVMContext());
+
+  // We want to pass as <2 x float> if the LLVM IR type contains a float at
+  // offset+0 and offset+4.  Walk the LLVM IR type to find out if this is the
+  // case.
+  if (ContainsFloatAtOffset(IRType, IROffset, getTargetData()) &&
+      ContainsFloatAtOffset(IRType, IROffset+4, getTargetData()))
+    return llvm::VectorType::get(llvm::Type::getFloatTy(getVMContext()), 2);
+
+  return llvm::Type::getDoubleTy(getVMContext());
+}
+
+
+/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
+/// an 8-byte GPR.  This means that we either have a scalar or we are talking
+/// about the high or low part of an up-to-16-byte struct.  This routine picks
+/// the best LLVM IR type to represent this, which may be i64 or may be anything
+/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
+/// etc).
+///
+/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
+/// the source type.  IROffset is an offset in bytes into the LLVM IR type that
+/// the 8-byte value references.  PrefType may be null.
+///
+/// SourceTy is the source level type for the entire argument.  SourceOffset is
+/// an offset into this that we're processing (which is always either 0 or 8).
+///
+const llvm::Type *X86_64ABIInfo::
+GetINTEGERTypeAtOffset(const llvm::Type *IRType, unsigned IROffset,
+                       QualType SourceTy, unsigned SourceOffset) const {
+  // If we're dealing with an un-offset LLVM IR type, then it means that we're
+  // returning an 8-byte unit starting with it.  See if we can safely use it.
+  if (IROffset == 0) {
+    // Pointers and int64's always fill the 8-byte unit.
+    if (isa<llvm::PointerType>(IRType) || IRType->isIntegerTy(64))
+      return IRType;
+
+    // If we have a 1/2/4-byte integer, we can use it only if the rest of the
+    // goodness in the source type is just tail padding.  This is allowed to
+    // kick in for struct {double,int} on the int, but not on
+    // struct{double,int,int} because we wouldn't return the second int.  We
+    // have to do this analysis on the source type because we can't depend on
+    // unions being lowered a specific way etc.
+    if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||
+        IRType->isIntegerTy(32)) {
+      unsigned BitWidth = cast<llvm::IntegerType>(IRType)->getBitWidth();
+
+      if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
+                                SourceOffset*8+64, getContext()))
+        return IRType;
+    }
+  }
+
+  if (const llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
+    // If this is a struct, recurse into the field at the specified offset.
+    const llvm::StructLayout *SL = getTargetData().getStructLayout(STy);
+    if (IROffset < SL->getSizeInBytes()) {
+      unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
+      IROffset -= SL->getElementOffset(FieldIdx);
+
+      return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
+                                    SourceTy, SourceOffset);
+    }
+  }
+
+  if (const llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
+    const llvm::Type *EltTy = ATy->getElementType();
+    unsigned EltSize = getTargetData().getTypeAllocSize(EltTy);
+    unsigned EltOffset = IROffset/EltSize*EltSize;
+    return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
+                                  SourceOffset);
+  }
+
+  // Okay, we don't have any better idea of what to pass, so we pass this in an
+  // integer register that isn't too big to fit the rest of the struct.
+  unsigned TySizeInBytes =
+    (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
+
+  assert(TySizeInBytes != SourceOffset && "Empty field?");
+
+  // It is always safe to classify this as an integer type up to i64 that
+  // isn't larger than the structure.
+  return llvm::IntegerType::get(getVMContext(),
+                                std::min(TySizeInBytes-SourceOffset, 8U)*8);
+}
+
+
+/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
+/// be used as elements of a two register pair to pass or return, return a
+/// first class aggregate to represent them.  For example, if the low part of
+/// a by-value argument should be passed as i32* and the high part as float,
+/// return {i32*, float}.
+static const llvm::Type *
+GetX86_64ByValArgumentPair(const llvm::Type *Lo, const llvm::Type *Hi,
+                           const llvm::TargetData &TD) {
+  // In order to correctly satisfy the ABI, we need to the high part to start
+  // at offset 8.  If the high and low parts we inferred are both 4-byte types
+  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
+  // the second element at offset 8.  Check for this:
+  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
+  unsigned HiAlign = TD.getABITypeAlignment(Hi);
+  unsigned HiStart = llvm::TargetData::RoundUpAlignment(LoSize, HiAlign);
+  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
+  
+  // To handle this, we have to increase the size of the low part so that the
+  // second element will start at an 8 byte offset.  We can't increase the size
+  // of the second element because it might make us access off the end of the
+  // struct.
+  if (HiStart != 8) {
+    // There are only two sorts of types the ABI generation code can produce for
+    // the low part of a pair that aren't 8 bytes in size: float or i8/i16/i32.
+    // Promote these to a larger type.
+    if (Lo->isFloatTy())
+      Lo = llvm::Type::getDoubleTy(Lo->getContext());
+    else {
+      assert(Lo->isIntegerTy() && "Invalid/unknown lo type");
+      Lo = llvm::Type::getInt64Ty(Lo->getContext());
+    }
+  }
+  
+  const llvm::StructType *Result = 
+    llvm::StructType::get(Lo->getContext(), Lo, Hi, NULL);
+  
+  
+  // Verify that the second element is at an 8-byte offset.
+  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
+         "Invalid x86-64 argument pair!");
+  return Result;
+}
+
 ABIArgInfo X86_64ABIInfo::
-classifyReturnType(QualType RetTy, llvm::LLVMContext &VMContext) const {
+classifyReturnType(QualType RetTy) const {
   // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
   // classification algorithm.
   X86_64ABIInfo::Class Lo, Hi;
@@ -1200,13 +1479,18 @@ classifyReturnType(QualType RetTy, llvm::LLVMContext &VMContext) const {
 
   // Check some invariants.
   assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
-  assert((Lo != NoClass || Hi == NoClass) && "Invalid null classification.");
   assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
 
   const llvm::Type *ResType = 0;
   switch (Lo) {
   case NoClass:
-    return ABIArgInfo::getIgnore();
+    if (Hi == NoClass)
+      return ABIArgInfo::getIgnore();
+    // If the low part is just padding, it takes no register, leave ResType
+    // null.
+    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
+           "Unknown missing lo part");
+    break;
 
   case SSEUp:
   case X87Up:
@@ -1220,30 +1504,47 @@ classifyReturnType(QualType RetTy, llvm::LLVMContext &VMContext) const {
     // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
     // available register of the sequence %rax, %rdx is used.
   case Integer:
-    ResType = llvm::Type::getInt64Ty(VMContext); break;
+    ResType = GetINTEGERTypeAtOffset(CGT.ConvertTypeRecursive(RetTy), 0,
+                                     RetTy, 0);
+
+    // If we have a sign or zero extended integer, make sure to return Extend
+    // so that the parameter gets the right LLVM IR attributes.
+    if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
+      // Treat an enum type as its underlying type.
+      if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+        RetTy = EnumTy->getDecl()->getIntegerType();
+
+      if (RetTy->isIntegralOrEnumerationType() &&
+          RetTy->isPromotableIntegerType())
+        return ABIArgInfo::getExtend();
+    }
+    break;
 
     // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
     // available SSE register of the sequence %xmm0, %xmm1 is used.
   case SSE:
-    ResType = llvm::Type::getDoubleTy(VMContext); break;
+    ResType = GetSSETypeAtOffset(CGT.ConvertTypeRecursive(RetTy), 0, RetTy, 0);
+    break;
 
     // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
     // returned on the X87 stack in %st0 as 80-bit x87 number.
   case X87:
-    ResType = llvm::Type::getX86_FP80Ty(VMContext); break;
+    ResType = llvm::Type::getX86_FP80Ty(getVMContext());
+    break;
 
     // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
     // part of the value is returned in %st0 and the imaginary part in
     // %st1.
   case ComplexX87:
     assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
-    ResType = llvm::StructType::get(VMContext,
-                                    llvm::Type::getX86_FP80Ty(VMContext),
-                                    llvm::Type::getX86_FP80Ty(VMContext),
+    ResType = llvm::StructType::get(getVMContext(),
+                                    llvm::Type::getX86_FP80Ty(getVMContext()),
+                                    llvm::Type::getX86_FP80Ty(getVMContext()),
                                     NULL);
     break;
   }
 
+  const llvm::Type *HighPart = 0;
   switch (Hi) {
     // Memory was handled previously and X87 should
     // never occur as a hi class.
@@ -1252,15 +1553,19 @@ classifyReturnType(QualType RetTy, llvm::LLVMContext &VMContext) const {
     assert(0 && "Invalid classification for hi word.");
 
   case ComplexX87: // Previously handled.
-  case NoClass: break;
+  case NoClass:
+    break;
 
   case Integer:
-    ResType = llvm::StructType::get(VMContext, ResType,
-                                    llvm::Type::getInt64Ty(VMContext), NULL);
+    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertTypeRecursive(RetTy),
+                                      8, RetTy, 8);
+    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
+      return ABIArgInfo::getDirect(HighPart, 8);
     break;
   case SSE:
-    ResType = llvm::StructType::get(VMContext, ResType,
-                                    llvm::Type::getDoubleTy(VMContext), NULL);
+    HighPart = GetSSETypeAtOffset(CGT.ConvertTypeRecursive(RetTy), 8, RetTy, 8);
+    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
+      return ABIArgInfo::getDirect(HighPart, 8);
     break;
 
     // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
@@ -1269,7 +1574,7 @@ classifyReturnType(QualType RetTy, llvm::LLVMContext &VMContext) const {
     // SSEUP should always be preceeded by SSE, just widen.
   case SSEUp:
     assert(Lo == SSE && "Unexpected SSEUp classification.");
-    ResType = llvm::VectorType::get(llvm::Type::getDoubleTy(VMContext), 2);
+    ResType = Get16ByteVectorType(RetTy);
     break;
 
     // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
@@ -1279,51 +1584,32 @@ classifyReturnType(QualType RetTy, llvm::LLVMContext &VMContext) const {
     // anything. However, in some cases with unions it may not be
     // preceeded by X87. In such situations we follow gcc and pass the
     // extra bits in an SSE reg.
-    if (Lo != X87)
-      ResType = llvm::StructType::get(VMContext, ResType,
-                                      llvm::Type::getDoubleTy(VMContext), NULL);
+    if (Lo != X87) {
+      HighPart = GetSSETypeAtOffset(CGT.ConvertTypeRecursive(RetTy),
+                                    8, RetTy, 8);
+      if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
+        return ABIArgInfo::getDirect(HighPart, 8);
+    }
     break;
   }
-
-  return getCoerceResult(RetTy, ResType);
-}
-
-static const llvm::Type *Get8ByteTypeAtOffset(const llvm::Type *PrefType,
-                                              unsigned Offset,
-                                              const llvm::TargetData &TD) {
-  if (PrefType == 0) return 0;
-  
-  // Pointers are always 8-bytes at offset 0.
-  if (Offset == 0 && isa<llvm::PointerType>(PrefType))
-    return PrefType;
-  
-  // TODO: 1/2/4/8 byte integers are also interesting, but we have to know that
-  // the "hole" is not used in the containing struct (just undef padding).
-  const llvm::StructType *STy = dyn_cast<llvm::StructType>(PrefType);
-  if (STy == 0) return 0;
- 
-  // If this is a struct, recurse into the field at the specified offset.
-  const llvm::StructLayout *SL = TD.getStructLayout(STy);
-  if (Offset >= SL->getSizeInBytes()) return 0;
-  
-  unsigned FieldIdx = SL->getElementContainingOffset(Offset);
-  Offset -= SL->getElementOffset(FieldIdx);
   
-  return Get8ByteTypeAtOffset(STy->getElementType(FieldIdx), Offset, TD);
+  // If a high part was specified, merge it together with the low part.  It is
+  // known to pass in the high eightbyte of the result.  We do this by forming a
+  // first class struct aggregate with the high and low part: {low, high}
+  if (HighPart)
+    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getTargetData());
+
+  return ABIArgInfo::getDirect(ResType);
 }
 
-ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty,
-                                               llvm::LLVMContext &VMContext,
-                                               unsigned &neededInt,
-                                               unsigned &neededSSE,
-                                               const llvm::Type *PrefType)const{
+ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned &neededInt,
+                                               unsigned &neededSSE) const {
   X86_64ABIInfo::Class Lo, Hi;
   classify(Ty, 0, Lo, Hi);
 
   // Check some invariants.
   // FIXME: Enforce these by construction.
   assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
-  assert((Lo != NoClass || Hi == NoClass) && "Invalid null classification.");
   assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
 
   neededInt = 0;
@@ -1331,7 +1617,13 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty,
   const llvm::Type *ResType = 0;
   switch (Lo) {
   case NoClass:
-    return ABIArgInfo::getIgnore();
+    if (Hi == NoClass)
+      return ABIArgInfo::getIgnore();
+    // If the low part is just padding, it takes no register, leave ResType
+    // null.
+    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
+           "Unknown missing lo part");
+    break;
 
     // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
     // on the stack.
@@ -1351,16 +1643,23 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty,
     // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
     // and %r9 is used.
   case Integer:
-    // It is always safe to classify this as an i64 argument.
-    ResType = llvm::Type::getInt64Ty(VMContext);
     ++neededInt;
-      
-    // If we can choose a better 8-byte type based on the preferred type, and if
-    // that type is still passed in a GPR, use it.
-    if (const llvm::Type *PrefTypeLo = Get8ByteTypeAtOffset(PrefType, 0, TD))
-      if (isa<llvm::IntegerType>(PrefTypeLo) ||
-          isa<llvm::PointerType>(PrefTypeLo))
-        ResType = PrefTypeLo;
+
+    // Pick an 8-byte type based on the preferred type.
+    ResType = GetINTEGERTypeAtOffset(CGT.ConvertTypeRecursive(Ty), 0, Ty, 0);
+
+    // If we have a sign or zero extended integer, make sure to return Extend
+    // so that the parameter gets the right LLVM IR attributes.
+    if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
+      // Treat an enum type as its underlying type.
+      if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+        Ty = EnumTy->getDecl()->getIntegerType();
+
+      if (Ty->isIntegralOrEnumerationType() &&
+          Ty->isPromotableIntegerType())
+        return ABIArgInfo::getExtend();
+    }
+
     break;
 
     // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
@@ -1368,10 +1667,11 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty,
     // order from %xmm0 to %xmm7.
   case SSE:
     ++neededSSE;
-    ResType = llvm::Type::getDoubleTy(VMContext);
+    ResType = GetSSETypeAtOffset(CGT.ConvertTypeRecursive(Ty), 0, Ty, 0);
     break;
   }
 
+  const llvm::Type *HighPart = 0;
   switch (Hi) {
     // Memory was handled previously, ComplexX87 and X87 should
     // never occur as hi classes, and X87Up must be preceed by X87,
@@ -1383,49 +1683,49 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty,
     break;
 
   case NoClass: break;
-      
-  case Integer: {
-    // It is always safe to classify this as an i64 argument.
-    const llvm::Type *HiType = llvm::Type::getInt64Ty(VMContext);
+
+  case Integer:
     ++neededInt;
+    // Pick an 8-byte type based on the preferred type.
+    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertTypeRecursive(Ty), 8, Ty, 8);
 
-    // If we can choose a better 8-byte type based on the preferred type, and if
-    // that type is still passed in a GPR, use it.
-    if (const llvm::Type *PrefTypeHi = Get8ByteTypeAtOffset(PrefType, 8, TD))
-      if (isa<llvm::IntegerType>(PrefTypeHi) ||
-          isa<llvm::PointerType>(PrefTypeHi))
-        HiType = PrefTypeHi;
-      
-    ResType = llvm::StructType::get(VMContext, ResType, HiType, NULL);
+    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
+      return ABIArgInfo::getDirect(HighPart, 8);
     break;
-  }
 
     // X87Up generally doesn't occur here (long double is passed in
     // memory), except in situations involving unions.
   case X87Up:
   case SSE:
-    ResType = llvm::StructType::get(VMContext, ResType,
-                                    llvm::Type::getDoubleTy(VMContext), NULL);
+    HighPart = GetSSETypeAtOffset(CGT.ConvertTypeRecursive(Ty), 8, Ty, 8);
+
+    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
+      return ABIArgInfo::getDirect(HighPart, 8);
+
     ++neededSSE;
     break;
 
     // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
     // eightbyte is passed in the upper half of the last used SSE
-    // register.
+    // register.  This only happens when 128-bit vectors are passed.
   case SSEUp:
-    assert(Lo == SSE && "Unexpected SSEUp classification.");
-    ResType = llvm::VectorType::get(llvm::Type::getDoubleTy(VMContext), 2);
+    assert(Lo == SSE && "Unexpected SSEUp classification");
+    ResType = Get16ByteVectorType(Ty);
     break;
   }
 
-  return getCoerceResult(Ty, ResType);
+  // If a high part was specified, merge it together with the low part.  It is
+  // known to pass in the high eightbyte of the result.  We do this by forming a
+  // first class struct aggregate with the high and low part: {low, high}
+  if (HighPart)
+    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getTargetData());
+  
+  return ABIArgInfo::getDirect(ResType);
 }
 
-void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                                llvm::LLVMContext &VMContext,
-                                const llvm::Type *const *PrefTypes,
-                                unsigned NumPrefTypes) const {
-  FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), VMContext);
+void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
+
+  FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
 
   // Keep track of the number of assigned registers.
   unsigned freeIntRegs = 6, freeSSERegs = 8;
@@ -1439,17 +1739,8 @@ void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI, ASTContext &Context,
   // get assigned (in left-to-right order) for passing as follows...
   for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
        it != ie; ++it) {
-    // If the client specified a preferred IR type to use, pass it down to
-    // classifyArgumentType.
-    const llvm::Type *PrefType = 0;
-    if (NumPrefTypes) {
-      PrefType = *PrefTypes++;
-      --NumPrefTypes;
-    }
-      
     unsigned neededInt, neededSSE;
-    it->info = classifyArgumentType(it->type, VMContext,
-                                    neededInt, neededSSE, PrefType);
+    it->info = classifyArgumentType(it->type, neededInt, neededSSE);
 
     // AMD64-ABI 3.2.3p3: If there are no registers available for any
     // eightbyte of an argument, the whole argument is passed on the
@@ -1527,9 +1818,9 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   //   i8* reg_save_area;
   // };
   unsigned neededInt, neededSSE;
-  
+
   Ty = CGF.getContext().getCanonicalType(Ty);
-  ABIArgInfo AI = classifyArgumentType(Ty, VMContext, neededInt, neededSSE, 0);
+  ABIArgInfo AI = classifyArgumentType(Ty, neededInt, neededSSE);
 
   // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
   // in the registers. If not go to step 7.
@@ -1591,13 +1882,13 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                            "reg_save_area");
   if (neededInt && neededSSE) {
     // FIXME: Cleanup.
-    assert(AI.isCoerce() && "Unexpected ABI info for mixed regs");
+    assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
     const llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
     llvm::Value *Tmp = CGF.CreateTempAlloca(ST);
     assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
     const llvm::Type *TyLo = ST->getElementType(0);
     const llvm::Type *TyHi = ST->getElementType(1);
-    assert((TyLo->isFloatingPointTy() ^ TyHi->isFloatingPointTy()) &&
+    assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
            "Unexpected ABI info for mixed regs");
     const llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
     const llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
@@ -1674,7 +1965,28 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return ResAddr;
 }
 
+llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                      CodeGenFunction &CGF) const {
+  const llvm::Type *BP = llvm::Type::getInt8PtrTy(CGF.getLLVMContext());
+  const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
 
+  CGBuilderTy &Builder = CGF.Builder;
+  llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
+                                                       "ap");
+  llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
+  llvm::Type *PTy =
+    llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
+  llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
+
+  uint64_t Offset =
+    llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 8);
+  llvm::Value *NextAddr =
+    Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
+                      "ap.next");
+  Builder.CreateStore(NextAddr, VAListAddrAsBPP);
+
+  return AddrTyped;
+}
 
 //===----------------------------------------------------------------------===//
 // PIC16 ABI Implementation
@@ -1683,23 +1995,18 @@ llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
 namespace {
 
 class PIC16ABIInfo : public ABIInfo {
-  ABIArgInfo classifyReturnType(QualType RetTy,
-                                ASTContext &Context,
-                                llvm::LLVMContext &VMContext) const;
-
-  ABIArgInfo classifyArgumentType(QualType RetTy,
-                                  ASTContext &Context,
-                                  llvm::LLVMContext &VMContext) const;
-
-  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                           llvm::LLVMContext &VMContext,
-                           const llvm::Type *const *PrefTypes,
-                           unsigned NumPrefTypes) const {
-    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context,
-                                            VMContext);
+public:
+  PIC16ABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+
+  virtual void computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
     for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
          it != ie; ++it)
-      it->info = classifyArgumentType(it->type, Context, VMContext);
+      it->info = classifyArgumentType(it->type);
   }
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -1708,14 +2015,13 @@ class PIC16ABIInfo : public ABIInfo {
 
 class PIC16TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  PIC16TargetCodeGenInfo():TargetCodeGenInfo(new PIC16ABIInfo()) {}
+  PIC16TargetCodeGenInfo(CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new PIC16ABIInfo(CGT)) {}
 };
 
 }
 
-ABIArgInfo PIC16ABIInfo::classifyReturnType(QualType RetTy,
-                                            ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
+ABIArgInfo PIC16ABIInfo::classifyReturnType(QualType RetTy) const {
   if (RetTy->isVoidType()) {
     return ABIArgInfo::getIgnore();
   } else {
@@ -1723,9 +2029,7 @@ ABIArgInfo PIC16ABIInfo::classifyReturnType(QualType RetTy,
   }
 }
 
-ABIArgInfo PIC16ABIInfo::classifyArgumentType(QualType Ty,
-                                              ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
+ABIArgInfo PIC16ABIInfo::classifyArgumentType(QualType Ty) const {
   return ABIArgInfo::getDirect();
 }
 
@@ -1759,13 +2063,15 @@ llvm::Value *PIC16ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
 namespace {
 class PPC32TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
 public:
+  PPC32TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
+
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const {
     // This is recovered from gcc output.
     return 1; // r1 is the dedicated stack pointer
   }
 
   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
-                               llvm::Value *Address) const;  
+                               llvm::Value *Address) const;
 };
 
 }
@@ -1809,7 +2115,7 @@ PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
   // 113: sfp
   AssignToArrayRange(Builder, Address, Four8, 109, 113);
 
-  return false;  
+  return false;
 }
 
 
@@ -1831,23 +2137,15 @@ private:
   ABIKind Kind;
 
 public:
-  ARMABIInfo(ABIKind _Kind) : Kind(_Kind) {}
+  ARMABIInfo(CodeGenTypes &CGT, ABIKind _Kind) : ABIInfo(CGT), Kind(_Kind) {}
 
 private:
   ABIKind getABIKind() const { return Kind; }
 
-  ABIArgInfo classifyReturnType(QualType RetTy,
-                                ASTContext &Context,
-                                llvm::LLVMContext &VMCOntext) const;
-
-  ABIArgInfo classifyArgumentType(QualType RetTy,
-                                  ASTContext &Context,
-                                  llvm::LLVMContext &VMContext) const;
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
 
-  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                           llvm::LLVMContext &VMContext,
-                           const llvm::Type *const *PrefTypes,
-                           unsigned NumPrefTypes) const;
+  virtual void computeInfo(CGFunctionInfo &FI) const;
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                  CodeGenFunction &CGF) const;
@@ -1855,8 +2153,8 @@ private:
 
 class ARMTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  ARMTargetCodeGenInfo(ARMABIInfo::ABIKind K)
-    :TargetCodeGenInfo(new ARMABIInfo(K)) {}
+  ARMTargetCodeGenInfo(CodeGenTypes &CGT, ARMABIInfo::ABIKind K)
+    :TargetCodeGenInfo(new ARMABIInfo(CGT, K)) {}
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const {
     return 13;
@@ -1865,18 +2163,13 @@ public:
 
 }
 
-void ARMABIInfo::computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                             llvm::LLVMContext &VMContext,
-                             const llvm::Type *const *PrefTypes,
-                             unsigned NumPrefTypes) const {
-  FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context,
-                                          VMContext);
+void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
+  FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
   for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
-       it != ie; ++it) {
-    it->info = classifyArgumentType(it->type, Context, VMContext);
-  }
+       it != ie; ++it)
+    it->info = classifyArgumentType(it->type);
 
-  const llvm::Triple &Triple(Context.Target.getTriple());
+  const llvm::Triple &Triple(getContext().Target.getTriple());
   llvm::CallingConv::ID DefaultCC;
   if (Triple.getEnvironmentName() == "gnueabi" ||
       Triple.getEnvironmentName() == "eabi")
@@ -1901,10 +2194,8 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI, ASTContext &Context,
   }
 }
 
-ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty,
-                                            ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
-  if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
+ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty) const {
+  if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
@@ -1914,7 +2205,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty,
   }
 
   // Ignore empty records.
-  if (isEmptyRecord(Context, Ty, true))
+  if (isEmptyRecord(getContext(), Ty, true))
     return ABIArgInfo::getIgnore();
 
   // Structures with either a non-trivial destructor or a non-trivial
@@ -1927,21 +2218,21 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty,
   // FIXME: This doesn't handle alignment > 64 bits.
   const llvm::Type* ElemTy;
   unsigned SizeRegs;
-  if (Context.getTypeAlign(Ty) > 32) {
-    ElemTy = llvm::Type::getInt64Ty(VMContext);
-    SizeRegs = (Context.getTypeSize(Ty) + 63) / 64;
+  if (getContext().getTypeAlign(Ty) > 32) {
+    ElemTy = llvm::Type::getInt64Ty(getVMContext());
+    SizeRegs = (getContext().getTypeSize(Ty) + 63) / 64;
   } else {
-    ElemTy = llvm::Type::getInt32Ty(VMContext);
-    SizeRegs = (Context.getTypeSize(Ty) + 31) / 32;
+    ElemTy = llvm::Type::getInt32Ty(getVMContext());
+    SizeRegs = (getContext().getTypeSize(Ty) + 31) / 32;
   }
   std::vector<const llvm::Type*> LLVMFields;
   LLVMFields.push_back(llvm::ArrayType::get(ElemTy, SizeRegs));
-  const llvm::Type* STy = llvm::StructType::get(VMContext, LLVMFields, true);
-  return ABIArgInfo::getCoerce(STy);
+  const llvm::Type* STy = llvm::StructType::get(getVMContext(), LLVMFields,
+                                                true);
+  return ABIArgInfo::getDirect(STy);
 }
 
-static bool isIntegerLikeType(QualType Ty,
-                              ASTContext &Context,
+static bool isIntegerLikeType(QualType Ty, ASTContext &Context,
                               llvm::LLVMContext &VMContext) {
   // APCS, C Language Calling Conventions, Non-Simple Return Values: A structure
   // is called integer-like if its size is less than or equal to one word, and
@@ -2011,7 +2302,7 @@ static bool isIntegerLikeType(QualType Ty,
 
     if (!isIntegerLikeType(FD->getType(), Context, VMContext))
       return false;
-    
+
     // Only allow at most one field in a structure. This doesn't match the
     // wording above, but follows gcc in situations with a field following an
     // empty structure.
@@ -2026,13 +2317,11 @@ static bool isIntegerLikeType(QualType Ty,
   return true;
 }
 
-ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy,
-                                          ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
+ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy) const {
   if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
 
-  if (!CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+  if (!isAggregateTypeForABI(RetTy)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
       RetTy = EnumTy->getDecl()->getIntegerType();
@@ -2048,7 +2337,7 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy,
 
   // Are we following APCS?
   if (getABIKind() == APCS) {
-    if (isEmptyRecord(Context, RetTy, false))
+    if (isEmptyRecord(getContext(), RetTy, false))
       return ABIArgInfo::getIgnore();
 
     // Complex types are all returned as packed integers.
@@ -2056,18 +2345,18 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy,
     // FIXME: Consider using 2 x vector types if the back end handles them
     // correctly.
     if (RetTy->isAnyComplexType())
-      return ABIArgInfo::getCoerce(llvm::IntegerType::get(
-                                     VMContext, Context.getTypeSize(RetTy)));
+      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
+                                              getContext().getTypeSize(RetTy)));
 
     // Integer like structures are returned in r0.
-    if (isIntegerLikeType(RetTy, Context, VMContext)) {
+    if (isIntegerLikeType(RetTy, getContext(), getVMContext())) {
       // Return in the smallest viable integer type.
-      uint64_t Size = Context.getTypeSize(RetTy);
+      uint64_t Size = getContext().getTypeSize(RetTy);
       if (Size <= 8)
-        return ABIArgInfo::getCoerce(llvm::Type::getInt8Ty(VMContext));
+        return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
       if (Size <= 16)
-        return ABIArgInfo::getCoerce(llvm::Type::getInt16Ty(VMContext));
-      return ABIArgInfo::getCoerce(llvm::Type::getInt32Ty(VMContext));
+        return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
+      return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
     }
 
     // Otherwise return in memory.
@@ -2076,19 +2365,19 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy,
 
   // Otherwise this is an AAPCS variant.
 
-  if (isEmptyRecord(Context, RetTy, true))
+  if (isEmptyRecord(getContext(), RetTy, true))
     return ABIArgInfo::getIgnore();
 
   // Aggregates <= 4 bytes are returned in r0; other aggregates
   // are returned indirectly.
-  uint64_t Size = Context.getTypeSize(RetTy);
+  uint64_t Size = getContext().getTypeSize(RetTy);
   if (Size <= 32) {
     // Return in the smallest viable integer type.
     if (Size <= 8)
-      return ABIArgInfo::getCoerce(llvm::Type::getInt8Ty(VMContext));
+      return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
     if (Size <= 16)
-      return ABIArgInfo::getCoerce(llvm::Type::getInt16Ty(VMContext));
-    return ABIArgInfo::getCoerce(llvm::Type::getInt32Ty(VMContext));
+      return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
+    return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
   }
 
   return ABIArgInfo::getIndirect(0);
@@ -2118,21 +2407,19 @@ llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return AddrTyped;
 }
 
-ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy,
-                                              ASTContext &Context,
-                                          llvm::LLVMContext &VMContext) const {
-  if (RetTy->isVoidType()) {
+ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
-  } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+
+  if (isAggregateTypeForABI(RetTy))
     return ABIArgInfo::getIndirect(0);
-  } else {
-    // Treat an enum type as its underlying type.
-    if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-      RetTy = EnumTy->getDecl()->getIntegerType();
 
-    return (RetTy->isPromotableIntegerType() ?
-            ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
-  }
+  // Treat an enum type as its underlying type.
+  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+    RetTy = EnumTy->getDecl()->getIntegerType();
+
+  return (RetTy->isPromotableIntegerType() ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
 //===----------------------------------------------------------------------===//
@@ -2142,23 +2429,19 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy,
 namespace {
 
 class SystemZABIInfo : public ABIInfo {
-  bool isPromotableIntegerType(QualType Ty) const;
+public:
+  SystemZABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
 
-  ABIArgInfo classifyReturnType(QualType RetTy, ASTContext &Context,
-                                llvm::LLVMContext &VMContext) const;
+  bool isPromotableIntegerType(QualType Ty) const;
 
-  ABIArgInfo classifyArgumentType(QualType RetTy, ASTContext &Context,
-                                  llvm::LLVMContext &VMContext) const;
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy) const;
 
-  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context,
-                          llvm::LLVMContext &VMContext,
-                           const llvm::Type *const *PrefTypes,
-                           unsigned NumPrefTypes) const {
-    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(),
-                                            Context, VMContext);
+  virtual void computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
     for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
          it != ie; ++it)
-      it->info = classifyArgumentType(it->type, Context, VMContext);
+      it->info = classifyArgumentType(it->type);
   }
 
   virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -2167,7 +2450,8 @@ class SystemZABIInfo : public ABIInfo {
 
 class SystemZTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  SystemZTargetCodeGenInfo():TargetCodeGenInfo(new SystemZABIInfo()) {}
+  SystemZTargetCodeGenInfo(CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new SystemZABIInfo(CGT)) {}
 };
 
 }
@@ -2199,28 +2483,22 @@ llvm::Value *SystemZABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
 }
 
 
-ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy,
-                                              ASTContext &Context,
-                                           llvm::LLVMContext &VMContext) const {
-  if (RetTy->isVoidType()) {
+ABIArgInfo SystemZABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
-  } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
+  if (isAggregateTypeForABI(RetTy))
     return ABIArgInfo::getIndirect(0);
-  } else {
-    return (isPromotableIntegerType(RetTy) ?
-            ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
-  }
+
+  return (isPromotableIntegerType(RetTy) ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
-ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty,
-                                                ASTContext &Context,
-                                           llvm::LLVMContext &VMContext) const {
-  if (CodeGenFunction::hasAggregateLLVMType(Ty)) {
+ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
+  if (isAggregateTypeForABI(Ty))
     return ABIArgInfo::getIndirect(0);
-  } else {
-    return (isPromotableIntegerType(Ty) ?
-            ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
-  }
+
+  return (isPromotableIntegerType(Ty) ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
 //===----------------------------------------------------------------------===//
@@ -2231,7 +2509,8 @@ namespace {
 
 class MSP430TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  MSP430TargetCodeGenInfo():TargetCodeGenInfo(new DefaultABIInfo()) {}
+  MSP430TargetCodeGenInfo(CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {}
   void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &M) const;
 };
@@ -2270,14 +2549,15 @@ void MSP430TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
 namespace {
 class MIPSTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
-  MIPSTargetCodeGenInfo(): TargetCodeGenInfo(new DefaultABIInfo()) {}
+  MIPSTargetCodeGenInfo(CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new DefaultABIInfo(CGT)) {}
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const {
     return 29;
   }
 
   bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
-                               llvm::Value *Address) const;  
+                               llvm::Value *Address) const;
 };
 }
 
@@ -2315,7 +2595,7 @@ MIPSTargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
 }
 
 
-const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() const {
+const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   if (TheTargetCodeGenInfo)
     return *TheTargetCodeGenInfo;
 
@@ -2325,56 +2605,61 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() const {
   const llvm::Triple &Triple = getContext().Target.getTriple();
   switch (Triple.getArch()) {
   default:
-    return *(TheTargetCodeGenInfo = new DefaultTargetCodeGenInfo());
+    return *(TheTargetCodeGenInfo = new DefaultTargetCodeGenInfo(Types));
 
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
-    return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo());
+    return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types));
 
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
     // FIXME: We want to know the float calling convention as well.
     if (strcmp(getContext().Target.getABI(), "apcs-gnu") == 0)
       return *(TheTargetCodeGenInfo =
-               new ARMTargetCodeGenInfo(ARMABIInfo::APCS));
+               new ARMTargetCodeGenInfo(Types, ARMABIInfo::APCS));
 
     return *(TheTargetCodeGenInfo =
-             new ARMTargetCodeGenInfo(ARMABIInfo::AAPCS));
+             new ARMTargetCodeGenInfo(Types, ARMABIInfo::AAPCS));
 
   case llvm::Triple::pic16:
-    return *(TheTargetCodeGenInfo = new PIC16TargetCodeGenInfo());
+    return *(TheTargetCodeGenInfo = new PIC16TargetCodeGenInfo(Types));
 
   case llvm::Triple::ppc:
-    return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo());
+    return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo(Types));
 
   case llvm::Triple::systemz:
-    return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo());
+    return *(TheTargetCodeGenInfo = new SystemZTargetCodeGenInfo(Types));
 
   case llvm::Triple::msp430:
-    return *(TheTargetCodeGenInfo = new MSP430TargetCodeGenInfo());
+    return *(TheTargetCodeGenInfo = new MSP430TargetCodeGenInfo(Types));
 
   case llvm::Triple::x86:
     switch (Triple.getOS()) {
     case llvm::Triple::Darwin:
       return *(TheTargetCodeGenInfo =
-               new X86_32TargetCodeGenInfo(Context, true, true));
+               new X86_32TargetCodeGenInfo(Types, true, true));
     case llvm::Triple::Cygwin:
     case llvm::Triple::MinGW32:
-    case llvm::Triple::MinGW64:
     case llvm::Triple::AuroraUX:
     case llvm::Triple::DragonFly:
     case llvm::Triple::FreeBSD:
     case llvm::Triple::OpenBSD:
       return *(TheTargetCodeGenInfo =
-               new X86_32TargetCodeGenInfo(Context, false, true));
+               new X86_32TargetCodeGenInfo(Types, false, true));
 
     default:
       return *(TheTargetCodeGenInfo =
-               new X86_32TargetCodeGenInfo(Context, false, false));
+               new X86_32TargetCodeGenInfo(Types, false, false));
     }
 
   case llvm::Triple::x86_64:
-    return *(TheTargetCodeGenInfo =
-               new X86_64TargetCodeGenInfo(Context, TheTargetData));
+    switch (Triple.getOS()) {
+    case llvm::Triple::Win32:
+    case llvm::Triple::MinGW64:
+    case llvm::Triple::Cygwin:
+      return *(TheTargetCodeGenInfo = new WinX86_64TargetCodeGenInfo(Types));
+    default:
+      return *(TheTargetCodeGenInfo = new X86_64TargetCodeGenInfo(Types));
+    }
   }
 }
diff --git a/lib/CodeGen/TargetInfo.h b/lib/CodeGen/TargetInfo.h
index f0a78243ff79..9d4cf1610308 100644
--- a/lib/CodeGen/TargetInfo.h
+++ b/lib/CodeGen/TargetInfo.h
@@ -47,6 +47,16 @@ namespace clang {
     virtual void SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                                      CodeGen::CodeGenModule &M) const { }
 
+    /// Determines the size of struct _Unwind_Exception on this platform,
+    /// in 8-bit units.  The Itanium ABI defines this as:
+    ///   struct _Unwind_Exception {
+    ///     uint64 exception_class;
+    ///     _Unwind_Exception_Cleanup_Fn exception_cleanup;
+    ///     uint64 private_1;
+    ///     uint64 private_2;
+    ///   };
+    unsigned getSizeOfUnwindException() const { return 32; }
+
     /// Controls whether __builtin_extend_pointer should sign-extend
     /// pointers to uint64_t or zero-extend them (the default).  Has
     /// no effect for targets:
diff --git a/lib/Driver/Compilation.cpp b/lib/Driver/Compilation.cpp
index 282e9fe82e41..c059afd45439 100644
--- a/lib/Driver/Compilation.cpp
+++ b/lib/Driver/Compilation.cpp
@@ -83,10 +83,6 @@ void Compilation::PrintJob(llvm::raw_ostream &OS, const Job &J,
       OS << '"';
     }
     OS << Terminator;
-  } else if (const PipedJob *PJ = dyn_cast<PipedJob>(&J)) {
-    for (PipedJob::const_iterator
-           it = PJ->begin(), ie = PJ->end(); it != ie; ++it)
-      PrintJob(OS, **it, (it + 1 != PJ->end()) ? " |\n" : "\n", Quote);
   } else {
     const JobList *Jobs = cast<JobList>(&J);
     for (JobList::const_iterator
@@ -190,14 +186,6 @@ int Compilation::ExecuteJob(const Job &J,
                             const Command *&FailingCommand) const {
   if (const Command *C = dyn_cast<Command>(&J)) {
     return ExecuteCommand(*C, FailingCommand);
-  } else if (const PipedJob *PJ = dyn_cast<PipedJob>(&J)) {
-    // Piped commands with a single job are easy.
-    if (PJ->size() == 1)
-      return ExecuteCommand(**PJ->begin(), FailingCommand);
-
-    FailingCommand = *PJ->begin();
-    getDriver().Diag(clang::diag::err_drv_unsupported_opt) << "-pipe";
-    return 1;
   } else {
     const JobList *Jobs = cast<JobList>(&J);
     for (JobList::const_iterator
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
index 2fc0a5354d80..82f913484f48 100644
--- a/lib/Driver/Driver.cpp
+++ b/lib/Driver/Driver.cpp
@@ -25,6 +25,7 @@
 
 #include "clang/Basic/Version.h"
 
+#include "llvm/Config/config.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/Support/PrettyStackTrace.h"
@@ -39,13 +40,13 @@
 using namespace clang::driver;
 using namespace clang;
 
-Driver::Driver(llvm::StringRef _Name, llvm::StringRef _Dir,
+Driver::Driver(llvm::StringRef _ClangExecutable,
                llvm::StringRef _DefaultHostTriple,
                llvm::StringRef _DefaultImageName,
                bool IsProduction, bool CXXIsProduction,
                Diagnostic &_Diags)
   : Opts(createDriverOptTable()), Diags(_Diags),
-    Name(_Name), Dir(_Dir), DefaultHostTriple(_DefaultHostTriple),
+    ClangExecutable(_ClangExecutable), DefaultHostTriple(_DefaultHostTriple),
     DefaultImageName(_DefaultImageName),
     DriverTitle("clang \"gcc-compatible\" driver"),
     Host(0),
@@ -68,6 +69,10 @@ Driver::Driver(llvm::StringRef _Name, llvm::StringRef _Dir,
       CCCUseClangCXX = false;
   }
 
+  llvm::sys::Path Executable(ClangExecutable);
+  Name = Executable.getBasename();
+  Dir = Executable.getDirname();
+
   // Compute the path to the resource directory.
   llvm::sys::Path P(Dir);
   P.eraseComponent(); // Remove /bin from foo/bin
@@ -75,11 +80,6 @@ Driver::Driver(llvm::StringRef _Name, llvm::StringRef _Dir,
   P.appendComponent("clang");
   P.appendComponent(CLANG_VERSION_STRING);
   ResourceDir = P.str();
-
-  // Save the original clang executable path.
-  P = Dir;
-  P.appendComponent(Name);
-  ClangExecutable = P.str();
 }
 
 Driver::~Driver() {
@@ -160,6 +160,16 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
     DAL->append(*it);
   }
 
+  // Add a default value of -mlinker-version=, if one was given and the user
+  // didn't specify one.
+#if defined(HOST_LINK_VERSION)
+  if (!Args.hasArg(options::OPT_mlinker_version_EQ)) {
+    DAL->AddJoinedArg(0, Opts->getOption(options::OPT_mlinker_version_EQ),
+                      HOST_LINK_VERSION);
+    DAL->getLastArg(options::OPT_mlinker_version_EQ)->claim();
+  }
+#endif
+
   return DAL;
 }
 
@@ -176,13 +186,15 @@ Compilation *Driver::BuildCompilation(int argc, const char **argv) {
   bool CCCPrintOptions = false, CCCPrintActions = false;
 
   const char **Start = argv + 1, **End = argv + argc;
-  const char *HostTriple = DefaultHostTriple.c_str();
 
   InputArgList *Args = ParseArgStrings(Start, End);
 
   // -no-canonical-prefixes is used very early in main.
   Args->ClaimAllArgs(options::OPT_no_canonical_prefixes);
 
+  // Ignore -pipe.
+  Args->ClaimAllArgs(options::OPT_pipe);
+
   // Extract -ccc args.
   //
   // FIXME: We need to figure out where this behavior should live. Most of it
@@ -223,14 +235,16 @@ Compilation *Driver::BuildCompilation(int argc, const char **argv) {
       Cur = Split.second;
     }
   }
+  // FIXME: We shouldn't overwrite the default host triple here, but we have
+  // nowhere else to put this currently.
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_host_triple))
-    HostTriple = A->getValue(*Args);
+    DefaultHostTriple = A->getValue(*Args);
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_install_dir))
-    Dir = A->getValue(*Args);
+    Dir = InstalledDir = A->getValue(*Args);
   if (const Arg *A = Args->getLastArg(options::OPT_B))
     PrefixDir = A->getValue(*Args);
 
-  Host = GetHostInfo(HostTriple);
+  Host = GetHostInfo(DefaultHostTriple.c_str());
 
   // Perform the default argument translations.
   DerivedArgList *TranslatedArgs = TranslateInputArgs(*Args);
@@ -248,14 +262,12 @@ Compilation *Driver::BuildCompilation(int argc, const char **argv) {
   if (!HandleImmediateArgs(*C))
     return C;
 
-  // Construct the list of abstract actions to perform for this compilation. We
-  // avoid passing a Compilation here simply to enforce the abstraction that
-  // pipelining is not host or toolchain dependent (other than the driver driver
-  // test).
+  // Construct the list of abstract actions to perform for this compilation.
   if (Host->useDriverDriver())
-    BuildUniversalActions(C->getArgs(), C->getActions());
+    BuildUniversalActions(C->getDefaultToolChain(), C->getArgs(),
+                          C->getActions());
   else
-    BuildActions(C->getArgs(), C->getActions());
+    BuildActions(C->getDefaultToolChain(), C->getArgs(), C->getActions());
 
   if (CCCPrintActions) {
     PrintActions(*C);
@@ -525,7 +537,8 @@ static bool ContainsCompileAction(const Action *A) {
   return false;
 }
 
-void Driver::BuildUniversalActions(const ArgList &Args,
+void Driver::BuildUniversalActions(const ToolChain &TC,
+                                   const ArgList &Args,
                                    ActionList &Actions) const {
   llvm::PrettyStackTraceString CrashInfo("Building universal build actions");
   // Collect the list of architectures. Duplicates are allowed, but should only
@@ -570,7 +583,7 @@ void Driver::BuildUniversalActions(const ArgList &Args,
   }
 
   ActionList SingleActions;
-  BuildActions(Args, SingleActions);
+  BuildActions(TC, Args, SingleActions);
 
   // Add in arch bindings for every top level action, as well as lipo and
   // dsymutil steps if needed.
@@ -620,7 +633,8 @@ void Driver::BuildUniversalActions(const ArgList &Args,
   }
 }
 
-void Driver::BuildActions(const ArgList &Args, ActionList &Actions) const {
+void Driver::BuildActions(const ToolChain &TC, const ArgList &Args,
+                          ActionList &Actions) const {
   llvm::PrettyStackTraceString CrashInfo("Building compilation actions");
   // Start by constructing the list of inputs and their types.
 
@@ -660,7 +674,7 @@ void Driver::BuildActions(const ArgList &Args, ActionList &Actions) const {
           // found. We use a host hook here because Darwin at least has its own
           // idea of what .s is.
           if (const char *Ext = strrchr(Value, '.'))
-            Ty = Host->lookupTypeForExtension(Ext + 1);
+            Ty = TC.LookupTypeForExtension(Ext + 1);
 
           if (Ty == types::TY_INVALID)
             Ty = types::TY_Object;
@@ -885,16 +899,6 @@ Action *Driver::ConstructPhaseAction(const ArgList &Args, phases::ID Phase,
 
 void Driver::BuildJobs(Compilation &C) const {
   llvm::PrettyStackTraceString CrashInfo("Building compilation jobs");
-  bool SaveTemps = C.getArgs().hasArg(options::OPT_save_temps);
-  bool UsePipes = C.getArgs().hasArg(options::OPT_pipe);
-
-  // FIXME: Pipes are forcibly disabled until we support executing them.
-  if (!CCCPrintBindings)
-    UsePipes = false;
-
-  // -save-temps inhibits pipes.
-  if (SaveTemps && UsePipes)
-    Diag(clang::diag::warn_drv_pipe_ignored_with_save_temps);
 
   Arg *FinalOutput = C.getArgs().getLastArg(options::OPT_o);
 
@@ -934,7 +938,6 @@ void Driver::BuildJobs(Compilation &C) const {
     InputInfo II;
     BuildJobsForAction(C, A, &C.getDefaultToolChain(),
                        /*BoundArch*/0,
-                       /*CanAcceptPipe*/ true,
                        /*AtTopLevel*/ true,
                        /*LinkingOutput*/ LinkingOutput,
                        II);
@@ -1032,17 +1035,11 @@ void Driver::BuildJobsForAction(Compilation &C,
                                 const Action *A,
                                 const ToolChain *TC,
                                 const char *BoundArch,
-                                bool CanAcceptPipe,
                                 bool AtTopLevel,
                                 const char *LinkingOutput,
                                 InputInfo &Result) const {
   llvm::PrettyStackTraceString CrashInfo("Building compilation jobs");
 
-  bool UsePipes = C.getArgs().hasArg(options::OPT_pipe);
-  // FIXME: Pipes are forcibly disabled until we support executing them.
-  if (!CCCPrintBindings)
-    UsePipes = false;
-
   if (const InputAction *IA = dyn_cast<InputAction>(A)) {
     // FIXME: It would be nice to not claim this here; maybe the old scheme of
     // just using Args was better?
@@ -1064,7 +1061,7 @@ void Driver::BuildJobsForAction(Compilation &C,
       TC = Host->CreateToolChain(C.getArgs(), BAA->getArchName());
 
     BuildJobsForAction(C, *BAA->begin(), TC, BAA->getArchName(),
-                       CanAcceptPipe, AtTopLevel, LinkingOutput, Result);
+                       AtTopLevel, LinkingOutput, Result);
     return;
   }
 
@@ -1074,7 +1071,6 @@ void Driver::BuildJobsForAction(Compilation &C,
   const Tool &T = SelectToolForJob(C, TC, JA, Inputs);
 
   // Only use pipes when there is exactly one input.
-  bool TryToUsePipeInput = Inputs->size() == 1 && T.acceptsPipedInput();
   InputInfoList InputInfos;
   for (ActionList::const_iterator it = Inputs->begin(), ie = Inputs->end();
        it != ie; ++it) {
@@ -1087,33 +1083,11 @@ void Driver::BuildJobsForAction(Compilation &C,
       SubJobAtTopLevel = true;
 
     InputInfo II;
-    BuildJobsForAction(C, *it, TC, BoundArch, TryToUsePipeInput,
+    BuildJobsForAction(C, *it, TC, BoundArch,
                        SubJobAtTopLevel, LinkingOutput, II);
     InputInfos.push_back(II);
   }
 
-  // Determine if we should output to a pipe.
-  bool OutputToPipe = false;
-  if (CanAcceptPipe && T.canPipeOutput()) {
-    // Some actions default to writing to a pipe if they are the top level phase
-    // and there was no user override.
-    //
-    // FIXME: Is there a better way to handle this?
-    if (AtTopLevel) {
-      if (isa<PreprocessJobAction>(A) && !C.getArgs().hasArg(options::OPT_o))
-        OutputToPipe = true;
-    } else if (UsePipes)
-      OutputToPipe = true;
-  }
-
-  // Figure out where to put the job (pipes).
-  Job *Dest = &C.getJobs();
-  if (InputInfos[0].isPipe()) {
-    assert(TryToUsePipeInput && "Unrequested pipe!");
-    assert(InputInfos.size() == 1 && "Unexpected pipe with multiple inputs.");
-    Dest = &InputInfos[0].getPipe();
-  }
-
   // Always use the first input as the base input.
   const char *BaseInput = InputInfos[0].getBaseInput();
 
@@ -1122,22 +1096,9 @@ void Driver::BuildJobsForAction(Compilation &C,
   if (JA->getType() == types::TY_dSYM)
     BaseInput = InputInfos[0].getFilename();
 
-  // Determine the place to write output to (nothing, pipe, or filename) and
-  // where to put the new job.
+  // Determine the place to write output to, if any.
   if (JA->getType() == types::TY_Nothing) {
     Result = InputInfo(A->getType(), BaseInput);
-  } else if (OutputToPipe) {
-    // Append to current piped job or create a new one as appropriate.
-    PipedJob *PJ = dyn_cast<PipedJob>(Dest);
-    if (!PJ) {
-      PJ = new PipedJob();
-      // FIXME: Temporary hack so that -ccc-print-bindings work until we have
-      // pipe support. Please remove later.
-      if (!CCCPrintBindings)
-        cast<JobList>(Dest)->addJob(PJ);
-      Dest = PJ;
-    }
-    Result = InputInfo(PJ, A->getType(), BaseInput);
   } else {
     Result = InputInfo(GetNamedOutputPath(C, *JA, BaseInput, AtTopLevel),
                        A->getType(), BaseInput);
@@ -1153,7 +1114,7 @@ void Driver::BuildJobsForAction(Compilation &C,
     }
     llvm::errs() << "], output: " << Result.getAsString() << "\n";
   } else {
-    T.ConstructJob(C, *JA, *Dest, Result, InputInfos,
+    T.ConstructJob(C, *JA, Result, InputInfos,
                    C.getArgsForToolChain(TC, BoundArch), LinkingOutput);
   }
 }
@@ -1169,6 +1130,10 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
       return C.addResultFile(FinalOutput->getValue(C.getArgs()));
   }
 
+  // Default to writing to stdout?
+  if (AtTopLevel && isa<PreprocessJobAction>(JA))
+    return "-";
+
   // Output to a temporary file?
   if (!AtTopLevel && !C.getArgs().hasArg(options::OPT_save_temps)) {
     std::string TmpName =
@@ -1307,6 +1272,11 @@ const HostInfo *Driver::GetHostInfo(const char *TripleStr) const {
     return createMinixHostInfo(*this, Triple);
   case llvm::Triple::Linux:
     return createLinuxHostInfo(*this, Triple);
+  case llvm::Triple::Win32:
+    return createWindowsHostInfo(*this, Triple);
+  case llvm::Triple::MinGW32:
+  case llvm::Triple::MinGW64:
+    return createMinGWHostInfo(*this, Triple);
   default:
     return createUnknownHostInfo(*this, Triple);
   }
diff --git a/lib/Driver/HostInfo.cpp b/lib/Driver/HostInfo.cpp
index 0636d9eab15c..7c5e430bb7d5 100644
--- a/lib/Driver/HostInfo.cpp
+++ b/lib/Driver/HostInfo.cpp
@@ -38,12 +38,6 @@ namespace {
 
 /// DarwinHostInfo - Darwin host information implementation.
 class DarwinHostInfo : public HostInfo {
-  /// Darwin version of host.
-  unsigned DarwinVersion[3];
-
-  /// GCC version to use on this host.
-  unsigned GCCVersion[3];
-
   /// Cache of tool chains we have created.
   mutable llvm::DenseMap<unsigned, ToolChain*> ToolChains;
 
@@ -53,37 +47,12 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    types::ID Ty = types::lookupTypeForExtension(Ext);
-
-    // Darwin always preprocesses assembly files (unless -x is used
-    // explicitly).
-    if (Ty == types::TY_PP_Asm)
-      return types::TY_Asm;
-
-    return Ty;
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
 
 DarwinHostInfo::DarwinHostInfo(const Driver &D, const llvm::Triple& Triple)
   : HostInfo(D, Triple) {
-
-  assert(Triple.getArch() != llvm::Triple::UnknownArch && "Invalid arch!");
-  assert(memcmp(&getOSName()[0], "darwin", 6) == 0 &&
-         "Unknown Darwin platform.");
-  bool HadExtra;
-  if (!Driver::GetReleaseVersion(&getOSName()[6],
-                                 DarwinVersion[0], DarwinVersion[1],
-                                 DarwinVersion[2], HadExtra))
-    D.Diag(clang::diag::err_drv_invalid_darwin_version) << getOSName();
-
-  // We can only call 4.2.1 for now.
-  GCCVersion[0] = 4;
-  GCCVersion[1] = 2;
-  GCCVersion[2] = 1;
 }
 
 DarwinHostInfo::~DarwinHostInfo() {
@@ -147,11 +116,10 @@ ToolChain *DarwinHostInfo::CreateToolChain(const ArgList &Args,
     const char *UseNewToolChain = ::getenv("CCC_ENABLE_NEW_DARWIN_TOOLCHAIN");
     if (UseNewToolChain || 
         Arch == llvm::Triple::arm || Arch == llvm::Triple::thumb) {
-      TC = new toolchains::DarwinClang(*this, TCTriple, DarwinVersion);
+      TC = new toolchains::DarwinClang(*this, TCTriple);
     } else if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64) {
       // We still use the legacy DarwinGCC toolchain on X86.
-      TC = new toolchains::DarwinGCC(*this, TCTriple, DarwinVersion,
-                                     GCCVersion);
+      TC = new toolchains::DarwinGCC(*this, TCTriple);
     } else
       TC = new toolchains::Darwin_Generic_GCC(*this, TCTriple);
   }
@@ -170,15 +138,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    types::ID Ty = types::lookupTypeForExtension(Ext);
-
-    if (Ty == types::TY_PP_Asm)
-      return types::TY_Asm;
-
-    return Ty;
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args, 
                                      const char *ArchName) const;
 };
@@ -212,10 +171,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -279,10 +234,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -330,10 +281,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -379,10 +326,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -399,19 +342,22 @@ bool FreeBSDHostInfo::useDriverDriver() const {
 
 ToolChain *FreeBSDHostInfo::CreateToolChain(const ArgList &Args,
                                             const char *ArchName) const {
-  bool Lib32 = false;
-
   assert(!ArchName &&
          "Unexpected arch name on platform without driver driver support.");
 
-  // On x86_64 we need to be able to compile 32-bits binaries as well.
-  // Compiling 64-bit binaries on i386 is not supported. We don't have a
-  // lib64.
+  // Automatically handle some instances of -m32/-m64 we know about.
   std::string Arch = getArchName();
   ArchName = Arch.c_str();
-  if (Args.hasArg(options::OPT_m32) && getArchName() == "x86_64") {
-    ArchName = "i386";
-    Lib32 = true;
+  if (Arg *A = Args.getLastArg(options::OPT_m32, options::OPT_m64)) {
+    if (Triple.getArch() == llvm::Triple::x86 ||
+        Triple.getArch() == llvm::Triple::x86_64) {
+      ArchName =
+        (A->getOption().matches(options::OPT_m32)) ? "i386" : "x86_64";
+    } else if (Triple.getArch() == llvm::Triple::ppc ||
+               Triple.getArch() == llvm::Triple::ppc64) {
+      ArchName =
+        (A->getOption().matches(options::OPT_m32)) ? "powerpc" : "powerpc64";
+    }
   }
 
   ToolChain *&TC = ToolChains[ArchName];
@@ -419,7 +365,7 @@ ToolChain *FreeBSDHostInfo::CreateToolChain(const ArgList &Args,
     llvm::Triple TCTriple(getTriple());
     TCTriple.setArchName(ArchName);
 
-    TC = new toolchains::FreeBSD(*this, TCTriple, Lib32);
+    TC = new toolchains::FreeBSD(*this, TCTriple);
   }
 
   return TC;
@@ -439,10 +385,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -491,10 +433,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -540,10 +478,6 @@ public:
 
   virtual bool useDriverDriver() const;
 
-  virtual types::ID lookupTypeForExtension(const char *Ext) const {
-    return types::lookupTypeForExtension(Ext);
-  }
-
   virtual ToolChain *CreateToolChain(const ArgList &Args,
                                      const char *ArchName) const;
 };
@@ -591,8 +525,79 @@ ToolChain *LinuxHostInfo::CreateToolChain(const ArgList &Args,
   return TC;
 }
 
+// Windows Host Info
+
+/// WindowsHostInfo - Host information to use on Microsoft Windows.
+class WindowsHostInfo : public HostInfo {
+  /// Cache of tool chains we have created.
+  mutable llvm::StringMap<ToolChain*> ToolChains;
+
+public:
+  WindowsHostInfo(const Driver &D, const llvm::Triple& Triple);
+  ~WindowsHostInfo();
+
+  virtual bool useDriverDriver() const;
+
+  virtual types::ID lookupTypeForExtension(const char *Ext) const {
+    return types::lookupTypeForExtension(Ext);
+  }
+
+  virtual ToolChain *CreateToolChain(const ArgList &Args,
+                                     const char *ArchName) const;
+};
+
+WindowsHostInfo::WindowsHostInfo(const Driver &D, const llvm::Triple& Triple)
+  : HostInfo(D, Triple) {
+}
+
+WindowsHostInfo::~WindowsHostInfo() {
+  for (llvm::StringMap<ToolChain*>::iterator
+         it = ToolChains.begin(), ie = ToolChains.end(); it != ie; ++it)
+    delete it->second;
+}
+
+bool WindowsHostInfo::useDriverDriver() const {
+  return false;
+}
+
+ToolChain *WindowsHostInfo::CreateToolChain(const ArgList &Args,
+                                            const char *ArchName) const {
+  assert(!ArchName &&
+         "Unexpected arch name on platform without driver driver support.");
+
+  // Automatically handle some instances of -m32/-m64 we know about.
+  std::string Arch = getArchName();
+  ArchName = Arch.c_str();
+  if (Arg *A = Args.getLastArg(options::OPT_m32, options::OPT_m64)) {
+    if (Triple.getArch() == llvm::Triple::x86 ||
+        Triple.getArch() == llvm::Triple::x86_64) {
+      ArchName =
+        (A->getOption().matches(options::OPT_m32)) ? "i386" : "x86_64";
+    }
+  }
+
+  ToolChain *&TC = ToolChains[ArchName];
+  if (!TC) {
+    llvm::Triple TCTriple(getTriple());
+    TCTriple.setArchName(ArchName);
+
+    TC = new toolchains::Windows(*this, TCTriple);
+  }
+
+  return TC;
 }
 
+// FIXME: This is a placeholder.
+class MinGWHostInfo : public UnknownHostInfo {
+public:
+  MinGWHostInfo(const Driver &D, const llvm::Triple& Triple);
+};
+
+MinGWHostInfo::MinGWHostInfo(const Driver &D, const llvm::Triple& Triple)
+  : UnknownHostInfo(D, Triple) {}
+
+} // end anon namespace
+
 const HostInfo *
 clang::driver::createAuroraUXHostInfo(const Driver &D,
                                       const llvm::Triple& Triple){
@@ -642,6 +647,18 @@ clang::driver::createTCEHostInfo(const Driver &D,
 }
 
 const HostInfo *
+clang::driver::createWindowsHostInfo(const Driver &D,
+                                     const llvm::Triple& Triple) {
+  return new WindowsHostInfo(D, Triple);
+}
+
+const HostInfo *
+clang::driver::createMinGWHostInfo(const Driver &D,
+                                   const llvm::Triple& Triple) {
+  return new MinGWHostInfo(D, Triple);
+}
+
+const HostInfo *
 clang::driver::createUnknownHostInfo(const Driver &D,
                                      const llvm::Triple& Triple) {
   return new UnknownHostInfo(D, Triple);
diff --git a/lib/Driver/InputInfo.h b/lib/Driver/InputInfo.h
index c657bef95af0..2a2f4b995d2a 100644
--- a/lib/Driver/InputInfo.h
+++ b/lib/Driver/InputInfo.h
@@ -17,7 +17,6 @@
 
 namespace clang {
 namespace driver {
-  class PipedJob;
 
 /// InputInfo - Wrapper for information about an input source.
 class InputInfo {
@@ -37,7 +36,6 @@ class InputInfo {
   union {
     const char *Filename;
     const Arg *InputArg;
-    PipedJob *Pipe;
   } Data;
   Class Kind;
   types::ID Type;
@@ -56,15 +54,10 @@ public:
     : Kind(InputArg), Type(_Type), BaseInput(_BaseInput) {
     Data.InputArg = _InputArg;
   }
-  InputInfo(PipedJob *_Pipe, types::ID _Type, const char *_BaseInput)
-    : Kind(Pipe), Type(_Type), BaseInput(_BaseInput) {
-    Data.Pipe = _Pipe;
-  }
 
   bool isNothing() const { return Kind == Nothing; }
   bool isFilename() const { return Kind == Filename; }
   bool isInputArg() const { return Kind == InputArg; }
-  bool isPipe() const { return Kind == Pipe; }
   types::ID getType() const { return Type; }
   const char *getBaseInput() const { return BaseInput; }
 
@@ -76,17 +69,11 @@ public:
     assert(isInputArg() && "Invalid accessor.");
     return *Data.InputArg;
   }
-  PipedJob &getPipe() const {
-    assert(isPipe() && "Invalid accessor.");
-    return *Data.Pipe;
-  }
 
   /// getAsString - Return a string name for this input, for
   /// debugging.
   std::string getAsString() const {
-    if (isPipe())
-      return "(pipe)";
-    else if (isFilename())
+    if (isFilename())
       return std::string("\"") + getFilename() + '"';
     else if (isInputArg())
       return "(input arg)";
diff --git a/lib/Driver/Job.cpp b/lib/Driver/Job.cpp
index bfeb41a94299..fa7d0607728e 100644
--- a/lib/Driver/Job.cpp
+++ b/lib/Driver/Job.cpp
@@ -21,13 +21,6 @@ Command::Command(const Action &_Source, const Tool &_Creator,
 {
 }
 
-PipedJob::PipedJob() : Job(PipedJobClass) {}
-
-PipedJob::~PipedJob() {
-  for (iterator it = begin(), ie = end(); it != ie; ++it)
-    delete *it;
-}
-
 JobList::JobList() : Job(JobListClass) {}
 
 JobList::~JobList() {
@@ -36,9 +29,6 @@ JobList::~JobList() {
 }
 
 void Job::addCommand(Command *C) {
-  if (PipedJob *PJ = dyn_cast<PipedJob>(this))
-    PJ->addCommand(C);
-  else
-    cast<JobList>(this)->addJob(C);
+  cast<JobList>(this)->addJob(C);
 }
 
diff --git a/lib/Driver/Makefile b/lib/Driver/Makefile
index 7bc340e109d3..454ab8626c2c 100644
--- a/lib/Driver/Makefile
+++ b/lib/Driver/Makefile
@@ -9,6 +9,5 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangDriver
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
diff --git a/lib/Driver/OptTable.cpp b/lib/Driver/OptTable.cpp
index 39530f211d80..3c363142d7fd 100644
--- a/lib/Driver/OptTable.cpp
+++ b/lib/Driver/OptTable.cpp
@@ -164,6 +164,8 @@ Option *OptTable::CreateOption(unsigned id) const {
     Opt->setLinkerInput(true);
   if (info.Flags & NoArgumentUnused)
     Opt->setNoArgumentUnused(true);
+  if (info.Flags & NoForward)
+    Opt->setNoForward(true);
   if (info.Flags & RenderAsInput)
     Opt->setNoOptAsInput(true);
   if (info.Flags & RenderJoined) {
diff --git a/lib/Driver/Option.cpp b/lib/Driver/Option.cpp
index dd48af8018ac..5396250dfae1 100644
--- a/lib/Driver/Option.cpp
+++ b/lib/Driver/Option.cpp
@@ -20,7 +20,7 @@ Option::Option(OptionClass _Kind, OptSpecifier _ID, const char *_Name,
                const OptionGroup *_Group, const Option *_Alias)
   : Kind(_Kind), ID(_ID.getID()), Name(_Name), Group(_Group), Alias(_Alias),
     Unsupported(false), LinkerInput(false), NoOptAsInput(false),
-    DriverOption(false), NoArgumentUnused(false) {
+    DriverOption(false), NoArgumentUnused(false), NoForward(false) {
 
   // Multi-level aliases are not supported, and alias options cannot
   // have groups. This just simplifies option tracking, it is not an
diff --git a/lib/Driver/ToolChain.cpp b/lib/Driver/ToolChain.cpp
index 9fae67de32f4..94c1c6beb99f 100644
--- a/lib/Driver/ToolChain.cpp
+++ b/lib/Driver/ToolChain.cpp
@@ -10,8 +10,12 @@
 #include "clang/Driver/ToolChain.h"
 
 #include "clang/Driver/Action.h"
+#include "clang/Driver/Arg.h"
+#include "clang/Driver/ArgList.h"
 #include "clang/Driver/Driver.h"
+#include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/HostInfo.h"
+#include "clang/Driver/Options.h"
 
 using namespace clang::driver;
 
@@ -34,3 +38,139 @@ std::string ToolChain::GetFilePath(const char *Name) const {
 std::string ToolChain::GetProgramPath(const char *Name, bool WantFile) const {
   return Host.getDriver().GetProgramPath(Name, *this, WantFile);
 }
+
+types::ID ToolChain::LookupTypeForExtension(const char *Ext) const {
+  return types::lookupTypeForExtension(Ext);
+}
+
+/// getARMTargetCPU - Get the (LLVM) name of the ARM cpu we are targetting.
+//
+// FIXME: tblgen this.
+static const char *getARMTargetCPU(const ArgList &Args,
+                                   const llvm::Triple &Triple) {
+  // FIXME: Warn on inconsistent use of -mcpu and -march.
+
+  // If we have -mcpu=, use that.
+  if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
+    return A->getValue(Args);
+
+  llvm::StringRef MArch;
+  if (Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
+    // Otherwise, if we have -march= choose the base CPU for that arch.
+    MArch = A->getValue(Args);
+  } else {
+    // Otherwise, use the Arch from the triple.
+    MArch = Triple.getArchName();
+  }
+
+  if (MArch == "armv2" || MArch == "armv2a")
+    return "arm2";
+  if (MArch == "armv3")
+    return "arm6";
+  if (MArch == "armv3m")
+    return "arm7m";
+  if (MArch == "armv4" || MArch == "armv4t")
+    return "arm7tdmi";
+  if (MArch == "armv5" || MArch == "armv5t")
+    return "arm10tdmi";
+  if (MArch == "armv5e" || MArch == "armv5te")
+    return "arm1026ejs";
+  if (MArch == "armv5tej")
+    return "arm926ej-s";
+  if (MArch == "armv6" || MArch == "armv6k")
+    return "arm1136jf-s";
+  if (MArch == "armv6j")
+    return "arm1136j-s";
+  if (MArch == "armv6z" || MArch == "armv6zk")
+    return "arm1176jzf-s";
+  if (MArch == "armv6t2")
+    return "arm1156t2-s";
+  if (MArch == "armv7" || MArch == "armv7a" || MArch == "armv7-a")
+    return "cortex-a8";
+  if (MArch == "armv7r" || MArch == "armv7-r")
+    return "cortex-r4";
+  if (MArch == "armv7m" || MArch == "armv7-m")
+    return "cortex-m3";
+  if (MArch == "ep9312")
+    return "ep9312";
+  if (MArch == "iwmmxt")
+    return "iwmmxt";
+  if (MArch == "xscale")
+    return "xscale";
+
+  // If all else failed, return the most base CPU LLVM supports.
+  return "arm7tdmi";
+}
+
+/// getLLVMArchSuffixForARM - Get the LLVM arch name to use for a particular
+/// CPU.
+//
+// FIXME: This is redundant with -mcpu, why does LLVM use this.
+// FIXME: tblgen this, or kill it!
+static const char *getLLVMArchSuffixForARM(llvm::StringRef CPU) {
+  if (CPU == "arm7tdmi" || CPU == "arm7tdmi-s" || CPU == "arm710t" ||
+      CPU == "arm720t" || CPU == "arm9" || CPU == "arm9tdmi" ||
+      CPU == "arm920" || CPU == "arm920t" || CPU == "arm922t" ||
+      CPU == "arm940t" || CPU == "ep9312")
+    return "v4t";
+
+  if (CPU == "arm10tdmi" || CPU == "arm1020t")
+    return "v5";
+
+  if (CPU == "arm9e" || CPU == "arm926ej-s" || CPU == "arm946e-s" ||
+      CPU == "arm966e-s" || CPU == "arm968e-s" || CPU == "arm10e" ||
+      CPU == "arm1020e" || CPU == "arm1022e" || CPU == "xscale" ||
+      CPU == "iwmmxt")
+    return "v5e";
+
+  if (CPU == "arm1136j-s" || CPU == "arm1136jf-s" || CPU == "arm1176jz-s" ||
+      CPU == "arm1176jzf-s" || CPU == "mpcorenovfp" || CPU == "mpcore")
+    return "v6";
+
+  if (CPU == "arm1156t2-s" || CPU == "arm1156t2f-s")
+    return "v6t2";
+
+  if (CPU == "cortex-a8" || CPU == "cortex-a9")
+    return "v7";
+
+  return "";
+}
+
+std::string ToolChain::ComputeLLVMTriple(const ArgList &Args) const {
+  switch (getTriple().getArch()) {
+  default:
+    return getTripleString();
+
+  case llvm::Triple::arm:
+  case llvm::Triple::thumb: {
+    // FIXME: Factor into subclasses.
+    llvm::Triple Triple = getTriple();
+
+    // Thumb2 is the default for V7 on Darwin.
+    //
+    // FIXME: Thumb should just be another -target-feaure, not in the triple.
+    llvm::StringRef Suffix =
+      getLLVMArchSuffixForARM(getARMTargetCPU(Args, Triple));
+    bool ThumbDefault =
+      (Suffix == "v7" && getTriple().getOS() == llvm::Triple::Darwin);
+    std::string ArchName = "arm";
+    if (Args.hasFlag(options::OPT_mthumb, options::OPT_mno_thumb, ThumbDefault))
+      ArchName = "thumb";
+    Triple.setArchName(ArchName + Suffix.str());
+
+    return Triple.getTriple();
+  }
+  }
+}
+
+std::string ToolChain::ComputeEffectiveClangTriple(const ArgList &Args) const {
+  // Diagnose use of -mmacosx-version-min and -miphoneos-version-min on
+  // non-Darwin.
+  if (Arg *A = Args.getLastArg(options::OPT_mmacosx_version_min_EQ,
+                               options::OPT_miphoneos_version_min_EQ))
+    getDriver().Diag(clang::diag::err_drv_clang_unsupported)
+      << A->getAsString(Args);
+
+  return ComputeLLVMTriple(Args);
+}
+
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
index a78d1536fa1e..471c47dd2b50 100644
--- a/lib/Driver/ToolChains.cpp
+++ b/lib/Driver/ToolChains.cpp
@@ -18,7 +18,9 @@
 #include "clang/Driver/OptTable.h"
 #include "clang/Driver/Option.h"
 #include "clang/Driver/Options.h"
+#include "clang/Basic/Version.h"
 
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -31,13 +33,30 @@ using namespace clang::driver::toolchains;
 
 /// Darwin - Darwin tool chain for i386 and x86_64.
 
-Darwin::Darwin(const HostInfo &Host, const llvm::Triple& Triple,
-               const unsigned (&_DarwinVersion)[3])
+Darwin::Darwin(const HostInfo &Host, const llvm::Triple& Triple)
   : ToolChain(Host, Triple), TargetInitialized(false)
 {
+  // Compute the initial Darwin version based on the host.
+  bool HadExtra;
+  std::string OSName = Triple.getOSName();
+  if (!Driver::GetReleaseVersion(&OSName[6],
+                                 DarwinVersion[0], DarwinVersion[1],
+                                 DarwinVersion[2], HadExtra))
+    getDriver().Diag(clang::diag::err_drv_invalid_darwin_version) << OSName;
+
   llvm::raw_string_ostream(MacosxVersionMin)
-    << "10." << std::max(0, (int)_DarwinVersion[0] - 4) << '.'
-    << _DarwinVersion[1];
+    << "10." << std::max(0, (int)DarwinVersion[0] - 4) << '.'
+    << DarwinVersion[1];
+}
+
+types::ID Darwin::LookupTypeForExtension(const char *Ext) const {
+  types::ID Ty = types::lookupTypeForExtension(Ext);
+
+  // Darwin always preprocesses assembly files (unless -x is used explicitly).
+  if (Ty == types::TY_PP_Asm)
+    return types::TY_Asm;
+
+  return Ty;
 }
 
 // FIXME: Can we tablegen this?
@@ -103,14 +122,13 @@ llvm::StringRef Darwin::getDarwinArchName(const ArgList &Args) const {
   }
 }
 
-DarwinGCC::DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple,
-                     const unsigned (&DarwinVersion)[3],
-                     const unsigned (&_GCCVersion)[3])
-  : Darwin(Host, Triple, DarwinVersion)
+DarwinGCC::DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple)
+  : Darwin(Host, Triple)
 {
-  GCCVersion[0] = _GCCVersion[0];
-  GCCVersion[1] = _GCCVersion[1];
-  GCCVersion[2] = _GCCVersion[2];
+  // We can only work with 4.2.1 currently.
+  GCCVersion[0] = 4;
+  GCCVersion[1] = 2;
+  GCCVersion[2] = 1;
 
   // Set up the tool chain paths to match gcc.
   ToolChainDir = "i686-apple-darwin";
@@ -174,7 +192,9 @@ DarwinGCC::DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple,
   Path += ToolChainDir;
   getProgramPaths().push_back(Path);
 
-  getProgramPaths().push_back(getDriver().Dir);
+  getProgramPaths().push_back(getDriver().getInstalledDir());
+  if (getDriver().getInstalledDir() != getDriver().Dir)
+    getProgramPaths().push_back(getDriver().Dir);
 }
 
 Darwin::~Darwin() {
@@ -184,6 +204,38 @@ Darwin::~Darwin() {
     delete it->second;
 }
 
+std::string Darwin::ComputeEffectiveClangTriple(const ArgList &Args) const {
+  llvm::Triple Triple(ComputeLLVMTriple(Args));
+
+  // If the target isn't initialized (e.g., an unknown Darwin platform, return
+  // the default triple).
+  if (!isTargetInitialized())
+    return Triple.getTriple();
+    
+  unsigned Version[3];
+  getTargetVersion(Version);
+
+  // Mangle the target version into the OS triple component.  For historical
+  // reasons that make little sense, the version passed here is the "darwin"
+  // version, which drops the 10 and offsets by 4. See inverse code when
+  // setting the OS version preprocessor define.
+  if (!isTargetIPhoneOS()) {
+    Version[0] = Version[1] + 4;
+    Version[1] = Version[2];
+    Version[2] = 0;
+  } else {
+    // Use the environment to communicate that we are targetting iPhoneOS.
+    Triple.setEnvironmentName("iphoneos");
+  }
+
+  llvm::SmallString<16> Str;
+  llvm::raw_svector_ostream(Str) << "darwin" << Version[0]
+                                 << "." << Version[1] << "." << Version[2];
+  Triple.setOSName(Str.str());
+
+  return Triple.getTriple();
+}
+
 Tool &Darwin::SelectTool(const Compilation &C, const JobAction &JA) const {
   Action::ActionClass Key;
   if (getDriver().ShouldUseClangCompiler(C, JA, getTriple()))
@@ -245,7 +297,7 @@ void DarwinGCC::AddLinkSearchPathArgs(const ArgList &Args,
     CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/gcc/" + ToolChainDir +
                                          "/x86_64"));
   }
-  
+
   CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/" + ToolChainDir));
 
   Tmp = getDriver().Dir + "/../lib/gcc/" + ToolChainDir;
@@ -314,12 +366,13 @@ void DarwinGCC::AddLinkRuntimeLibArgs(const ArgList &Args,
   }
 }
 
-DarwinClang::DarwinClang(const HostInfo &Host, const llvm::Triple& Triple,
-                         const unsigned (&DarwinVersion)[3])
-  : Darwin(Host, Triple, DarwinVersion)
+DarwinClang::DarwinClang(const HostInfo &Host, const llvm::Triple& Triple)
+  : Darwin(Host, Triple)
 {
   // We expect 'as', 'ld', etc. to be adjacent to our install dir.
-  getProgramPaths().push_back(getDriver().Dir);
+  getProgramPaths().push_back(getDriver().getInstalledDir());
+  if (getDriver().getInstalledDir() != getDriver().Dir)
+    getProgramPaths().push_back(getDriver().Dir);
 }
 
 void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
@@ -350,6 +403,39 @@ void DarwinClang::AddLinkSearchPathArgs(const ArgList &Args,
     break;
   }
   P.appendComponent("4.2.1");
+
+  // Determine the arch specific GCC subdirectory.
+  const char *ArchSpecificDir = 0;
+  switch (getTriple().getArch()) {
+  default:
+    break;
+  case llvm::Triple::arm:
+  case llvm::Triple::thumb: {
+    std::string Triple = ComputeLLVMTriple(Args);
+    llvm::StringRef TripleStr = Triple;
+    if (TripleStr.startswith("armv5") || TripleStr.startswith("thumbv5"))
+      ArchSpecificDir = "v5";
+    else if (TripleStr.startswith("armv6") || TripleStr.startswith("thumbv6"))
+      ArchSpecificDir = "v6";
+    else if (TripleStr.startswith("armv7") || TripleStr.startswith("thumbv7"))
+      ArchSpecificDir = "v7";
+    break;
+  }
+  case llvm::Triple::ppc64:
+    ArchSpecificDir = "ppc64";
+    break;
+  case llvm::Triple::x86_64:
+    ArchSpecificDir = "x86_64";
+    break;
+  }
+
+  if (ArchSpecificDir) {
+    P.appendComponent(ArchSpecificDir);
+    if (P.exists())
+      CmdArgs.push_back(Args.MakeArgString("-L" + P.str()));
+    P.eraseComponent();
+  }
+
   if (P.exists())
     CmdArgs.push_back(Args.MakeArgString("-L" + P.str()));
 }
@@ -415,18 +501,9 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
   }
 }
 
-DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
-                                      const char *BoundArch) const {
-  DerivedArgList *DAL = new DerivedArgList(Args.getBaseArgs());
+void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   const OptTable &Opts = getDriver().getOpts();
 
-  // FIXME: We really want to get out of the tool chain level argument
-  // translation business, as it makes the driver functionality much
-  // more opaque. For now, we follow gcc closely solely for the
-  // purpose of easily achieving feature parity & testability. Once we
-  // have something that works, we should reevaluate each translation
-  // and try to push it down into tool specific logic.
-
   Arg *OSXVersion = Args.getLastArg(options::OPT_mmacosx_version_min_EQ);
   Arg *iPhoneVersion = Args.getLastArg(options::OPT_miphoneos_version_min_EQ);
   if (OSXVersion && iPhoneVersion) {
@@ -462,17 +539,17 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
 
     if (OSXTarget) {
       const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
-      OSXVersion = DAL->MakeJoinedArg(0, O, OSXTarget);
-      DAL->append(OSXVersion);
+      OSXVersion = Args.MakeJoinedArg(0, O, OSXTarget);
+      Args.append(OSXVersion);
     } else if (iPhoneOSTarget) {
       const Option *O = Opts.getOption(options::OPT_miphoneos_version_min_EQ);
-      iPhoneVersion = DAL->MakeJoinedArg(0, O, iPhoneOSTarget);
-      DAL->append(iPhoneVersion);
+      iPhoneVersion = Args.MakeJoinedArg(0, O, iPhoneOSTarget);
+      Args.append(iPhoneVersion);
     } else {
       // Otherwise, assume we are targeting OS X.
       const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
-      OSXVersion = DAL->MakeJoinedArg(0, O, MacosxVersionMin);
-      DAL->append(OSXVersion);
+      OSXVersion = Args.MakeJoinedArg(0, O, MacosxVersionMin);
+      Args.append(OSXVersion);
     }
   }
 
@@ -495,6 +572,19 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
         << iPhoneVersion->getAsString(Args);
   }
   setTarget(iPhoneVersion, Major, Minor, Micro);
+}
+
+DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
+                                      const char *BoundArch) const {
+  DerivedArgList *DAL = new DerivedArgList(Args.getBaseArgs());
+  const OptTable &Opts = getDriver().getOpts();
+
+  // FIXME: We really want to get out of the tool chain level argument
+  // translation business, as it makes the driver functionality much
+  // more opaque. For now, we follow gcc closely solely for the
+  // purpose of easily achieving feature parity & testability. Once we
+  // have something that works, we should reevaluate each translation
+  // and try to push it down into tool specific logic.
 
   for (ArgList::const_iterator it = Args.begin(),
          ie = Args.end(); it != ie; ++it) {
@@ -673,6 +763,11 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       llvm_unreachable("invalid Darwin arch");
   }
 
+  // Add an explicit version min argument for the deployment target. We do this
+  // after argument translation because -Xarch_ arguments may add a version min
+  // argument.
+  AddDeploymentTarget(*DAL);
+
   return DAL;
 }
 
@@ -709,13 +804,20 @@ bool Darwin::SupportsObjCGC() const {
   return !isTargetIPhoneOS();
 }
 
+std::string
+Darwin_Generic_GCC::ComputeEffectiveClangTriple(const ArgList &Args) const {
+  return ComputeLLVMTriple(Args);
+}
+
 /// Generic_GCC - A tool chain using the 'gcc' command to perform
 /// all subcommands; this relies on gcc translating the majority of
 /// command line options.
 
 Generic_GCC::Generic_GCC(const HostInfo &Host, const llvm::Triple& Triple)
   : ToolChain(Host, Triple) {
-  getProgramPaths().push_back(getDriver().Dir);
+  getProgramPaths().push_back(getDriver().getInstalledDir());
+  if (getDriver().getInstalledDir() != getDriver().Dir.c_str())
+    getProgramPaths().push_back(getDriver().Dir);
 }
 
 Generic_GCC::~Generic_GCC() {
@@ -864,8 +966,16 @@ Tool &OpenBSD::SelectTool(const Compilation &C, const JobAction &JA) const {
 
 /// FreeBSD - FreeBSD tool chain which can call as(1) and ld(1) directly.
 
-FreeBSD::FreeBSD(const HostInfo &Host, const llvm::Triple& Triple, bool Lib32)
+FreeBSD::FreeBSD(const HostInfo &Host, const llvm::Triple& Triple)
   : Generic_GCC(Host, Triple) {
+
+  // Determine if we are compiling 32-bit code on an x86_64 platform.
+  bool Lib32 = false;
+  if (Triple.getArch() == llvm::Triple::x86 &&
+      llvm::Triple(getDriver().DefaultHostTriple).getArch() ==
+        llvm::Triple::x86_64)
+    Lib32 = true;
+    
   getProgramPaths().push_back(getDriver().Dir + "/../libexec");
   getProgramPaths().push_back("/usr/libexec");
   if (Lib32) {
@@ -936,7 +1046,9 @@ Tool &Minix::SelectTool(const Compilation &C, const JobAction &JA) const {
 AuroraUX::AuroraUX(const HostInfo &Host, const llvm::Triple& Triple)
   : Generic_GCC(Host, Triple) {
 
-  getProgramPaths().push_back(getDriver().Dir);
+  getProgramPaths().push_back(getDriver().getInstalledDir());
+  if (getDriver().getInstalledDir() != getDriver().Dir.c_str())
+    getProgramPaths().push_back(getDriver().Dir);
 
   getFilePaths().push_back(getDriver().Dir + "/../lib");
   getFilePaths().push_back("/usr/lib");
@@ -973,7 +1085,8 @@ Tool &AuroraUX::SelectTool(const Compilation &C, const JobAction &JA) const {
 
 Linux::Linux(const HostInfo &Host, const llvm::Triple& Triple)
   : Generic_GCC(Host, Triple) {
-  getFilePaths().push_back(getDriver().Dir + "/../lib/clang/1.0/");
+  getFilePaths().push_back(getDriver().Dir +
+                           "/../lib/clang/" CLANG_VERSION_STRING "/");
   getFilePaths().push_back("/lib/");
   getFilePaths().push_back("/usr/lib/");
 
@@ -994,13 +1107,35 @@ Linux::Linux(const HostInfo &Host, const llvm::Triple& Triple)
   // list), but that's messy at best.
 }
 
+Tool &Linux::SelectTool(const Compilation &C, const JobAction &JA) const {
+  Action::ActionClass Key;
+  if (getDriver().ShouldUseClangCompiler(C, JA, getTriple()))
+    Key = Action::AnalyzeJobClass;
+  else
+    Key = JA.getKind();
+
+  Tool *&T = Tools[Key];
+  if (!T) {
+    switch (Key) {
+    case Action::AssembleJobClass:
+      T = new tools::linuxtools::Assemble(*this); break;
+    default:
+      T = &Generic_GCC::SelectTool(C, JA);
+    }
+  }
+
+  return *T;
+}
+
 /// DragonFly - DragonFly tool chain which can call as(1) and ld(1) directly.
 
 DragonFly::DragonFly(const HostInfo &Host, const llvm::Triple& Triple)
   : Generic_GCC(Host, Triple) {
 
   // Path mangling to find libexec
-  getProgramPaths().push_back(getDriver().Dir);
+  getProgramPaths().push_back(getDriver().getInstalledDir());
+  if (getDriver().getInstalledDir() != getDriver().Dir.c_str())
+    getProgramPaths().push_back(getDriver().Dir);
 
   getFilePaths().push_back(getDriver().Dir + "/../lib");
   getFilePaths().push_back("/usr/lib");
@@ -1028,3 +1163,57 @@ Tool &DragonFly::SelectTool(const Compilation &C, const JobAction &JA) const {
 
   return *T;
 }
+
+Windows::Windows(const HostInfo &Host, const llvm::Triple& Triple)
+  : ToolChain(Host, Triple) {
+}
+
+Tool &Windows::SelectTool(const Compilation &C, const JobAction &JA) const {
+  Action::ActionClass Key;
+  if (getDriver().ShouldUseClangCompiler(C, JA, getTriple()))
+    Key = Action::AnalyzeJobClass;
+  else
+    Key = JA.getKind();
+
+  Tool *&T = Tools[Key];
+  if (!T) {
+    switch (Key) {
+    case Action::InputClass:
+    case Action::BindArchClass:
+    case Action::LipoJobClass:
+    case Action::DsymutilJobClass:
+      assert(0 && "Invalid tool kind.");
+    case Action::PreprocessJobClass:
+    case Action::PrecompileJobClass:
+    case Action::AnalyzeJobClass:
+    case Action::CompileJobClass:
+      T = new tools::Clang(*this); break;
+    case Action::AssembleJobClass:
+      T = new tools::ClangAs(*this); break;
+    case Action::LinkJobClass:
+      T = new tools::visualstudio::Link(*this); break;
+    }
+  }
+
+  return *T;
+}
+
+bool Windows::IsIntegratedAssemblerDefault() const {
+  return true;
+}
+
+bool Windows::IsUnwindTablesDefault() const {
+  // FIXME: Gross; we should probably have some separate target
+  // definition, possibly even reusing the one in clang.
+  return getArchName() == "x86_64";
+}
+
+const char *Windows::GetDefaultRelocationModel() const {
+  return "static";
+}
+
+const char *Windows::GetForcedPicModel() const {
+  if (getArchName() == "x86_64")
+    return "pic";
+  return 0;
+}
diff --git a/lib/Driver/ToolChains.h b/lib/Driver/ToolChains.h
index 4bdd00fc2583..d1f15565e6c6 100644
--- a/lib/Driver/ToolChains.h
+++ b/lib/Driver/ToolChains.h
@@ -42,6 +42,11 @@ public:
 
 /// Darwin - The base Darwin tool chain.
 class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
+public:
+  /// The host version.
+  unsigned DarwinVersion[3];
+
+private:
   mutable llvm::DenseMap<unsigned, Tool*> Tools;
 
   /// Whether the information on the target has been initialized.
@@ -61,11 +66,15 @@ class LLVM_LIBRARY_VISIBILITY Darwin : public ToolChain {
   /// initialized.
   std::string MacosxVersionMin;
 
+private:
+  void AddDeploymentTarget(DerivedArgList &Args) const;
+
 public:
-  Darwin(const HostInfo &Host, const llvm::Triple& Triple,
-         const unsigned (&DarwinVersion)[3]);
+  Darwin(const HostInfo &Host, const llvm::Triple& Triple);
   ~Darwin();
 
+  std::string ComputeEffectiveClangTriple(const ArgList &Args) const;
+
   /// @name Darwin Specific Toolchain API
   /// {
 
@@ -144,17 +153,17 @@ public:
   /// @name ToolChain Implementation
   /// {
 
+  virtual types::ID LookupTypeForExtension(const char *Ext) const;
+
   virtual DerivedArgList *TranslateArgs(const DerivedArgList &Args,
                                         const char *BoundArch) const;
 
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 
   virtual bool IsBlocksDefault() const {
-    // Blocks default to on for OS X 10.6 and iPhoneOS 3.0 and beyond.
-    if (isTargetIPhoneOS())
-      return !isIPhoneOSVersionLT(3);
-    else
-      return !isMacosxVersionLT(10, 6);
+    // Always allow blocks on Darwin; users interested in versioning are
+    // expected to use /usr/include/Blocks.h.
+    return true;
   }
   virtual bool IsIntegratedAssemblerDefault() const {
 #ifdef DISABLE_DEFAULT_INTEGRATED_ASSEMBLER
@@ -201,8 +210,7 @@ public:
 /// DarwinClang - The Darwin toolchain used by Clang.
 class LLVM_LIBRARY_VISIBILITY DarwinClang : public Darwin {
 public:
-  DarwinClang(const HostInfo &Host, const llvm::Triple& Triple,
-              const unsigned (&DarwinVersion)[3]);
+  DarwinClang(const HostInfo &Host, const llvm::Triple& Triple);
 
   /// @name Darwin ToolChain Implementation
   /// {
@@ -225,9 +233,7 @@ class LLVM_LIBRARY_VISIBILITY DarwinGCC : public Darwin {
   std::string ToolChainDir;
 
 public:
-  DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple,
-            const unsigned (&DarwinVersion)[3],
-            const unsigned (&GCCVersion)[3]);
+  DarwinGCC(const HostInfo &Host, const llvm::Triple& Triple);
 
   /// @name Darwin ToolChain Implementation
   /// {
@@ -247,6 +253,8 @@ public:
   Darwin_Generic_GCC(const HostInfo &Host, const llvm::Triple& Triple)
     : Generic_GCC(Host, Triple) {}
 
+  std::string ComputeEffectiveClangTriple(const ArgList &Args) const;
+
   virtual const char *GetDefaultRelocationModel() const { return "pic"; }
 };
 
@@ -266,7 +274,7 @@ public:
 
 class LLVM_LIBRARY_VISIBILITY FreeBSD : public Generic_GCC {
 public:
-  FreeBSD(const HostInfo &Host, const llvm::Triple& Triple, bool Lib32);
+  FreeBSD(const HostInfo &Host, const llvm::Triple& Triple);
 
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
@@ -288,6 +296,8 @@ public:
 class LLVM_LIBRARY_VISIBILITY Linux : public Generic_GCC {
 public:
   Linux(const HostInfo &Host, const llvm::Triple& Triple);
+
+  virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
 };
 
 
@@ -309,6 +319,20 @@ private:
 
 };
 
+class LLVM_LIBRARY_VISIBILITY Windows : public ToolChain {
+  mutable llvm::DenseMap<unsigned, Tool*> Tools;
+
+public:
+  Windows(const HostInfo &Host, const llvm::Triple& Triple);
+
+  virtual Tool &SelectTool(const Compilation &C, const JobAction &JA) const;
+
+  virtual bool IsIntegratedAssemblerDefault() const;
+  virtual bool IsUnwindTablesDefault() const;
+  virtual const char *GetDefaultRelocationModel() const;
+  virtual const char *GetForcedPicModel() const;
+};
+
 } // end namespace toolchains
 } // end namespace driver
 } // end namespace clang
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index f423d4e3b918..8436561e6ef9 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -105,10 +105,7 @@ void Clang::AddPreprocessingOptions(const Driver &D,
     // Determine the output location.
     const char *DepFile;
     if (Output.getType() == types::TY_Dependencies) {
-      if (Output.isPipe())
-        DepFile = "-";
-      else
-        DepFile = Output.getFilename();
+      DepFile = Output.getFilename();
     } else if (Arg *MF = Args.getLastArg(options::OPT_MF)) {
       DepFile = MF->getValue(Args);
     } else if (A->getOption().matches(options::OPT_M) ||
@@ -182,10 +179,8 @@ void Clang::AddPreprocessingOptions(const Driver &D,
     const Arg *A = it;
 
     if (A->getOption().matches(options::OPT_include)) {
-      // Use PCH if the user requested it, except for C++ (for now).
+      // Use PCH if the user requested it.
       bool UsePCH = D.CCCUsePCH;
-      if (types::isCXX(Inputs[0].getType()))
-        UsePCH = false;
 
       bool FoundPTH = false;
       bool FoundPCH = false;
@@ -342,35 +337,6 @@ static const char *getLLVMArchSuffixForARM(llvm::StringRef CPU) {
   return "";
 }
 
-/// getLLVMTriple - Get the LLVM triple to use for a particular toolchain, which
-/// may depend on command line arguments.
-static std::string getLLVMTriple(const ToolChain &TC, const ArgList &Args) {
-  switch (TC.getTriple().getArch()) {
-  default:
-    return TC.getTripleString();
-
-  case llvm::Triple::arm:
-  case llvm::Triple::thumb: {
-    // FIXME: Factor into subclasses.
-    llvm::Triple Triple = TC.getTriple();
-
-    // Thumb2 is the default for V7 on Darwin.
-    //
-    // FIXME: Thumb should just be another -target-feaure, not in the triple.
-    llvm::StringRef Suffix =
-      getLLVMArchSuffixForARM(getARMTargetCPU(Args, Triple));
-    bool ThumbDefault =
-      (Suffix == "v7" && TC.getTriple().getOS() == llvm::Triple::Darwin);
-    std::string ArchName = "arm";
-    if (Args.hasFlag(options::OPT_mthumb, options::OPT_mno_thumb, ThumbDefault))
-      ArchName = "thumb";
-    Triple.setArchName(ArchName + Suffix.str());
-
-    return Triple.getTriple();
-  }
-  }
-}
-
 // FIXME: Move to target hook.
 static bool isSignedCharDefault(const llvm::Triple &Triple) {
   switch (Triple.getArch()) {
@@ -633,6 +599,11 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
         CPUName = "x86-64";
       else if (getToolChain().getArchName() == "i386")
         CPUName = "i586";
+    } else if (getToolChain().getOS().startswith("openbsd"))  {
+      if (getToolChain().getArchName() == "x86_64")
+        CPUName = "x86-64";
+      else if (getToolChain().getArchName() == "i386")
+        CPUName = "i486";
     } else {
       if (getToolChain().getArchName() == "x86_64")
         CPUName = "x86-64";
@@ -694,57 +665,7 @@ static bool needsExceptions(const ArgList &Args,  types::ID InputType,
   }
 }
 
-/// getEffectiveClangTriple - Get the "effective" target triple, which is the
-/// triple for the target but with the OS version potentially modified for
-/// Darwin's -mmacosx-version-min.
-static std::string getEffectiveClangTriple(const Driver &D,
-                                           const ToolChain &TC,
-                                           const ArgList &Args) {
-  llvm::Triple Triple(getLLVMTriple(TC, Args));
-
-  // Handle -mmacosx-version-min and -miphoneos-version-min.
-  if (Triple.getOS() != llvm::Triple::Darwin) {
-    // Diagnose use of -mmacosx-version-min and -miphoneos-version-min on
-    // non-Darwin.
-    if (Arg *A = Args.getLastArg(options::OPT_mmacosx_version_min_EQ,
-                                 options::OPT_miphoneos_version_min_EQ))
-      D.Diag(clang::diag::err_drv_clang_unsupported) << A->getAsString(Args);
-  } else {
-    const toolchains::Darwin &DarwinTC(
-      reinterpret_cast<const toolchains::Darwin&>(TC));
-
-    // If the target isn't initialized (e.g., an unknown Darwin platform, return
-    // the default triple).
-    if (!DarwinTC.isTargetInitialized())
-      return Triple.getTriple();
-    
-    unsigned Version[3];
-    DarwinTC.getTargetVersion(Version);
-
-    // Mangle the target version into the OS triple component.  For historical
-    // reasons that make little sense, the version passed here is the "darwin"
-    // version, which drops the 10 and offsets by 4. See inverse code when
-    // setting the OS version preprocessor define.
-    if (!DarwinTC.isTargetIPhoneOS()) {
-      Version[0] = Version[1] + 4;
-      Version[1] = Version[2];
-      Version[2] = 0;
-    } else {
-      // Use the environment to communicate that we are targetting iPhoneOS.
-      Triple.setEnvironmentName("iphoneos");
-    }
-
-    llvm::SmallString<16> Str;
-    llvm::raw_svector_ostream(Str) << "darwin" << Version[0]
-                                   << "." << Version[1] << "." << Version[2];
-    Triple.setOSName(Str.str());
-  }
-
-  return Triple.getTriple();
-}
-
 void Clang::ConstructJob(Compilation &C, const JobAction &JA,
-                         Job &Dest,
                          const InputInfo &Output,
                          const InputInfoList &Inputs,
                          const ArgList &Args,
@@ -763,10 +684,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Add the "effective" target triple.
   CmdArgs.push_back("-triple");
-  std::string TripleStr = getEffectiveClangTriple(D, getToolChain(), Args);
+  std::string TripleStr = getToolChain().ComputeEffectiveClangTriple(Args);
   CmdArgs.push_back(Args.MakeArgString(TripleStr));
 
   // Select the appropriate action.
+  bool IsRewriter = false;
   if (isa<AnalyzeJobAction>(JA)) {
     assert(JA.getType() == types::TY_Plist && "Invalid output type.");
     CmdArgs.push_back("-analyze");
@@ -786,11 +708,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                       options::OPT_mno_relax_all,
                       !IsOpt))
       CmdArgs.push_back("-mrelax-all");
+
+    // When using an integrated assembler, we send -Wa, and -Xassembler options
+    // to -cc1.
+    Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
+                         options::OPT_Xassembler);
   } else if (isa<PrecompileJobAction>(JA)) {
-    // Use PCH if the user requested it, except for C++ (for now).
+    // Use PCH if the user requested it.
     bool UsePCH = D.CCCUsePCH;
-    if (types::isCXX(Inputs[0].getType()))
-      UsePCH = false;
 
     if (UsePCH)
       CmdArgs.push_back("-emit-pch");
@@ -813,6 +738,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("-emit-pch");
     } else if (JA.getType() == types::TY_RewrittenObjC) {
       CmdArgs.push_back("-rewrite-objc");
+      IsRewriter = true;
     } else {
       assert(JA.getType() == types::TY_PP_Asm &&
              "Unexpected output type!");
@@ -856,6 +782,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       // Do not enable the missing -dealloc check.
       // '-analyzer-check-objc-missing-dealloc',
       CmdArgs.push_back("-analyzer-check-objc-unused-ivars");
+      CmdArgs.push_back("-analyzer-check-idempotent-operations");
     }
 
     // Set the output format. The default is plist, for (lame) historical
@@ -999,6 +926,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     break;
   }
 
+  // Pass the linker version in use.
+  if (Arg *A = Args.getLastArg(options::OPT_mlinker_version_EQ)) {
+    CmdArgs.push_back("-target-linker-version");
+    CmdArgs.push_back(A->getValue(Args));
+  }
+
   // -mno-omit-leaf-frame-pointer is default.
   if (Args.hasFlag(options::OPT_momit_leaf_frame_pointer,
                    options::OPT_mno_omit_leaf_frame_pointer, false))
@@ -1029,6 +962,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_v);
+  Args.AddLastArg(CmdArgs, options::OPT_H);
   Args.AddLastArg(CmdArgs, options::OPT_P);
   Args.AddLastArg(CmdArgs, options::OPT_print_ivar_layout);
 
@@ -1201,10 +1135,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fno_show_column);
   Args.AddLastArg(CmdArgs, options::OPT_fobjc_sender_dependent_dispatch);
   Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_print_source_range_info);
+  Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_parseable_fixits);
   Args.AddLastArg(CmdArgs, options::OPT_ftime_report);
   Args.AddLastArg(CmdArgs, options::OPT_ftrapv);
   Args.AddLastArg(CmdArgs, options::OPT_fwrapv);
   Args.AddLastArg(CmdArgs, options::OPT_fwritable_strings);
+  Args.AddLastArg(CmdArgs, options::OPT_funroll_loops);
 
   Args.AddLastArg(CmdArgs, options::OPT_pthread);
 
@@ -1279,15 +1215,23 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    getToolChain().getTriple().getOS() == llvm::Triple::Win32))
     CmdArgs.push_back("-fms-extensions");
 
+  // -fborland-extensions=0 is default.
+  if (Args.hasFlag(options::OPT_fborland_extensions,
+                   options::OPT_fno_borland_extensions, false))
+    CmdArgs.push_back("-fborland-extensions");
+
   // -fgnu-keywords default varies depending on language; only pass if
   // specified.
   if (Arg *A = Args.getLastArg(options::OPT_fgnu_keywords,
                                options::OPT_fno_gnu_keywords))
     A->render(Args, CmdArgs);
 
-  // -fnext-runtime is default.
+  // -fnext-runtime defaults to on Darwin and when rewriting Objective-C, and is
+  // -the -cc1 default.
+  bool NeXTRuntimeIsDefault = 
+    IsRewriter || getToolChain().getTriple().getOS() == llvm::Triple::Darwin;
   if (!Args.hasFlag(options::OPT_fnext_runtime, options::OPT_fgnu_runtime,
-                    getToolChain().getTriple().getOS() == llvm::Triple::Darwin))
+                    NeXTRuntimeIsDefault))
     CmdArgs.push_back("-fgnu-runtime");
 
   // -fobjc-nonfragile-abi=0 is default.
@@ -1389,7 +1333,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-fdiagnostics-show-category");
     CmdArgs.push_back(A->getValue(Args));
   }
-  
+
   // Color diagnostics are the default, unless the terminal doesn't support
   // them.
   if (Args.hasFlag(options::OPT_fcolor_diagnostics,
@@ -1404,7 +1348,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (!Args.hasFlag(options::OPT_fspell_checking,
                     options::OPT_fno_spell_checking))
     CmdArgs.push_back("-fno-spell-checking");
-  
+
   if (Arg *A = Args.getLastArg(options::OPT_fshow_overloads_EQ))
     A->render(Args, CmdArgs);
 
@@ -1464,9 +1408,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Output.getType() == types::TY_Dependencies) {
     // Handled with other dependency code.
-  } else if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
   } else if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
@@ -1479,9 +1420,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     const InputInfo &II = *it;
     CmdArgs.push_back("-x");
     CmdArgs.push_back(types::getTypeName(II.getType()));
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       II.getInputArg().renderAsInput(Args, CmdArgs);
@@ -1489,7 +1428,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_undef);
 
-  std::string Exec = getToolChain().getDriver().getClangProgramPath();
+  const char *Exec = getToolChain().getDriver().getClangProgramPath();
 
   // Optionally embed the -cc1 level arguments into the debug info, for build
   // analysis.
@@ -1498,7 +1437,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     for (ArgList::const_iterator it = Args.begin(),
            ie = Args.end(); it != ie; ++it)
       (*it)->render(Args, OriginalArgs);
-    
+
     llvm::SmallString<256> Flags;
     Flags += Exec;
     for (unsigned i = 0, e = OriginalArgs.size(); i != e; ++i) {
@@ -1509,7 +1448,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(Flags.str()));
   }
 
-  Dest.addCommand(new Command(JA, *this, Exec.c_str(), CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 
   // Explicitly warn that these options are unsupported, even though
   // we are allowing compilation to continue.
@@ -1533,12 +1472,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 }
 
 void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
-                           Job &Dest,
                            const InputInfo &Output,
                            const InputInfoList &Inputs,
                            const ArgList &Args,
                            const char *LinkingOutput) const {
-  const Driver &D = getToolChain().getDriver();
   ArgStringList CmdArgs;
 
   assert(Inputs.size() == 1 && "Unexpected number of inputs.");
@@ -1551,7 +1488,7 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Add the "effective" target triple.
   CmdArgs.push_back("-triple");
-  std::string TripleStr = getEffectiveClangTriple(D, getToolChain(), Args);
+  std::string TripleStr = getToolChain().ComputeEffectiveClangTriple(Args);
   CmdArgs.push_back(Args.MakeArgString(TripleStr));
 
   // Set the output mode, we currently only expect to be used as a real
@@ -1581,19 +1518,14 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
-  if (Input.isPipe()) {
-    CmdArgs.push_back("-");
-  } else {
-    assert(Input.isFilename() && "Invalid input.");
-    CmdArgs.push_back(Input.getFilename());
-  }
+  assert(Input.isFilename() && "Invalid input.");
+  CmdArgs.push_back(Input.getFilename());
 
-  std::string Exec = getToolChain().getDriver().getClangProgramPath();
-  Dest.addCommand(new Command(JA, *this, Exec.c_str(), CmdArgs));
+  const char *Exec = getToolChain().getDriver().getClangProgramPath();
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
-                               Job &Dest,
                                const InputInfo &Output,
                                const InputInfoList &Inputs,
                                const ArgList &Args,
@@ -1605,6 +1537,11 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
          it = Args.begin(), ie = Args.end(); it != ie; ++it) {
     Arg *A = *it;
     if (A->getOption().hasForwardToGCC()) {
+      // Don't forward any -g arguments to assembly steps.
+      if (isa<AssembleJobAction>(JA) &&
+          A->getOption().matches(options::OPT_g_Group))
+        continue;
+
       // It is unfortunate that we have to claim here, as this means
       // we will basically never report anything interesting for
       // platforms using a generic gcc, even if we are just using gcc
@@ -1640,10 +1577,7 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
   else if (Arch == "x86_64" || Arch == "powerpc64")
     CmdArgs.push_back("-m64");
 
-  if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
-  } else if (Output.isFilename()) {
+  if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
@@ -1678,9 +1612,7 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back(types::getTypeName(II.getType()));
     }
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       // Don't render as input, we need gcc to do the translations.
@@ -1690,7 +1622,7 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
   const char *GCCName = getToolChain().getDriver().CCCGenericGCCName.c_str();
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath(GCCName));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void gcc::Preprocess::RenderExtraToolArgs(const JobAction &JA,
@@ -2022,10 +1954,7 @@ void darwin::CC1::AddCPPUniqueOptionsArgs(const ArgList &Args,
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else
-      CmdArgs.push_back(II.getFilename());
+    CmdArgs.push_back(II.getFilename());
   }
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wp_COMMA,
@@ -2063,7 +1992,7 @@ void darwin::CC1::AddCPPArgs(const ArgList &Args,
 }
 
 void darwin::Preprocess::ConstructJob(Compilation &C, const JobAction &JA,
-                                      Job &Dest, const InputInfo &Output,
+                                      const InputInfo &Output,
                                       const InputInfoList &Inputs,
                                       const ArgList &Args,
                                       const char *LinkingOutput) const {
@@ -2078,12 +2007,9 @@ void darwin::Preprocess::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-traditional-cpp");
 
   ArgStringList OutputArgs;
-  if (Output.isFilename()) {
-    OutputArgs.push_back("-o");
-    OutputArgs.push_back(Output.getFilename());
-  } else {
-    assert(Output.isPipe() && "Unexpected CC1 output.");
-  }
+  assert(Output.isFilename() && "Unexpected CC1 output.");
+  OutputArgs.push_back("-o");
+  OutputArgs.push_back(Output.getFilename());
 
   if (Args.hasArg(options::OPT_E)) {
     AddCPPOptionsArgs(Args, CmdArgs, Inputs, OutputArgs);
@@ -2097,11 +2023,11 @@ void darwin::Preprocess::ConstructJob(Compilation &C, const JobAction &JA,
   const char *CC1Name = getCC1Name(Inputs[0].getType());
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath(CC1Name));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
-                                   Job &Dest, const InputInfo &Output,
+                                   const InputInfo &Output,
                                    const InputInfoList &Inputs,
                                    const ArgList &Args,
                                    const char *LinkingOutput) const {
@@ -2133,9 +2059,7 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
   ArgStringList OutputArgs;
   if (Output.getType() != types::TY_PCH) {
     OutputArgs.push_back("-o");
-    if (Output.isPipe())
-      OutputArgs.push_back("-");
-    else if (Output.isNothing())
+    if (Output.isNothing())
       OutputArgs.push_back("/dev/null");
     else
       OutputArgs.push_back(Output.getFilename());
@@ -2168,10 +2092,7 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
         return;
       }
 
-      if (II.isPipe())
-        CmdArgs.push_back("-");
-      else
-        CmdArgs.push_back(II.getFilename());
+      CmdArgs.push_back(II.getFilename());
     }
 
     if (OutputArgsEarly) {
@@ -2197,11 +2118,11 @@ void darwin::Compile::ConstructJob(Compilation &C, const JobAction &JA,
   const char *CC1Name = getCC1Name(Inputs[0].getType());
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath(CC1Name));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
-                                    Job &Dest, const InputInfo &Output,
+                                    const InputInfo &Output,
                                     const InputInfoList &Inputs,
                                     const ArgList &Args,
                                     const char *LinkingOutput) const {
@@ -2224,7 +2145,9 @@ void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   // Derived from asm spec.
   AddDarwinArch(Args, CmdArgs);
 
-  if (!getDarwinToolChain().isTargetIPhoneOS() ||
+  // Use -force_cpusubtype_ALL on x86 by default.
+  if (getToolChain().getTriple().getArch() == llvm::Triple::x86 ||
+      getToolChain().getTriple().getArch() == llvm::Triple::x86_64 ||
       Args.hasArg(options::OPT_force__cpusubtype__ALL))
     CmdArgs.push_back("-force_cpusubtype_ALL");
 
@@ -2241,18 +2164,14 @@ void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
-  if (Input.isPipe()) {
-    CmdArgs.push_back("-");
-  } else {
-    assert(Input.isFilename() && "Invalid input.");
-    CmdArgs.push_back(Input.getFilename());
-  }
+  assert(Input.isFilename() && "Invalid input.");
+  CmdArgs.push_back(Input.getFilename());
 
   // asm_final spec is empty.
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void darwin::DarwinTool::AddDarwinArch(const ArgList &Args,
@@ -2272,6 +2191,22 @@ void darwin::Link::AddLinkArgs(const ArgList &Args,
                                ArgStringList &CmdArgs) const {
   const Driver &D = getToolChain().getDriver();
 
+  unsigned Version[3] = { 0, 0, 0 };
+  if (Arg *A = Args.getLastArg(options::OPT_mlinker_version_EQ)) {
+    bool HadExtra;
+    if (!Driver::GetReleaseVersion(A->getValue(Args), Version[0],
+                                   Version[1], Version[2], HadExtra) ||
+        HadExtra)
+      D.Diag(clang::diag::err_drv_invalid_version_number)
+        << A->getAsString(Args);
+  }
+
+  // Newer linkers support -demangle, pass it if supported and not disabled by
+  // the user.
+  if (Version[0] >= 100 && !Args.hasArg(options::OPT_Z_Xlinker__no_demangle)) {
+    CmdArgs.push_back("-demangle");
+  }
+
   // Derived from the "link" spec.
   Args.AddAllArgs(CmdArgs, options::OPT_static);
   if (!Args.hasArg(options::OPT_static))
@@ -2413,7 +2348,7 @@ void darwin::Link::AddLinkArgs(const ArgList &Args,
 }
 
 void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
-                                Job &Dest, const InputInfo &Output,
+                                const InputInfo &Output,
                                 const InputInfoList &Inputs,
                                 const ArgList &Args,
                                 const char *LinkingOutput) const {
@@ -2571,11 +2506,11 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("ld"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void darwin::Lipo::ConstructJob(Compilation &C, const JobAction &JA,
-                                Job &Dest, const InputInfo &Output,
+                                const InputInfo &Output,
                                 const InputInfoList &Inputs,
                                 const ArgList &Args,
                                 const char *LinkingOutput) const {
@@ -2595,11 +2530,11 @@ void darwin::Lipo::ConstructJob(Compilation &C, const JobAction &JA,
   }
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("lipo"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void darwin::Dsymutil::ConstructJob(Compilation &C, const JobAction &JA,
-                                    Job &Dest, const InputInfo &Output,
+                                    const InputInfo &Output,
                                     const InputInfoList &Inputs,
                                     const ArgList &Args,
                                     const char *LinkingOutput) const {
@@ -2615,11 +2550,11 @@ void darwin::Dsymutil::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("dsymutil"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void auroraux::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
-                                      Job &Dest, const InputInfo &Output,
+                                      const InputInfo &Output,
                                       const InputInfoList &Inputs,
                                       const ArgList &Args,
                                       const char *LinkingOutput) const {
@@ -2629,27 +2564,21 @@ void auroraux::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
                        options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
-  if (Output.isPipe())
-    CmdArgs.push_back("-");
-  else
-    CmdArgs.push_back(Output.getFilename());
+  CmdArgs.push_back(Output.getFilename());
 
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else
-      CmdArgs.push_back(II.getFilename());
+    CmdArgs.push_back(II.getFilename());
   }
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("gas"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
-                                  Job &Dest, const InputInfo &Output,
+                                  const InputInfo &Output,
                                   const InputInfoList &Inputs,
                                   const ArgList &Args,
                                   const char *LinkingOutput) const {
@@ -2676,10 +2605,7 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
-  } else if (Output.isFilename()) {
+  if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
@@ -2721,9 +2647,7 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
       D.Diag(clang::diag::err_drv_no_linker_llvm_support)
         << getToolChain().getTripleString();
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       II.getInputArg().renderAsInput(Args, CmdArgs);
@@ -2751,11 +2675,11 @@ void auroraux::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("ld"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void openbsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
-                                     Job &Dest, const InputInfo &Output,
+                                     const InputInfo &Output,
                                      const InputInfoList &Inputs,
                                      const ArgList &Args,
                                      const char *LinkingOutput) const {
@@ -2765,27 +2689,21 @@ void openbsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
                        options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
-  if (Output.isPipe())
-    CmdArgs.push_back("-");
-  else
-    CmdArgs.push_back(Output.getFilename());
+  CmdArgs.push_back(Output.getFilename());
 
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else
-      CmdArgs.push_back(II.getFilename());
+    CmdArgs.push_back(II.getFilename());
   }
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
-                                 Job &Dest, const InputInfo &Output,
+                                 const InputInfo &Output,
                                  const InputInfoList &Inputs,
                                  const ArgList &Args,
                                  const char *LinkingOutput) const {
@@ -2811,10 +2729,7 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
-  } else if (Output.isFilename()) {
+  if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
@@ -2838,7 +2753,7 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   if (Triple.substr(0, 6) == "x86_64")
     Triple.replace(0, 6, "amd64");
   CmdArgs.push_back(Args.MakeArgString("-L/usr/lib/gcc-lib/" + Triple +
-                                       "/3.3.5"));
+                                       "/4.2.1"));
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
@@ -2854,9 +2769,7 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       D.Diag(clang::diag::err_drv_no_linker_llvm_support)
         << getToolChain().getTripleString();
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       II.getInputArg().renderAsInput(Args, CmdArgs);
@@ -2864,6 +2777,11 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
+    if (D.CCCIsCXX) {
+      CmdArgs.push_back("-lstdc++");
+      CmdArgs.push_back("-lm");
+    }
+
     // FIXME: For some reason GCC passes -lgcc before adding
     // the default system libraries. Just mimic this for now.
     CmdArgs.push_back("-lgcc");
@@ -2887,11 +2805,11 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("ld"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void freebsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
-                                     Job &Dest, const InputInfo &Output,
+                                     const InputInfo &Output,
                                      const InputInfoList &Inputs,
                                      const ArgList &Args,
                                      const char *LinkingOutput) const {
@@ -2913,27 +2831,21 @@ void freebsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
                        options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
-  if (Output.isPipe())
-    CmdArgs.push_back("-");
-  else
-    CmdArgs.push_back(Output.getFilename());
+  CmdArgs.push_back(Output.getFilename());
 
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else
-      CmdArgs.push_back(II.getFilename());
+    CmdArgs.push_back(II.getFilename());
   }
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
-                                 Job &Dest, const InputInfo &Output,
+                                 const InputInfo &Output,
                                  const InputInfoList &Inputs,
                                  const ArgList &Args,
                                  const char *LinkingOutput) const {
@@ -2959,10 +2871,7 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("elf_i386_fbsd");
   }
 
-  if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
-  } else if (Output.isFilename()) {
+  if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
@@ -2989,6 +2898,10 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   Args.AddAllArgs(CmdArgs, options::OPT_T_Group);
   Args.AddAllArgs(CmdArgs, options::OPT_e);
+  Args.AddAllArgs(CmdArgs, options::OPT_s);
+  Args.AddAllArgs(CmdArgs, options::OPT_t);
+  Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag);
+  Args.AddAllArgs(CmdArgs, options::OPT_r);
 
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
@@ -3000,9 +2913,7 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       D.Diag(clang::diag::err_drv_no_linker_llvm_support)
         << getToolChain().getTripleString();
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       II.getInputArg().renderAsInput(Args, CmdArgs);
@@ -3053,51 +2964,79 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("ld"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
+void linuxtools::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
+                                        const InputInfo &Output,
+                                        const InputInfoList &Inputs,
+                                        const ArgList &Args,
+                                        const char *LinkingOutput) const {
+  ArgStringList CmdArgs;
+
+  // Add --32/--64 to make sure we get the format we want.
+  // This is incomplete
+  if (getToolChain().getArch() == llvm::Triple::x86) {
+    CmdArgs.push_back("--32");
+  } else if (getToolChain().getArch() == llvm::Triple::x86_64) {
+    CmdArgs.push_back("--64");
+  } else if (getToolChain().getArch() == llvm::Triple::arm) {
+    llvm::StringRef MArch = getToolChain().getArchName();
+    if (MArch == "armv7" || MArch == "armv7a" || MArch == "armv7-a")
+      CmdArgs.push_back("-mfpu=neon");
+  }
+
+  Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
+                       options::OPT_Xassembler);
+
+  CmdArgs.push_back("-o");
+  CmdArgs.push_back(Output.getFilename());
+
+  for (InputInfoList::const_iterator
+         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
+    const InputInfo &II = *it;
+    CmdArgs.push_back(II.getFilename());
+  }
+
+  const char *Exec =
+    Args.MakeArgString(getToolChain().GetProgramPath("as"));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
+}
+
+
 void minix::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
-                                     Job &Dest, const InputInfo &Output,
-                                     const InputInfoList &Inputs,
-                                     const ArgList &Args,
-                                     const char *LinkingOutput) const {
+                                   const InputInfo &Output,
+                                   const InputInfoList &Inputs,
+                                   const ArgList &Args,
+                                   const char *LinkingOutput) const {
   ArgStringList CmdArgs;
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
                        options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
-  if (Output.isPipe())
-    CmdArgs.push_back("-");
-  else
-    CmdArgs.push_back(Output.getFilename());
+  CmdArgs.push_back(Output.getFilename());
 
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else
-      CmdArgs.push_back(II.getFilename());
+    CmdArgs.push_back(II.getFilename());
   }
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("gas"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
-                                 Job &Dest, const InputInfo &Output,
-                                 const InputInfoList &Inputs,
-                                 const ArgList &Args,
-                                 const char *LinkingOutput) const {
+                               const InputInfo &Output,
+                               const InputInfoList &Inputs,
+                               const ArgList &Args,
+                               const char *LinkingOutput) const {
   const Driver &D = getToolChain().getDriver();
   ArgStringList CmdArgs;
 
-  if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
-  } else if (Output.isFilename()) {
+  if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
@@ -3123,9 +3062,7 @@ void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
       D.Diag(clang::diag::err_drv_no_linker_llvm_support)
         << getToolChain().getTripleString();
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       II.getInputArg().renderAsInput(Args, CmdArgs);
@@ -3156,7 +3093,7 @@ void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("/usr/gnu/bin/gld"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 /// DragonFly Tools
@@ -3164,7 +3101,7 @@ void minix::Link::ConstructJob(Compilation &C, const JobAction &JA,
 // For now, DragonFly Assemble does just about the same as for
 // FreeBSD, but this may change soon.
 void dragonfly::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
-                                       Job &Dest, const InputInfo &Output,
+                                       const InputInfo &Output,
                                        const InputInfoList &Inputs,
                                        const ArgList &Args,
                                        const char *LinkingOutput) const {
@@ -3179,30 +3116,24 @@ void dragonfly::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
                        options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
-  if (Output.isPipe())
-    CmdArgs.push_back("-");
-  else
-    CmdArgs.push_back(Output.getFilename());
+  CmdArgs.push_back(Output.getFilename());
 
   for (InputInfoList::const_iterator
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else
-      CmdArgs.push_back(II.getFilename());
+    CmdArgs.push_back(II.getFilename());
   }
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("as"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
 void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
-                                 Job &Dest, const InputInfo &Output,
-                                 const InputInfoList &Inputs,
-                                 const ArgList &Args,
-                                 const char *LinkingOutput) const {
+                                   const InputInfo &Output,
+                                   const InputInfoList &Inputs,
+                                   const ArgList &Args,
+                                   const char *LinkingOutput) const {
   const Driver &D = getToolChain().getDriver();
   ArgStringList CmdArgs;
 
@@ -3224,10 +3155,7 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("elf_i386");
   }
 
-  if (Output.isPipe()) {
-    CmdArgs.push_back("-o");
-    CmdArgs.push_back("-");
-  } else if (Output.isFilename()) {
+  if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
@@ -3265,9 +3193,7 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
       D.Diag(clang::diag::err_drv_no_linker_llvm_support)
         << getToolChain().getTripleString();
 
-    if (II.isPipe())
-      CmdArgs.push_back("-");
-    else if (II.isFilename())
+    if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
       II.getInputArg().renderAsInput(Args, CmdArgs);
@@ -3293,6 +3219,11 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("/usr/lib");
     }
 
+    if (D.CCCIsCXX) {
+      CmdArgs.push_back("-lstdc++");
+      CmdArgs.push_back("-lm");
+    }
+
     if (Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back("-lgcc_pic");
     } else {
@@ -3328,5 +3259,46 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *Exec =
     Args.MakeArgString(getToolChain().GetProgramPath("ld"));
-  Dest.addCommand(new Command(JA, *this, Exec, CmdArgs));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
+}
+
+void visualstudio::Link::ConstructJob(Compilation &C, const JobAction &JA,
+                                      const InputInfo &Output,
+                                      const InputInfoList &Inputs,
+                                      const ArgList &Args,
+                                      const char *LinkingOutput) const {
+  const Driver &D = getToolChain().getDriver();
+  ArgStringList CmdArgs;
+
+  if (Output.isFilename()) {
+    CmdArgs.push_back(Args.MakeArgString(std::string("-out:") + Output.getFilename()));
+  } else {
+    assert(Output.isNothing() && "Invalid output.");
+  }
+
+  if (!Args.hasArg(options::OPT_nostdlib) &&
+    !Args.hasArg(options::OPT_nostartfiles)) {
+    CmdArgs.push_back("-defaultlib:libcmt");
+  }
+
+  CmdArgs.push_back("-nologo");
+
+  for (InputInfoList::const_iterator
+    it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
+    const InputInfo &II = *it;
+
+    // Don't try to pass LLVM inputs to visual studio linker.
+    if (II.getType() == types::TY_LLVM_BC)
+      D.Diag(clang::diag::err_drv_no_linker_llvm_support)
+      << getToolChain().getTripleString();
+
+    if (II.isFilename())
+      CmdArgs.push_back(II.getFilename());
+    else
+      II.getInputArg().renderAsInput(Args, CmdArgs);
+  }
+
+  const char *Exec =
+  Args.MakeArgString(getToolChain().GetProgramPath("link.exe"));
+  C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
diff --git a/lib/Driver/Tools.h b/lib/Driver/Tools.h
index 2a181038996b..b5defa4569cf 100644
--- a/lib/Driver/Tools.h
+++ b/lib/Driver/Tools.h
@@ -41,14 +41,11 @@ namespace tools {
   public:
     Clang(const ToolChain &TC) : Tool("clang", "clang frontend", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasGoodDiagnostics() const { return true; }
     virtual bool hasIntegratedAssembler() const { return true; }
     virtual bool hasIntegratedCPP() const { return true; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -61,14 +58,11 @@ namespace tools {
     ClangAs(const ToolChain &TC) : Tool("clang::as",
                                         "clang integrated assembler", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasGoodDiagnostics() const { return true; }
     virtual bool hasIntegratedAssembler() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -83,7 +77,6 @@ namespace gcc {
            const ToolChain &TC) : Tool(Name, ShortName, TC) {}
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -101,8 +94,6 @@ namespace gcc {
     Preprocess(const ToolChain &TC) : Common("gcc::Preprocess",
                                              "gcc preprocessor", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasGoodDiagnostics() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
@@ -115,8 +106,6 @@ namespace gcc {
     Precompile(const ToolChain &TC) : Common("gcc::Precompile",
                                              "gcc precompile", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasGoodDiagnostics() const { return true; }
     virtual bool hasIntegratedCPP() const { return true; }
 
@@ -129,8 +118,6 @@ namespace gcc {
     Compile(const ToolChain &TC) : Common("gcc::Compile",
                                           "gcc frontend", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasGoodDiagnostics() const { return true; }
     virtual bool hasIntegratedCPP() const { return true; }
 
@@ -143,8 +130,6 @@ namespace gcc {
     Assemble(const ToolChain &TC) : Common("gcc::Assemble",
                                            "assembler (via gcc)", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void RenderExtraToolArgs(const JobAction &JA,
@@ -156,8 +141,6 @@ namespace gcc {
     Link(const ToolChain &TC) : Common("gcc::Link",
                                        "linker (via gcc)", TC) {}
 
-    virtual bool acceptsPipedInput() const { return false; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void RenderExtraToolArgs(const JobAction &JA,
@@ -207,8 +190,6 @@ namespace darwin {
     CC1(const char *Name, const char *ShortName,
         const ToolChain &TC) : DarwinTool(Name, ShortName, TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasGoodDiagnostics() const { return true; }
     virtual bool hasIntegratedCPP() const { return true; }
   };
@@ -219,7 +200,6 @@ namespace darwin {
                                           "gcc preprocessor", TC) {}
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -231,7 +211,6 @@ namespace darwin {
     Compile(const ToolChain &TC) : CC1("darwin::Compile", "gcc frontend", TC) {}
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -243,12 +222,9 @@ namespace darwin {
     Assemble(const ToolChain &TC) : DarwinTool("darwin::Assemble",
                                                "assembler", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -261,12 +237,9 @@ namespace darwin {
   public:
     Link(const ToolChain &TC) : DarwinTool("darwin::Link", "linker", TC) {}
 
-    virtual bool acceptsPipedInput() const { return false; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -277,12 +250,9 @@ namespace darwin {
   public:
     Lipo(const ToolChain &TC) : DarwinTool("darwin::Lipo", "lipo", TC) {}
 
-    virtual bool acceptsPipedInput() const { return false; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -294,12 +264,9 @@ namespace darwin {
     Dsymutil(const ToolChain &TC) : DarwinTool("darwin::Dsymutil",
                                                "dsymutil", TC) {}
 
-    virtual bool acceptsPipedInput() const { return false; }
-    virtual bool canPipeOutput() const { return false; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -314,12 +281,9 @@ namespace openbsd {
     Assemble(const ToolChain &TC) : Tool("openbsd::Assemble", "assembler",
                                          TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -329,12 +293,9 @@ namespace openbsd {
   public:
     Link(const ToolChain &TC) : Tool("openbsd::Link", "linker", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -349,12 +310,9 @@ namespace freebsd {
     Assemble(const ToolChain &TC) : Tool("freebsd::Assemble", "assembler",
                                          TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -364,12 +322,9 @@ namespace freebsd {
   public:
     Link(const ToolChain &TC) : Tool("freebsd::Link", "linker", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -377,6 +332,22 @@ namespace freebsd {
   };
 } // end namespace freebsd
 
+  /// linux -- Directly call GNU Binutils assembler and linker
+namespace linuxtools {
+  class LLVM_LIBRARY_VISIBILITY Assemble : public Tool  {
+  public:
+    Assemble(const ToolChain &TC) : Tool("linux::Assemble", "assembler",
+                                         TC) {}
+
+    virtual bool hasIntegratedCPP() const { return false; }
+
+    virtual void ConstructJob(Compilation &C, const JobAction &JA,
+                              const InputInfo &Output,
+                              const InputInfoList &Inputs,
+                              const ArgList &TCArgs,
+                              const char *LinkingOutput) const;
+  };
+}
   /// minix -- Directly call GNU Binutils assembler and linker
 namespace minix {
   class LLVM_LIBRARY_VISIBILITY Assemble : public Tool  {
@@ -384,12 +355,9 @@ namespace minix {
     Assemble(const ToolChain &TC) : Tool("minix::Assemble", "assembler",
                                          TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -399,12 +367,9 @@ namespace minix {
   public:
     Link(const ToolChain &TC) : Tool("minix::Link", "linker", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -419,12 +384,9 @@ namespace auroraux {
     Assemble(const ToolChain &TC) : Tool("auroraux::Assemble", "assembler",
                                          TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -434,12 +396,9 @@ namespace auroraux {
   public:
     Link(const ToolChain &TC) : Tool("auroraux::Link", "linker", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -454,12 +413,9 @@ namespace dragonfly {
     Assemble(const ToolChain &TC) : Tool("dragonfly::Assemble", "assembler",
                                          TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -469,12 +425,9 @@ namespace dragonfly {
   public:
     Link(const ToolChain &TC) : Tool("dragonfly::Link", "linker", TC) {}
 
-    virtual bool acceptsPipedInput() const { return true; }
-    virtual bool canPipeOutput() const { return true; }
     virtual bool hasIntegratedCPP() const { return false; }
 
     virtual void ConstructJob(Compilation &C, const JobAction &JA,
-                              Job &Dest,
                               const InputInfo &Output,
                               const InputInfoList &Inputs,
                               const ArgList &TCArgs,
@@ -482,6 +435,22 @@ namespace dragonfly {
   };
 } // end namespace dragonfly
 
+  /// Visual studio tools.
+namespace visualstudio {
+  class LLVM_LIBRARY_VISIBILITY Link : public Tool  {
+  public:
+    Link(const ToolChain &TC) : Tool("visualstudio::Link", "linker", TC) {}
+
+    virtual bool hasIntegratedCPP() const { return false; }
+
+    virtual void ConstructJob(Compilation &C, const JobAction &JA,
+                              const InputInfo &Output,
+                              const InputInfoList &Inputs,
+                              const ArgList &TCArgs,
+                              const char *LinkingOutput) const;
+  };
+} // end namespace visualstudio
+
 } // end namespace toolchains
 } // end namespace driver
 } // end namespace clang
diff --git a/lib/Frontend/ASTConsumers.cpp b/lib/Frontend/ASTConsumers.cpp
index 87b01d4a6a2a..eb7f270ae8fb 100644
--- a/lib/Frontend/ASTConsumers.cpp
+++ b/lib/Frontend/ASTConsumers.cpp
@@ -37,7 +37,7 @@ namespace {
 
   public:
     ASTPrinter(llvm::raw_ostream* o = NULL, bool Dump = false)
-      : Out(o? *o : llvm::errs()), Dump(Dump) { }
+      : Out(o? *o : llvm::outs()), Dump(Dump) { }
 
     virtual void HandleTranslationUnit(ASTContext &Context) {
       PrintingPolicy Policy = Context.PrintingPolicy;
diff --git a/lib/Frontend/ASTMerge.cpp b/lib/Frontend/ASTMerge.cpp
index e916e200659c..b46212fedaf4 100644
--- a/lib/Frontend/ASTMerge.cpp
+++ b/lib/Frontend/ASTMerge.cpp
@@ -40,10 +40,16 @@ void ASTMergeAction::ExecuteAction() {
                                        &CI.getASTContext());
   llvm::IntrusiveRefCntPtr<Diagnostic> Diags(&CI.getDiagnostics());
   for (unsigned I = 0, N = ASTFiles.size(); I != N; ++I) {
-    ASTUnit *Unit = ASTUnit::LoadFromPCHFile(ASTFiles[I], Diags, false);
+    ASTUnit *Unit = ASTUnit::LoadFromASTFile(ASTFiles[I], Diags, false);
     if (!Unit)
       continue;
 
+    // Reset the argument -> string function so that it has the AST
+    // context we want, since the Sema object created by
+    // LoadFromASTFile will override it.
+    CI.getDiagnostics().SetArgToStringFn(&FormatASTNodeDiagnosticArgument,
+                                         &CI.getASTContext());
+
     ASTImporter Importer(CI.getDiagnostics(),
                          CI.getASTContext(), 
                          CI.getFileManager(),
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 88f00376b722..c76488b2c62c 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -12,10 +12,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/ASTUnit.h"
-#include "clang/Frontend/PCHReader.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/DeclVisitor.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
@@ -25,30 +25,294 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/FrontendOptions.h"
+#include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/TargetOptions.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Diagnostic.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/System/Host.h"
 #include "llvm/System/Path.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Timer.h"
+#include <cstdlib>
+#include <cstdio>
+#include <sys/stat.h>
 using namespace clang;
 
+/// \brief After failing to build a precompiled preamble (due to
+/// errors in the source that occurs in the preamble), the number of
+/// reparses during which we'll skip even trying to precompile the
+/// preamble.
+const unsigned DefaultPreambleRebuildInterval = 5;
+
 ASTUnit::ASTUnit(bool _MainFileIsAST)
-  : MainFileIsAST(_MainFileIsAST), ConcurrencyCheckValue(CheckUnlocked) { }
+  : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), 
+    CompleteTranslationUnit(true), ConcurrencyCheckValue(CheckUnlocked), 
+    PreambleRebuildCounter(0), SavedMainFileBuffer(0), PreambleBuffer(0),
+    ShouldCacheCodeCompletionResults(false),
+    NumTopLevelDeclsAtLastCompletionCache(0),
+    CacheCodeCompletionCoolDown(0),
+    UnsafeToFree(false) { 
+}
 
 ASTUnit::~ASTUnit() {
   ConcurrencyCheckValue = CheckLocked;
+  CleanTemporaryFiles();
+  if (!PreambleFile.empty())
+    llvm::sys::Path(PreambleFile).eraseFromDisk();
+  
+  // Free the buffers associated with remapped files. We are required to
+  // perform this operation here because we explicitly request that the
+  // compiler instance *not* free these buffers for each invocation of the
+  // parser.
+  if (Invocation.get()) {
+    PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
+    for (PreprocessorOptions::remapped_file_buffer_iterator
+           FB = PPOpts.remapped_file_buffer_begin(),
+           FBEnd = PPOpts.remapped_file_buffer_end();
+         FB != FBEnd;
+         ++FB)
+      delete FB->second;
+  }
+  
+  delete SavedMainFileBuffer;
+  delete PreambleBuffer;
+
+  ClearCachedCompletionResults();
+  
+  for (unsigned I = 0, N = Timers.size(); I != N; ++I)
+    delete Timers[I];
+}
+
+void ASTUnit::CleanTemporaryFiles() {
   for (unsigned I = 0, N = TemporaryFiles.size(); I != N; ++I)
     TemporaryFiles[I].eraseFromDisk();
+  TemporaryFiles.clear();
+}
+
+/// \brief Determine the set of code-completion contexts in which this 
+/// declaration should be shown.
+static unsigned getDeclShowContexts(NamedDecl *ND,
+                                    const LangOptions &LangOpts,
+                                    bool &IsNestedNameSpecifier) {
+  IsNestedNameSpecifier = false;
+  
+  if (isa<UsingShadowDecl>(ND))
+    ND = dyn_cast<NamedDecl>(ND->getUnderlyingDecl());
+  if (!ND)
+    return 0;
+  
+  unsigned Contexts = 0;
+  if (isa<TypeDecl>(ND) || isa<ObjCInterfaceDecl>(ND) || 
+      isa<ClassTemplateDecl>(ND) || isa<TemplateTemplateParmDecl>(ND)) {
+    // Types can appear in these contexts.
+    if (LangOpts.CPlusPlus || !isa<TagDecl>(ND))
+      Contexts |= (1 << (CodeCompletionContext::CCC_TopLevel - 1))
+                | (1 << (CodeCompletionContext::CCC_ObjCIvarList - 1))
+                | (1 << (CodeCompletionContext::CCC_ClassStructUnion - 1))
+                | (1 << (CodeCompletionContext::CCC_Statement - 1))
+                | (1 << (CodeCompletionContext::CCC_Type - 1));
+
+    // In C++, types can appear in expressions contexts (for functional casts).
+    if (LangOpts.CPlusPlus)
+      Contexts |= (1 << (CodeCompletionContext::CCC_Expression - 1));
+    
+    // In Objective-C, message sends can send interfaces. In Objective-C++,
+    // all types are available due to functional casts.
+    if (LangOpts.CPlusPlus || isa<ObjCInterfaceDecl>(ND))
+      Contexts |= (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1));
+
+    // Deal with tag names.
+    if (isa<EnumDecl>(ND)) {
+      Contexts |= (1 << (CodeCompletionContext::CCC_EnumTag - 1));
+      
+      // Part of the nested-name-specifier in C++0x.
+      if (LangOpts.CPlusPlus0x)
+        IsNestedNameSpecifier = true;
+    } else if (RecordDecl *Record = dyn_cast<RecordDecl>(ND)) {
+      if (Record->isUnion())
+        Contexts |= (1 << (CodeCompletionContext::CCC_UnionTag - 1));
+      else
+        Contexts |= (1 << (CodeCompletionContext::CCC_ClassOrStructTag - 1));
+      
+      if (LangOpts.CPlusPlus)
+        IsNestedNameSpecifier = true;
+    } else if (isa<ClassTemplateDecl>(ND) || isa<TemplateTemplateParmDecl>(ND))
+      IsNestedNameSpecifier = true;
+  } else if (isa<ValueDecl>(ND) || isa<FunctionTemplateDecl>(ND)) {
+    // Values can appear in these contexts.
+    Contexts = (1 << (CodeCompletionContext::CCC_Statement - 1))
+             | (1 << (CodeCompletionContext::CCC_Expression - 1))
+             | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1));
+  } else if (isa<ObjCProtocolDecl>(ND)) {
+    Contexts = (1 << (CodeCompletionContext::CCC_ObjCProtocolName - 1));
+  } else if (isa<NamespaceDecl>(ND) || isa<NamespaceAliasDecl>(ND)) {
+    Contexts = (1 << (CodeCompletionContext::CCC_Namespace - 1));
+   
+    // Part of the nested-name-specifier.
+    IsNestedNameSpecifier = true;
+  }
+  
+  return Contexts;
+}
+
+void ASTUnit::CacheCodeCompletionResults() {
+  if (!TheSema)
+    return;
+  
+  llvm::Timer *CachingTimer = 0;
+  if (TimerGroup.get()) {
+    CachingTimer = new llvm::Timer("Cache global code completions", 
+                                   *TimerGroup);
+    CachingTimer->startTimer();
+    Timers.push_back(CachingTimer);
+  }
+
+  // Clear out the previous results.
+  ClearCachedCompletionResults();
+  
+  // Gather the set of global code completions.
+  typedef CodeCompletionResult Result;
+  llvm::SmallVector<Result, 8> Results;
+  TheSema->GatherGlobalCodeCompletions(Results);
+  
+  // Translate global code completions into cached completions.
+  llvm::DenseMap<CanQualType, unsigned> CompletionTypes;
+  
+  for (unsigned I = 0, N = Results.size(); I != N; ++I) {
+    switch (Results[I].Kind) {
+    case Result::RK_Declaration: {
+      bool IsNestedNameSpecifier = false;
+      CachedCodeCompletionResult CachedResult;
+      CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema);
+      CachedResult.ShowInContexts = getDeclShowContexts(Results[I].Declaration,
+                                                        Ctx->getLangOptions(),
+                                                        IsNestedNameSpecifier);
+      CachedResult.Priority = Results[I].Priority;
+      CachedResult.Kind = Results[I].CursorKind;
+      CachedResult.Availability = Results[I].Availability;
+
+      // Keep track of the type of this completion in an ASTContext-agnostic 
+      // way.
+      QualType UsageType = getDeclUsageType(*Ctx, Results[I].Declaration);
+      if (UsageType.isNull()) {
+        CachedResult.TypeClass = STC_Void;
+        CachedResult.Type = 0;
+      } else {
+        CanQualType CanUsageType
+          = Ctx->getCanonicalType(UsageType.getUnqualifiedType());
+        CachedResult.TypeClass = getSimplifiedTypeClass(CanUsageType);
+
+        // Determine whether we have already seen this type. If so, we save
+        // ourselves the work of formatting the type string by using the 
+        // temporary, CanQualType-based hash table to find the associated value.
+        unsigned &TypeValue = CompletionTypes[CanUsageType];
+        if (TypeValue == 0) {
+          TypeValue = CompletionTypes.size();
+          CachedCompletionTypes[QualType(CanUsageType).getAsString()]
+            = TypeValue;
+        }
+        
+        CachedResult.Type = TypeValue;
+      }
+      
+      CachedCompletionResults.push_back(CachedResult);
+      
+      /// Handle nested-name-specifiers in C++.
+      if (TheSema->Context.getLangOptions().CPlusPlus && 
+          IsNestedNameSpecifier && !Results[I].StartsNestedNameSpecifier) {
+        // The contexts in which a nested-name-specifier can appear in C++.
+        unsigned NNSContexts
+          = (1 << (CodeCompletionContext::CCC_TopLevel - 1))
+          | (1 << (CodeCompletionContext::CCC_ObjCIvarList - 1))
+          | (1 << (CodeCompletionContext::CCC_ClassStructUnion - 1))
+          | (1 << (CodeCompletionContext::CCC_Statement - 1))
+          | (1 << (CodeCompletionContext::CCC_Expression - 1))
+          | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1))
+          | (1 << (CodeCompletionContext::CCC_EnumTag - 1))
+          | (1 << (CodeCompletionContext::CCC_UnionTag - 1))
+          | (1 << (CodeCompletionContext::CCC_ClassOrStructTag - 1))
+          | (1 << (CodeCompletionContext::CCC_Type - 1))
+          | (1 << (CodeCompletionContext::CCC_PotentiallyQualifiedName - 1));
+
+        if (isa<NamespaceDecl>(Results[I].Declaration) ||
+            isa<NamespaceAliasDecl>(Results[I].Declaration))
+          NNSContexts |= (1 << (CodeCompletionContext::CCC_Namespace - 1));
+
+        if (unsigned RemainingContexts 
+                                = NNSContexts & ~CachedResult.ShowInContexts) {
+          // If there any contexts where this completion can be a 
+          // nested-name-specifier but isn't already an option, create a 
+          // nested-name-specifier completion.
+          Results[I].StartsNestedNameSpecifier = true;
+          CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema);
+          CachedResult.ShowInContexts = RemainingContexts;
+          CachedResult.Priority = CCP_NestedNameSpecifier;
+          CachedResult.TypeClass = STC_Void;
+          CachedResult.Type = 0;
+          CachedCompletionResults.push_back(CachedResult);
+        }
+      }
+      break;
+    }
+        
+    case Result::RK_Keyword:
+    case Result::RK_Pattern:
+      // Ignore keywords and patterns; we don't care, since they are so
+      // easily regenerated.
+      break;
+      
+    case Result::RK_Macro: {
+      CachedCodeCompletionResult CachedResult;
+      CachedResult.Completion = Results[I].CreateCodeCompletionString(*TheSema);
+      CachedResult.ShowInContexts
+        = (1 << (CodeCompletionContext::CCC_TopLevel - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCInterface - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCImplementation - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCIvarList - 1))
+        | (1 << (CodeCompletionContext::CCC_ClassStructUnion - 1))
+        | (1 << (CodeCompletionContext::CCC_Statement - 1))
+        | (1 << (CodeCompletionContext::CCC_Expression - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1))
+        | (1 << (CodeCompletionContext::CCC_MacroNameUse - 1))
+        | (1 << (CodeCompletionContext::CCC_PreprocessorExpression - 1));
+      
+      CachedResult.Priority = Results[I].Priority;
+      CachedResult.Kind = Results[I].CursorKind;
+      CachedResult.Availability = Results[I].Availability;
+      CachedResult.TypeClass = STC_Void;
+      CachedResult.Type = 0;
+      CachedCompletionResults.push_back(CachedResult);
+      break;
+    }
+    }
+    Results[I].Destroy();
+  }
+
+  if (CachingTimer)
+    CachingTimer->stopTimer();
+  
+  // Make a note of the state when we performed this caching.
+  NumTopLevelDeclsAtLastCompletionCache = top_level_size();
+  CacheCodeCompletionCoolDown = 15;
+}
+
+void ASTUnit::ClearCachedCompletionResults() {
+  for (unsigned I = 0, N = CachedCompletionResults.size(); I != N; ++I)
+    delete CachedCompletionResults[I].Completion;
+  CachedCompletionResults.clear();
+  CachedCompletionTypes.clear();
 }
 
 namespace {
 
-/// \brief Gathers information from PCHReader that will be used to initialize
+/// \brief Gathers information from ASTReader that will be used to initialize
 /// a Preprocessor.
-class PCHInfoCollector : public PCHReaderListener {
+class ASTInfoCollector : public ASTReaderListener {
   LangOptions &LangOpt;
   HeaderSearch &HSI;
   std::string &TargetTriple;
@@ -58,7 +322,7 @@ class PCHInfoCollector : public PCHReaderListener {
   unsigned NumHeaderInfos;
 
 public:
-  PCHInfoCollector(LangOptions &LangOpt, HeaderSearch &HSI,
+  ASTInfoCollector(LangOptions &LangOpt, HeaderSearch &HSI,
                    std::string &TargetTriple, std::string &Predefines,
                    unsigned &Counter)
     : LangOpt(LangOpt), HSI(HSI), TargetTriple(TargetTriple),
@@ -115,14 +379,19 @@ class CaptureDroppedDiagnostics {
 public:
   CaptureDroppedDiagnostics(bool RequestCapture, Diagnostic &Diags, 
                            llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiags)
-    : Diags(Diags), Client(StoredDiags), PreviousClient(Diags.getClient()) 
+    : Diags(Diags), Client(StoredDiags), PreviousClient(0)
   {
-    if (RequestCapture || Diags.getClient() == 0)
+    if (RequestCapture || Diags.getClient() == 0) {
+      PreviousClient = Diags.takeClient();
       Diags.setClient(&Client);
+    }
   }
 
   ~CaptureDroppedDiagnostics() {
-    Diags.setClient(PreviousClient);
+    if (Diags.getClient() == &Client) {
+      Diags.takeClient();
+      Diags.setClient(PreviousClient);
+    }
   }
 };
 
@@ -137,12 +406,12 @@ const std::string &ASTUnit::getOriginalSourceFileName() {
   return OriginalSourceFile;
 }
 
-const std::string &ASTUnit::getPCHFileName() {
-  assert(isMainFileAST() && "Not an ASTUnit from a PCH file!");
-  return static_cast<PCHReader *>(Ctx->getExternalSource())->getFileName();
+const std::string &ASTUnit::getASTFileName() {
+  assert(isMainFileAST() && "Not an ASTUnit from an AST file!");
+  return static_cast<ASTReader *>(Ctx->getExternalSource())->getFileName();
 }
 
-ASTUnit *ASTUnit::LoadFromPCHFile(const std::string &Filename,
+ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
                                   bool OnlyLocalDecls,
                                   RemappedFile *RemappedFiles,
@@ -156,13 +425,14 @@ ASTUnit *ASTUnit::LoadFromPCHFile(const std::string &Filename,
     DiagnosticOptions DiagOpts;
     Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
   }
-  
+
+  AST->CaptureDiagnostics = CaptureDiagnostics;
   AST->OnlyLocalDecls = OnlyLocalDecls;
   AST->Diagnostics = Diags;
   AST->FileMgr.reset(new FileManager);
   AST->SourceMgr.reset(new SourceManager(AST->getDiagnostics()));
   AST->HeaderInfo.reset(new HeaderSearch(AST->getFileManager()));
-
+  
   // If requested, capture diagnostics in the ASTUnit.
   CaptureDroppedDiagnostics Capture(CaptureDiagnostics, AST->getDiagnostics(),
                                     AST->StoredDiagnostics);
@@ -194,34 +464,33 @@ ASTUnit *ASTUnit::LoadFromPCHFile(const std::string &Filename,
   std::string Predefines;
   unsigned Counter;
 
-  llvm::OwningPtr<PCHReader> Reader;
-  llvm::OwningPtr<ExternalASTSource> Source;
+  llvm::OwningPtr<ASTReader> Reader;
 
-  Reader.reset(new PCHReader(AST->getSourceManager(), AST->getFileManager(),
+  Reader.reset(new ASTReader(AST->getSourceManager(), AST->getFileManager(),
                              AST->getDiagnostics()));
-  Reader->setListener(new PCHInfoCollector(LangInfo, HeaderInfo, TargetTriple,
+  Reader->setListener(new ASTInfoCollector(LangInfo, HeaderInfo, TargetTriple,
                                            Predefines, Counter));
 
-  switch (Reader->ReadPCH(Filename)) {
-  case PCHReader::Success:
+  switch (Reader->ReadAST(Filename)) {
+  case ASTReader::Success:
     break;
 
-  case PCHReader::Failure:
-  case PCHReader::IgnorePCH:
+  case ASTReader::Failure:
+  case ASTReader::IgnorePCH:
     AST->getDiagnostics().Report(diag::err_fe_unable_to_load_pch);
     return NULL;
   }
 
   AST->OriginalSourceFile = Reader->getOriginalSourceFile();
 
-  // PCH loaded successfully. Now create the preprocessor.
+  // AST file loaded successfully. Now create the preprocessor.
 
   // Get information about the target being compiled for.
   //
-  // FIXME: This is broken, we should store the TargetOptions in the PCH.
+  // FIXME: This is broken, we should store the TargetOptions in the AST file.
   TargetOptions TargetOpts;
   TargetOpts.ABI = "";
-  TargetOpts.CXXABI = "itanium";
+  TargetOpts.CXXABI = "";
   TargetOpts.CPU = "";
   TargetOpts.Features.clear();
   TargetOpts.Triple = TargetTriple;
@@ -244,18 +513,26 @@ ASTUnit *ASTUnit::LoadFromPCHFile(const std::string &Filename,
                                 PP.getIdentifierTable(),
                                 PP.getSelectorTable(),
                                 PP.getBuiltinInfo(),
-                                /* FreeMemory = */ false,
                                 /* size_reserve = */0));
   ASTContext &Context = *AST->Ctx.get();
 
   Reader->InitializeContext(Context);
 
-  // Attach the PCH reader to the AST context as an external AST
+  // Attach the AST reader to the AST context as an external AST
   // source, so that declarations will be deserialized from the
-  // PCH file as needed.
-  Source.reset(Reader.take());
+  // AST file as needed.
+  ASTReader *ReaderPtr = Reader.get();
+  llvm::OwningPtr<ExternalASTSource> Source(Reader.take());
   Context.setExternalSource(Source);
 
+  // Create an AST consumer, even though it isn't used.
+  AST->Consumer.reset(new ASTConsumer);
+  
+  // Create a semantic analysis object and tell the AST reader about it.
+  AST->TheSema.reset(new Sema(PP, Context, *AST->Consumer));
+  AST->TheSema->Initialize();
+  ReaderPtr->InitializeSema(*AST->TheSema);
+
   return AST.take();
 }
 
@@ -276,9 +553,12 @@ public:
       // fundamental problem in the parser right now.
       if (isa<ObjCMethodDecl>(D))
         continue;
-      Unit.getTopLevelDecls().push_back(D);
+      Unit.addTopLevelDecl(D);
     }
   }
+
+  // We're not interested in "interesting" decls.
+  void HandleInterestingDecl(DeclGroupRef) {}
 };
 
 class TopLevelDeclTrackerAction : public ASTFrontendAction {
@@ -294,37 +574,108 @@ public:
   TopLevelDeclTrackerAction(ASTUnit &_Unit) : Unit(_Unit) {}
 
   virtual bool hasCodeCompletionSupport() const { return false; }
+  virtual bool usesCompleteTranslationUnit()  { 
+    return Unit.isCompleteTranslationUnit(); 
+  }
 };
 
-}
+class PrecompilePreambleConsumer : public PCHGenerator {
+  ASTUnit &Unit;
+  std::vector<Decl *> TopLevelDecls;
 
-ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
-                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
-                                             bool OnlyLocalDecls,
-                                             bool CaptureDiagnostics) {
-  // Create the compiler instance to use for building the AST.
-  CompilerInstance Clang;
-  llvm::OwningPtr<ASTUnit> AST;
-  llvm::OwningPtr<TopLevelDeclTrackerAction> Act;
+public:
+  PrecompilePreambleConsumer(ASTUnit &Unit,
+                             const Preprocessor &PP, bool Chaining,
+                             const char *isysroot, llvm::raw_ostream *Out)
+    : PCHGenerator(PP, Chaining, isysroot, Out), Unit(Unit) { }
 
-  if (!Diags.getPtr()) {
-    // No diagnostics engine was provided, so create our own diagnostics object
-    // with the default options.
-    DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
+  virtual void HandleTopLevelDecl(DeclGroupRef D) {
+    for (DeclGroupRef::iterator it = D.begin(), ie = D.end(); it != ie; ++it) {
+      Decl *D = *it;
+      // FIXME: Currently ObjC method declarations are incorrectly being
+      // reported as top-level declarations, even though their DeclContext
+      // is the containing ObjC @interface/@implementation.  This is a
+      // fundamental problem in the parser right now.
+      if (isa<ObjCMethodDecl>(D))
+        continue;
+      TopLevelDecls.push_back(D);
+    }
   }
-  
-  Clang.setInvocation(CI);
 
-  Clang.setDiagnostics(Diags.getPtr());
-  Clang.setDiagnosticClient(Diags->getClient());
+  virtual void HandleTranslationUnit(ASTContext &Ctx) {
+    PCHGenerator::HandleTranslationUnit(Ctx);
+    if (!Unit.getDiagnostics().hasErrorOccurred()) {
+      // Translate the top-level declarations we captured during
+      // parsing into declaration IDs in the precompiled
+      // preamble. This will allow us to deserialize those top-level
+      // declarations when requested.
+      for (unsigned I = 0, N = TopLevelDecls.size(); I != N; ++I)
+        Unit.addTopLevelDeclFromPreamble(
+                                      getWriter().getDeclID(TopLevelDecls[I]));
+    }
+  }
+};
+
+class PrecompilePreambleAction : public ASTFrontendAction {
+  ASTUnit &Unit;
+
+public:
+  explicit PrecompilePreambleAction(ASTUnit &Unit) : Unit(Unit) {}
+
+  virtual ASTConsumer *CreateASTConsumer(CompilerInstance &CI,
+                                         llvm::StringRef InFile) {
+    std::string Sysroot;
+    llvm::raw_ostream *OS = 0;
+    bool Chaining;
+    if (GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot, 
+                                                       OS, Chaining))
+      return 0;
+    
+    const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
+                             Sysroot.c_str() : 0;  
+    return new PrecompilePreambleConsumer(Unit, CI.getPreprocessor(), Chaining,
+                                          isysroot, OS);
+  }
+
+  virtual bool hasCodeCompletionSupport() const { return false; }
+  virtual bool hasASTFileSupport() const { return false; }
+  virtual bool usesCompleteTranslationUnit() { return false; }
+};
+
+}
 
+/// Parse the source file into a translation unit using the given compiler
+/// invocation, replacing the current translation unit.
+///
+/// \returns True if a failure occurred that causes the ASTUnit not to
+/// contain any translation-unit information, false otherwise.
+bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
+  delete SavedMainFileBuffer;
+  SavedMainFileBuffer = 0;
+  
+  if (!Invocation.get()) {
+    delete OverrideMainBuffer;
+    return true;
+  }
+  
+  // Create the compiler instance to use for building the AST.
+  CompilerInstance Clang;
+  Clang.setInvocation(Invocation.take());
+  OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second;
+    
+  // Set up diagnostics, capturing any diagnostics that would
+  // otherwise be dropped.
+  Clang.setDiagnostics(&getDiagnostics());
+  CaptureDroppedDiagnostics Capture(CaptureDiagnostics, 
+                                    getDiagnostics(),
+                                    StoredDiagnostics);
+  
   // Create the target instance.
   Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
                                                Clang.getTargetOpts()));
   if (!Clang.hasTarget()) {
-    Clang.takeDiagnosticClient();
-    return 0;
+    delete OverrideMainBuffer;
+    return true;
   }
 
   // Inform the target of the language options.
@@ -332,7 +683,7 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   // FIXME: We shouldn't need to do this, the target should be immutable once
   // created. This complexity should be lifted elsewhere.
   Clang.getTarget().setForcedLangOptions(Clang.getLangOpts());
-
+  
   assert(Clang.getFrontendOpts().Inputs.size() == 1 &&
          "Invocation must have exactly one source file!");
   assert(Clang.getFrontendOpts().Inputs[0].first != IK_AST &&
@@ -340,53 +691,649 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   assert(Clang.getFrontendOpts().Inputs[0].first != IK_LLVM_IR &&
          "IR inputs not support here!");
 
-  // Create the AST unit.
-  AST.reset(new ASTUnit(false));
-  AST->Diagnostics = Diags;
-  AST->FileMgr.reset(new FileManager);
-  AST->SourceMgr.reset(new SourceManager(AST->getDiagnostics()));
-  AST->OnlyLocalDecls = OnlyLocalDecls;
-  AST->OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second;
-
-  // Capture any diagnostics that would otherwise be dropped.
-  CaptureDroppedDiagnostics Capture(CaptureDiagnostics, 
-                                    Clang.getDiagnostics(),
-                                    AST->StoredDiagnostics);
+  // Configure the various subsystems.
+  // FIXME: Should we retain the previous file manager?
+  FileMgr.reset(new FileManager);
+  SourceMgr.reset(new SourceManager(getDiagnostics()));
+  TheSema.reset();
+  Ctx.reset();
+  PP.reset();
+  
+  // Clear out old caches and data.
+  TopLevelDecls.clear();
+  CleanTemporaryFiles();
+  PreprocessedEntitiesByFile.clear();
+
+  if (!OverrideMainBuffer) {
+    StoredDiagnostics.clear();
+    TopLevelDeclsInPreamble.clear();
+  }
 
   // Create a file manager object to provide access to and cache the filesystem.
-  Clang.setFileManager(&AST->getFileManager());
-
+  Clang.setFileManager(&getFileManager());
+  
   // Create the source manager.
-  Clang.setSourceManager(&AST->getSourceManager());
-
-  Act.reset(new TopLevelDeclTrackerAction(*AST));
+  Clang.setSourceManager(&getSourceManager());
+  
+  // If the main file has been overridden due to the use of a preamble,
+  // make that override happen and introduce the preamble.
+  PreprocessorOptions &PreprocessorOpts = Clang.getPreprocessorOpts();
+  std::string PriorImplicitPCHInclude;
+  if (OverrideMainBuffer) {
+    PreprocessorOpts.addRemappedFile(OriginalSourceFile, OverrideMainBuffer);
+    PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size();
+    PreprocessorOpts.PrecompiledPreambleBytes.second
+                                                    = PreambleEndsAtStartOfLine;
+    PriorImplicitPCHInclude = PreprocessorOpts.ImplicitPCHInclude;
+    PreprocessorOpts.ImplicitPCHInclude = PreambleFile;
+    PreprocessorOpts.DisablePCHValidation = true;
+    
+    // Keep track of the override buffer;
+    SavedMainFileBuffer = OverrideMainBuffer;
+
+    // The stored diagnostic has the old source manager in it; update
+    // the locations to refer into the new source manager. Since we've
+    // been careful to make sure that the source manager's state
+    // before and after are identical, so that we can reuse the source
+    // location itself.
+    for (unsigned I = 0, N = StoredDiagnostics.size(); I != N; ++I) {
+      FullSourceLoc Loc(StoredDiagnostics[I].getLocation(),
+                        getSourceManager());
+      StoredDiagnostics[I].setLocation(Loc);
+    }
+  } else {
+    PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
+    PreprocessorOpts.PrecompiledPreambleBytes.second = false;
+  }
+  
+  llvm::OwningPtr<TopLevelDeclTrackerAction> Act;
+  Act.reset(new TopLevelDeclTrackerAction(*this));
   if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second,
                             Clang.getFrontendOpts().Inputs[0].first))
     goto error;
-
+  
   Act->Execute();
-
+  
   // Steal the created target, context, and preprocessor, and take back the
   // source and file managers.
-  AST->Ctx.reset(Clang.takeASTContext());
-  AST->PP.reset(Clang.takePreprocessor());
+  TheSema.reset(Clang.takeSema());
+  Consumer.reset(Clang.takeASTConsumer());
+  Ctx.reset(Clang.takeASTContext());
+  PP.reset(Clang.takePreprocessor());
   Clang.takeSourceManager();
   Clang.takeFileManager();
-  AST->Target.reset(Clang.takeTarget());
-
+  Target.reset(Clang.takeTarget());
+  
   Act->EndSourceFile();
 
-  Clang.takeDiagnosticClient();
-  Clang.takeInvocation();
-
-  AST->Invocation.reset(Clang.takeInvocation());
-  return AST.take();
+  // Remove the overridden buffer we used for the preamble.
+  if (OverrideMainBuffer) {
+    PreprocessorOpts.eraseRemappedFile(
+                               PreprocessorOpts.remapped_file_buffer_end() - 1);
+    PreprocessorOpts.ImplicitPCHInclude = PriorImplicitPCHInclude;
+  }
 
+  Invocation.reset(Clang.takeInvocation());
+  
+  // If we were asked to cache code-completion results and don't have any
+  // results yet, do so now.
+  if (ShouldCacheCodeCompletionResults && CachedCompletionResults.empty())
+    CacheCodeCompletionResults();
+  
+  return false;
+  
 error:
+  // Remove the overridden buffer we used for the preamble.
+  if (OverrideMainBuffer) {
+    PreprocessorOpts.eraseRemappedFile(
+                               PreprocessorOpts.remapped_file_buffer_end() - 1);
+    PreprocessorOpts.DisablePCHValidation = true;
+    PreprocessorOpts.ImplicitPCHInclude = PriorImplicitPCHInclude;
+    delete OverrideMainBuffer;
+  }
+  
   Clang.takeSourceManager();
   Clang.takeFileManager();
-  Clang.takeDiagnosticClient();
-  return 0;
+  Invocation.reset(Clang.takeInvocation());
+  return true;
+}
+
+/// \brief Simple function to retrieve a path for a preamble precompiled header.
+static std::string GetPreamblePCHPath() {
+  // FIXME: This is lame; sys::Path should provide this function (in particular,
+  // it should know how to find the temporary files dir).
+  // FIXME: This is really lame. I copied this code from the Driver!
+  std::string Error;
+  const char *TmpDir = ::getenv("TMPDIR");
+  if (!TmpDir)
+    TmpDir = ::getenv("TEMP");
+  if (!TmpDir)
+    TmpDir = ::getenv("TMP");
+  if (!TmpDir)
+    TmpDir = "/tmp";
+  llvm::sys::Path P(TmpDir);
+  P.appendComponent("preamble");
+  P.appendSuffix("pch");
+  if (P.createTemporaryFileOnDisk())
+    return std::string();
+  
+  return P.str();
+}
+
+/// \brief Compute the preamble for the main file, providing the source buffer
+/// that corresponds to the main file along with a pair (bytes, start-of-line)
+/// that describes the preamble.
+std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> > 
+ASTUnit::ComputePreamble(CompilerInvocation &Invocation, 
+                         unsigned MaxLines, bool &CreatedBuffer) {
+  FrontendOptions &FrontendOpts = Invocation.getFrontendOpts();
+  PreprocessorOptions &PreprocessorOpts
+    = Invocation.getPreprocessorOpts();
+  CreatedBuffer = false;
+  
+  // Try to determine if the main file has been remapped, either from the 
+  // command line (to another file) or directly through the compiler invocation
+  // (to a memory buffer).
+  llvm::MemoryBuffer *Buffer = 0;
+  llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].second);
+  if (const llvm::sys::FileStatus *MainFileStatus = MainFilePath.getFileStatus()) {
+    // Check whether there is a file-file remapping of the main file
+    for (PreprocessorOptions::remapped_file_iterator
+          M = PreprocessorOpts.remapped_file_begin(),
+          E = PreprocessorOpts.remapped_file_end();
+         M != E;
+         ++M) {
+      llvm::sys::PathWithStatus MPath(M->first);    
+      if (const llvm::sys::FileStatus *MStatus = MPath.getFileStatus()) {
+        if (MainFileStatus->uniqueID == MStatus->uniqueID) {
+          // We found a remapping. Try to load the resulting, remapped source.
+          if (CreatedBuffer) {
+            delete Buffer;
+            CreatedBuffer = false;
+          }
+          
+          Buffer = llvm::MemoryBuffer::getFile(M->second);
+          if (!Buffer)
+            return std::make_pair((llvm::MemoryBuffer*)0, 
+                                  std::make_pair(0, true));
+          CreatedBuffer = true;
+          
+          // Remove this remapping. We've captured the buffer already.
+          M = PreprocessorOpts.eraseRemappedFile(M);
+          E = PreprocessorOpts.remapped_file_end();
+          if (M == E)
+            break;
+        }
+      }
+    }
+    
+    // Check whether there is a file-buffer remapping. It supercedes the
+    // file-file remapping.
+    for (PreprocessorOptions::remapped_file_buffer_iterator
+           M = PreprocessorOpts.remapped_file_buffer_begin(),
+           E = PreprocessorOpts.remapped_file_buffer_end();
+         M != E;
+         ++M) {
+      llvm::sys::PathWithStatus MPath(M->first);    
+      if (const llvm::sys::FileStatus *MStatus = MPath.getFileStatus()) {
+        if (MainFileStatus->uniqueID == MStatus->uniqueID) {
+          // We found a remapping. 
+          if (CreatedBuffer) {
+            delete Buffer;
+            CreatedBuffer = false;
+          }
+          
+          Buffer = const_cast<llvm::MemoryBuffer *>(M->second);
+
+          // Remove this remapping. We've captured the buffer already.
+          M = PreprocessorOpts.eraseRemappedFile(M);
+          E = PreprocessorOpts.remapped_file_buffer_end();
+          if (M == E)
+            break;
+        }
+      }
+    }
+  }
+  
+  // If the main source file was not remapped, load it now.
+  if (!Buffer) {
+    Buffer = llvm::MemoryBuffer::getFile(FrontendOpts.Inputs[0].second);
+    if (!Buffer)
+      return std::make_pair((llvm::MemoryBuffer*)0, std::make_pair(0, true));    
+    
+    CreatedBuffer = true;
+  }
+  
+  return std::make_pair(Buffer, Lexer::ComputePreamble(Buffer, MaxLines));
+}
+
+static llvm::MemoryBuffer *CreatePaddedMainFileBuffer(llvm::MemoryBuffer *Old,
+                                                      bool DeleteOld,
+                                                      unsigned NewSize,
+                                                      llvm::StringRef NewName) {
+  llvm::MemoryBuffer *Result
+    = llvm::MemoryBuffer::getNewUninitMemBuffer(NewSize, NewName);
+  memcpy(const_cast<char*>(Result->getBufferStart()), 
+         Old->getBufferStart(), Old->getBufferSize());
+  memset(const_cast<char*>(Result->getBufferStart()) + Old->getBufferSize(), 
+         ' ', NewSize - Old->getBufferSize() - 1);
+  const_cast<char*>(Result->getBufferEnd())[-1] = '\n';  
+  
+  if (DeleteOld)
+    delete Old;
+  
+  return Result;
+}
+
+/// \brief Attempt to build or re-use a precompiled preamble when (re-)parsing
+/// the source file.
+///
+/// This routine will compute the preamble of the main source file. If a
+/// non-trivial preamble is found, it will precompile that preamble into a 
+/// precompiled header so that the precompiled preamble can be used to reduce
+/// reparsing time. If a precompiled preamble has already been constructed,
+/// this routine will determine if it is still valid and, if so, avoid 
+/// rebuilding the precompiled preamble.
+///
+/// \param AllowRebuild When true (the default), this routine is
+/// allowed to rebuild the precompiled preamble if it is found to be
+/// out-of-date.
+///
+/// \param MaxLines When non-zero, the maximum number of lines that
+/// can occur within the preamble.
+///
+/// \returns If the precompiled preamble can be used, returns a newly-allocated
+/// buffer that should be used in place of the main file when doing so.
+/// Otherwise, returns a NULL pointer.
+llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
+                                          CompilerInvocation PreambleInvocation,
+                                                           bool AllowRebuild,
+                                                           unsigned MaxLines) {
+  FrontendOptions &FrontendOpts = PreambleInvocation.getFrontendOpts();
+  PreprocessorOptions &PreprocessorOpts
+    = PreambleInvocation.getPreprocessorOpts();
+
+  bool CreatedPreambleBuffer = false;
+  std::pair<llvm::MemoryBuffer *, std::pair<unsigned, bool> > NewPreamble 
+    = ComputePreamble(PreambleInvocation, MaxLines, CreatedPreambleBuffer);
+
+  if (!NewPreamble.second.first) {
+    // We couldn't find a preamble in the main source. Clear out the current
+    // preamble, if we have one. It's obviously no good any more.
+    Preamble.clear();
+    if (!PreambleFile.empty()) {
+      llvm::sys::Path(PreambleFile).eraseFromDisk();
+      PreambleFile.clear();
+    }
+    if (CreatedPreambleBuffer)
+      delete NewPreamble.first;
+
+    // The next time we actually see a preamble, precompile it.
+    PreambleRebuildCounter = 1;
+    return 0;
+  }
+  
+  if (!Preamble.empty()) {
+    // We've previously computed a preamble. Check whether we have the same
+    // preamble now that we did before, and that there's enough space in
+    // the main-file buffer within the precompiled preamble to fit the
+    // new main file.
+    if (Preamble.size() == NewPreamble.second.first &&
+        PreambleEndsAtStartOfLine == NewPreamble.second.second &&
+        NewPreamble.first->getBufferSize() < PreambleReservedSize-2 &&
+        memcmp(&Preamble[0], NewPreamble.first->getBufferStart(),
+               NewPreamble.second.first) == 0) {
+      // The preamble has not changed. We may be able to re-use the precompiled
+      // preamble.
+
+      // Check that none of the files used by the preamble have changed.
+      bool AnyFileChanged = false;
+          
+      // First, make a record of those files that have been overridden via
+      // remapping or unsaved_files.
+      llvm::StringMap<std::pair<off_t, time_t> > OverriddenFiles;
+      for (PreprocessorOptions::remapped_file_iterator
+                R = PreprocessorOpts.remapped_file_begin(),
+             REnd = PreprocessorOpts.remapped_file_end();
+           !AnyFileChanged && R != REnd;
+           ++R) {
+        struct stat StatBuf;
+        if (stat(R->second.c_str(), &StatBuf)) {
+          // If we can't stat the file we're remapping to, assume that something
+          // horrible happened.
+          AnyFileChanged = true;
+          break;
+        }
+        
+        OverriddenFiles[R->first] = std::make_pair(StatBuf.st_size, 
+                                                   StatBuf.st_mtime);
+      }
+      for (PreprocessorOptions::remapped_file_buffer_iterator
+                R = PreprocessorOpts.remapped_file_buffer_begin(),
+             REnd = PreprocessorOpts.remapped_file_buffer_end();
+           !AnyFileChanged && R != REnd;
+           ++R) {
+        // FIXME: Should we actually compare the contents of file->buffer
+        // remappings?
+        OverriddenFiles[R->first] = std::make_pair(R->second->getBufferSize(), 
+                                                   0);
+      }
+       
+      // Check whether anything has changed.
+      for (llvm::StringMap<std::pair<off_t, time_t> >::iterator 
+             F = FilesInPreamble.begin(), FEnd = FilesInPreamble.end();
+           !AnyFileChanged && F != FEnd; 
+           ++F) {
+        llvm::StringMap<std::pair<off_t, time_t> >::iterator Overridden
+          = OverriddenFiles.find(F->first());
+        if (Overridden != OverriddenFiles.end()) {
+          // This file was remapped; check whether the newly-mapped file 
+          // matches up with the previous mapping.
+          if (Overridden->second != F->second)
+            AnyFileChanged = true;
+          continue;
+        }
+        
+        // The file was not remapped; check whether it has changed on disk.
+        struct stat StatBuf;
+        if (stat(F->first(), &StatBuf)) {
+          // If we can't stat the file, assume that something horrible happened.
+          AnyFileChanged = true;
+        } else if (StatBuf.st_size != F->second.first || 
+                   StatBuf.st_mtime != F->second.second)
+          AnyFileChanged = true;
+      }
+          
+      if (!AnyFileChanged) {
+        // Okay! We can re-use the precompiled preamble.
+
+        // Set the state of the diagnostic object to mimic its state
+        // after parsing the preamble.
+        getDiagnostics().Reset();
+        getDiagnostics().setNumWarnings(NumWarningsInPreamble);
+        if (StoredDiagnostics.size() > NumStoredDiagnosticsInPreamble)
+          StoredDiagnostics.erase(
+            StoredDiagnostics.begin() + NumStoredDiagnosticsInPreamble,
+                                  StoredDiagnostics.end());
+
+        // Create a version of the main file buffer that is padded to
+        // buffer size we reserved when creating the preamble.
+        return CreatePaddedMainFileBuffer(NewPreamble.first, 
+                                          CreatedPreambleBuffer,
+                                          PreambleReservedSize,
+                                          FrontendOpts.Inputs[0].second);
+      }
+    }
+
+    // If we aren't allowed to rebuild the precompiled preamble, just
+    // return now.
+    if (!AllowRebuild)
+      return 0;
+    
+    // We can't reuse the previously-computed preamble. Build a new one.
+    Preamble.clear();
+    llvm::sys::Path(PreambleFile).eraseFromDisk();
+    PreambleRebuildCounter = 1;
+  } else if (!AllowRebuild) {
+    // We aren't allowed to rebuild the precompiled preamble; just
+    // return now.
+    return 0;
+  }
+
+  // If the preamble rebuild counter > 1, it's because we previously
+  // failed to build a preamble and we're not yet ready to try
+  // again. Decrement the counter and return a failure.
+  if (PreambleRebuildCounter > 1) {
+    --PreambleRebuildCounter;
+    return 0;
+  }
+
+  // We did not previously compute a preamble, or it can't be reused anyway.
+  llvm::Timer *PreambleTimer = 0;
+  if (TimerGroup.get()) {
+    PreambleTimer = new llvm::Timer("Precompiling preamble", *TimerGroup);
+    PreambleTimer->startTimer();
+    Timers.push_back(PreambleTimer);
+  }
+  
+  // Create a new buffer that stores the preamble. The buffer also contains
+  // extra space for the original contents of the file (which will be present
+  // when we actually parse the file) along with more room in case the file
+  // grows.  
+  PreambleReservedSize = NewPreamble.first->getBufferSize();
+  if (PreambleReservedSize < 4096)
+    PreambleReservedSize = 8191;
+  else
+    PreambleReservedSize *= 2;
+
+  // Save the preamble text for later; we'll need to compare against it for
+  // subsequent reparses.
+  Preamble.assign(NewPreamble.first->getBufferStart(), 
+                  NewPreamble.first->getBufferStart() 
+                                                  + NewPreamble.second.first);
+  PreambleEndsAtStartOfLine = NewPreamble.second.second;
+
+  delete PreambleBuffer;
+  PreambleBuffer
+    = llvm::MemoryBuffer::getNewUninitMemBuffer(PreambleReservedSize,
+                                                FrontendOpts.Inputs[0].second);
+  memcpy(const_cast<char*>(PreambleBuffer->getBufferStart()), 
+         NewPreamble.first->getBufferStart(), Preamble.size());
+  memset(const_cast<char*>(PreambleBuffer->getBufferStart()) + Preamble.size(), 
+         ' ', PreambleReservedSize - Preamble.size() - 1);
+  const_cast<char*>(PreambleBuffer->getBufferEnd())[-1] = '\n';  
+  
+  // Remap the main source file to the preamble buffer.
+  llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].second);
+  PreprocessorOpts.addRemappedFile(MainFilePath.str(), PreambleBuffer);
+  
+  // Tell the compiler invocation to generate a temporary precompiled header.
+  FrontendOpts.ProgramAction = frontend::GeneratePCH;
+  // FIXME: Set ChainedPCH unconditionally, once it is ready.
+  if (::getenv("LIBCLANG_CHAINING"))
+    FrontendOpts.ChainedPCH = true;
+  // FIXME: Generate the precompiled header into memory?
+  FrontendOpts.OutputFile = GetPreamblePCHPath();
+  
+  // Create the compiler instance to use for building the precompiled preamble.
+  CompilerInstance Clang;
+  Clang.setInvocation(&PreambleInvocation);
+  OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second;
+  
+  // Set up diagnostics, capturing all of the diagnostics produced.
+  Clang.setDiagnostics(&getDiagnostics());
+  CaptureDroppedDiagnostics Capture(CaptureDiagnostics, 
+                                    getDiagnostics(),
+                                    StoredDiagnostics);
+  
+  // Create the target instance.
+  Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
+                                               Clang.getTargetOpts()));
+  if (!Clang.hasTarget()) {
+    llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
+    Preamble.clear();
+    if (CreatedPreambleBuffer)
+      delete NewPreamble.first;
+    if (PreambleTimer)
+      PreambleTimer->stopTimer();
+    PreambleRebuildCounter = DefaultPreambleRebuildInterval;
+    PreprocessorOpts.eraseRemappedFile(
+                               PreprocessorOpts.remapped_file_buffer_end() - 1);
+    return 0;
+  }
+  
+  // Inform the target of the language options.
+  //
+  // FIXME: We shouldn't need to do this, the target should be immutable once
+  // created. This complexity should be lifted elsewhere.
+  Clang.getTarget().setForcedLangOptions(Clang.getLangOpts());
+  
+  assert(Clang.getFrontendOpts().Inputs.size() == 1 &&
+         "Invocation must have exactly one source file!");
+  assert(Clang.getFrontendOpts().Inputs[0].first != IK_AST &&
+         "FIXME: AST inputs not yet supported here!");
+  assert(Clang.getFrontendOpts().Inputs[0].first != IK_LLVM_IR &&
+         "IR inputs not support here!");
+  
+  // Clear out old caches and data.
+  StoredDiagnostics.clear();
+  TopLevelDecls.clear();
+  TopLevelDeclsInPreamble.clear();
+  
+  // Create a file manager object to provide access to and cache the filesystem.
+  Clang.setFileManager(new FileManager);
+  
+  // Create the source manager.
+  Clang.setSourceManager(new SourceManager(getDiagnostics()));
+  
+  llvm::OwningPtr<PrecompilePreambleAction> Act;
+  Act.reset(new PrecompilePreambleAction(*this));
+  if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second,
+                            Clang.getFrontendOpts().Inputs[0].first)) {
+    Clang.takeInvocation();
+    llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
+    Preamble.clear();
+    if (CreatedPreambleBuffer)
+      delete NewPreamble.first;
+    if (PreambleTimer)
+      PreambleTimer->stopTimer();
+    PreambleRebuildCounter = DefaultPreambleRebuildInterval;
+    PreprocessorOpts.eraseRemappedFile(
+                               PreprocessorOpts.remapped_file_buffer_end() - 1);
+    return 0;
+  }
+  
+  Act->Execute();
+  Act->EndSourceFile();
+  Clang.takeInvocation();
+  
+  if (Diagnostics->hasErrorOccurred()) {
+    // There were errors parsing the preamble, so no precompiled header was
+    // generated. Forget that we even tried.
+    // FIXME: Should we leave a note for ourselves to try again?
+    llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
+    Preamble.clear();
+    if (CreatedPreambleBuffer)
+      delete NewPreamble.first;
+    if (PreambleTimer)
+      PreambleTimer->stopTimer();
+    TopLevelDeclsInPreamble.clear();
+    PreambleRebuildCounter = DefaultPreambleRebuildInterval;
+    PreprocessorOpts.eraseRemappedFile(
+                               PreprocessorOpts.remapped_file_buffer_end() - 1);
+    return 0;
+  }
+  
+  // Keep track of the preamble we precompiled.
+  PreambleFile = FrontendOpts.OutputFile;
+  NumStoredDiagnosticsInPreamble = StoredDiagnostics.size();
+  NumWarningsInPreamble = getDiagnostics().getNumWarnings();
+  
+  // Keep track of all of the files that the source manager knows about,
+  // so we can verify whether they have changed or not.
+  FilesInPreamble.clear();
+  SourceManager &SourceMgr = Clang.getSourceManager();
+  const llvm::MemoryBuffer *MainFileBuffer
+    = SourceMgr.getBuffer(SourceMgr.getMainFileID());
+  for (SourceManager::fileinfo_iterator F = SourceMgr.fileinfo_begin(),
+                                     FEnd = SourceMgr.fileinfo_end();
+       F != FEnd;
+       ++F) {
+    const FileEntry *File = F->second->Entry;
+    if (!File || F->second->getRawBuffer() == MainFileBuffer)
+      continue;
+    
+    FilesInPreamble[File->getName()]
+      = std::make_pair(F->second->getSize(), File->getModificationTime());
+  }
+  
+  if (PreambleTimer)
+    PreambleTimer->stopTimer();
+  
+  PreambleRebuildCounter = 1;
+  PreprocessorOpts.eraseRemappedFile(
+                               PreprocessorOpts.remapped_file_buffer_end() - 1);
+  return CreatePaddedMainFileBuffer(NewPreamble.first, 
+                                    CreatedPreambleBuffer,
+                                    PreambleReservedSize,
+                                    FrontendOpts.Inputs[0].second);
+}
+
+void ASTUnit::RealizeTopLevelDeclsFromPreamble() {
+  std::vector<Decl *> Resolved;
+  Resolved.reserve(TopLevelDeclsInPreamble.size());
+  ExternalASTSource &Source = *getASTContext().getExternalSource();
+  for (unsigned I = 0, N = TopLevelDeclsInPreamble.size(); I != N; ++I) {
+    // Resolve the declaration ID to an actual declaration, possibly
+    // deserializing the declaration in the process.
+    Decl *D = Source.GetExternalDecl(TopLevelDeclsInPreamble[I]);
+    if (D)
+      Resolved.push_back(D);
+  }
+  TopLevelDeclsInPreamble.clear();
+  TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end());
+}
+
+unsigned ASTUnit::getMaxPCHLevel() const {
+  if (!getOnlyLocalDecls())
+    return Decl::MaxPCHLevel;
+
+  unsigned Result = 0;
+  if (isMainFileAST() || SavedMainFileBuffer)
+    ++Result;
+  return Result;
+}
+
+ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
+                                   llvm::IntrusiveRefCntPtr<Diagnostic> Diags,
+                                             bool OnlyLocalDecls,
+                                             bool CaptureDiagnostics,
+                                             bool PrecompilePreamble,
+                                             bool CompleteTranslationUnit,
+                                             bool CacheCodeCompletionResults) {
+  if (!Diags.getPtr()) {
+    // No diagnostics engine was provided, so create our own diagnostics object
+    // with the default options.
+    DiagnosticOptions DiagOpts;
+    Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
+  }
+  
+  // Create the AST unit.
+  llvm::OwningPtr<ASTUnit> AST;
+  AST.reset(new ASTUnit(false));
+  AST->Diagnostics = Diags;
+  AST->CaptureDiagnostics = CaptureDiagnostics;
+  AST->OnlyLocalDecls = OnlyLocalDecls;
+  AST->CompleteTranslationUnit = CompleteTranslationUnit;
+  AST->ShouldCacheCodeCompletionResults = CacheCodeCompletionResults;
+  AST->Invocation.reset(CI);
+  CI->getPreprocessorOpts().RetainRemappedFileBuffers = true;
+  
+  if (getenv("LIBCLANG_TIMING"))
+    AST->TimerGroup.reset(
+                  new llvm::TimerGroup(CI->getFrontendOpts().Inputs[0].second));
+  
+  
+  llvm::MemoryBuffer *OverrideMainBuffer = 0;
+  // FIXME: When C++ PCH is ready, allow use of it for a precompiled preamble.
+  if (PrecompilePreamble && !CI->getLangOpts().CPlusPlus) {
+    AST->PreambleRebuildCounter = 1;
+    OverrideMainBuffer
+      = AST->getMainBufferWithPrecompiledPreamble(*AST->Invocation);
+  }
+  
+  llvm::Timer *ParsingTimer = 0;
+  if (AST->TimerGroup.get()) {
+    ParsingTimer = new llvm::Timer("Initial parse", *AST->TimerGroup);
+    ParsingTimer->startTimer();
+    AST->Timers.push_back(ParsingTimer);
+  }
+  
+  bool Failed = AST->Parse(OverrideMainBuffer);
+  if (ParsingTimer)
+    ParsingTimer->stopTimer();
+  
+  return Failed? 0 : AST.take();
 }
 
 ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
@@ -396,12 +1343,18 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                       bool OnlyLocalDecls,
                                       RemappedFile *RemappedFiles,
                                       unsigned NumRemappedFiles,
-                                      bool CaptureDiagnostics) {
+                                      bool CaptureDiagnostics,
+                                      bool PrecompilePreamble,
+                                      bool CompleteTranslationUnit,
+                                      bool CacheCodeCompletionResults) {
+  bool CreatedDiagnosticsObject = false;
+  
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
     DiagnosticOptions DiagOpts;
     Diags = CompilerInstance::createDiagnostics(DiagOpts, 0, 0);
+    CreatedDiagnosticsObject = true;
   }
   
   llvm::SmallVector<const char *, 16> Args;
@@ -413,7 +1366,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   Args.push_back("-fsyntax-only");
 
   // FIXME: We shouldn't have to pass in the path info.
-  driver::Driver TheDriver("clang", "/", llvm::sys::getHostTriple(),
+  driver::Driver TheDriver("clang", llvm::sys::getHostTriple(),
                            "a.out", false, false, *Diags);
 
   // Don't check that inputs exist, they have been remapped.
@@ -444,7 +1397,7 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   CompilerInvocation::CreateFromArgs(*CI,
                                      const_cast<const char **>(CCArgs.data()),
                                      const_cast<const char **>(CCArgs.data()) +
-                                       CCArgs.size(),
+                                     CCArgs.size(),
                                      *Diags);
 
   // Override any files that need remapping
@@ -455,7 +1408,468 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   // Override the resources path.
   CI->getHeaderSearchOpts().ResourceDir = ResourceFilesPath;
 
-  CI->getFrontendOpts().DisableFree = true;
+  CI->getFrontendOpts().DisableFree = false;
   return LoadFromCompilerInvocation(CI.take(), Diags, OnlyLocalDecls,
-                                    CaptureDiagnostics);
+                                    CaptureDiagnostics, PrecompilePreamble,
+                                    CompleteTranslationUnit,
+                                    CacheCodeCompletionResults);
+}
+
+bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) {
+  if (!Invocation.get())
+    return true;
+  
+  llvm::Timer *ReparsingTimer = 0;
+  if (TimerGroup.get()) {
+    ReparsingTimer = new llvm::Timer("Reparse", *TimerGroup);
+    ReparsingTimer->startTimer();
+    Timers.push_back(ReparsingTimer);
+  }
+  
+  // Remap files.
+  PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
+  for (PreprocessorOptions::remapped_file_buffer_iterator 
+         R = PPOpts.remapped_file_buffer_begin(),
+         REnd = PPOpts.remapped_file_buffer_end();
+       R != REnd; 
+       ++R) {
+    delete R->second;
+  }
+  Invocation->getPreprocessorOpts().clearRemappedFiles();
+  for (unsigned I = 0; I != NumRemappedFiles; ++I)
+    Invocation->getPreprocessorOpts().addRemappedFile(RemappedFiles[I].first,
+                                                      RemappedFiles[I].second);
+  
+  // If we have a preamble file lying around, or if we might try to
+  // build a precompiled preamble, do so now.
+  llvm::MemoryBuffer *OverrideMainBuffer = 0;
+  if (!PreambleFile.empty() || PreambleRebuildCounter > 0)
+    OverrideMainBuffer = getMainBufferWithPrecompiledPreamble(*Invocation);
+    
+  // Clear out the diagnostics state.
+  if (!OverrideMainBuffer)
+    getDiagnostics().Reset();
+  
+  // Parse the sources
+  bool Result = Parse(OverrideMainBuffer);  
+  if (ReparsingTimer)
+    ReparsingTimer->stopTimer();
+  
+  if (ShouldCacheCodeCompletionResults) {
+    if (CacheCodeCompletionCoolDown > 0)
+      --CacheCodeCompletionCoolDown;
+    else if (top_level_size() != NumTopLevelDeclsAtLastCompletionCache)
+      CacheCodeCompletionResults();
+  }
+  
+  return Result;
+}
+
+//----------------------------------------------------------------------------//
+// Code completion
+//----------------------------------------------------------------------------//
+
+namespace {
+  /// \brief Code completion consumer that combines the cached code-completion
+  /// results from an ASTUnit with the code-completion results provided to it,
+  /// then passes the result on to 
+  class AugmentedCodeCompleteConsumer : public CodeCompleteConsumer {
+    unsigned NormalContexts;
+    ASTUnit &AST;
+    CodeCompleteConsumer &Next;
+    
+  public:
+    AugmentedCodeCompleteConsumer(ASTUnit &AST, CodeCompleteConsumer &Next,
+                                  bool IncludeMacros, bool IncludeCodePatterns,
+                                  bool IncludeGlobals)
+      : CodeCompleteConsumer(IncludeMacros, IncludeCodePatterns, IncludeGlobals,
+                             Next.isOutputBinary()), AST(AST), Next(Next) 
+    { 
+      // Compute the set of contexts in which we will look when we don't have
+      // any information about the specific context.
+      NormalContexts 
+        = (1 << (CodeCompletionContext::CCC_TopLevel - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCInterface - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCImplementation - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCIvarList - 1))
+        | (1 << (CodeCompletionContext::CCC_Statement - 1))
+        | (1 << (CodeCompletionContext::CCC_Expression - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCMessageReceiver - 1))
+        | (1 << (CodeCompletionContext::CCC_MemberAccess - 1))
+        | (1 << (CodeCompletionContext::CCC_ObjCProtocolName - 1));
+      
+      if (AST.getASTContext().getLangOptions().CPlusPlus)
+        NormalContexts |= (1 << (CodeCompletionContext::CCC_EnumTag - 1))
+                    | (1 << (CodeCompletionContext::CCC_UnionTag - 1))
+                    | (1 << (CodeCompletionContext::CCC_ClassOrStructTag - 1));
+    }
+    
+    virtual void ProcessCodeCompleteResults(Sema &S, 
+                                            CodeCompletionContext Context,
+                                            CodeCompletionResult *Results,
+                                            unsigned NumResults);
+    
+    virtual void ProcessOverloadCandidates(Sema &S, unsigned CurrentArg,
+                                           OverloadCandidate *Candidates,
+                                           unsigned NumCandidates) { 
+      Next.ProcessOverloadCandidates(S, CurrentArg, Candidates, NumCandidates);
+    }
+  };
+}
+
+/// \brief Helper function that computes which global names are hidden by the
+/// local code-completion results.
+void CalculateHiddenNames(const CodeCompletionContext &Context,
+                          CodeCompletionResult *Results,
+                          unsigned NumResults,
+                          ASTContext &Ctx,
+                          llvm::StringSet<> &HiddenNames) {
+  bool OnlyTagNames = false;
+  switch (Context.getKind()) {
+  case CodeCompletionContext::CCC_Other:
+  case CodeCompletionContext::CCC_TopLevel:
+  case CodeCompletionContext::CCC_ObjCInterface:
+  case CodeCompletionContext::CCC_ObjCImplementation:
+  case CodeCompletionContext::CCC_ObjCIvarList:
+  case CodeCompletionContext::CCC_ClassStructUnion:
+  case CodeCompletionContext::CCC_Statement:
+  case CodeCompletionContext::CCC_Expression:
+  case CodeCompletionContext::CCC_ObjCMessageReceiver:
+  case CodeCompletionContext::CCC_MemberAccess:
+  case CodeCompletionContext::CCC_Namespace:
+  case CodeCompletionContext::CCC_Type:
+  case CodeCompletionContext::CCC_Name:
+  case CodeCompletionContext::CCC_PotentiallyQualifiedName:
+    break;
+    
+  case CodeCompletionContext::CCC_EnumTag:
+  case CodeCompletionContext::CCC_UnionTag:
+  case CodeCompletionContext::CCC_ClassOrStructTag:
+    OnlyTagNames = true;
+    break;
+    
+  case CodeCompletionContext::CCC_ObjCProtocolName:
+  case CodeCompletionContext::CCC_MacroName:
+  case CodeCompletionContext::CCC_MacroNameUse:
+  case CodeCompletionContext::CCC_PreprocessorExpression:
+  case CodeCompletionContext::CCC_PreprocessorDirective:
+  case CodeCompletionContext::CCC_NaturalLanguage:
+  case CodeCompletionContext::CCC_SelectorName:
+  case CodeCompletionContext::CCC_TypeQualifiers:
+    // We're looking for nothing, or we're looking for names that cannot
+    // be hidden.
+    return;
+  }
+  
+  typedef CodeCompletionResult Result;
+  for (unsigned I = 0; I != NumResults; ++I) {
+    if (Results[I].Kind != Result::RK_Declaration)
+      continue;
+    
+    unsigned IDNS
+      = Results[I].Declaration->getUnderlyingDecl()->getIdentifierNamespace();
+
+    bool Hiding = false;
+    if (OnlyTagNames)
+      Hiding = (IDNS & Decl::IDNS_Tag);
+    else {
+      unsigned HiddenIDNS = (Decl::IDNS_Type | Decl::IDNS_Member | 
+                             Decl::IDNS_Namespace | Decl::IDNS_Ordinary |
+                             Decl::IDNS_NonMemberOperator);
+      if (Ctx.getLangOptions().CPlusPlus)
+        HiddenIDNS |= Decl::IDNS_Tag;
+      Hiding = (IDNS & HiddenIDNS);
+    }
+  
+    if (!Hiding)
+      continue;
+    
+    DeclarationName Name = Results[I].Declaration->getDeclName();
+    if (IdentifierInfo *Identifier = Name.getAsIdentifierInfo())
+      HiddenNames.insert(Identifier->getName());
+    else
+      HiddenNames.insert(Name.getAsString());
+  }
+}
+
+
+void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
+                                            CodeCompletionContext Context,
+                                            CodeCompletionResult *Results,
+                                            unsigned NumResults) { 
+  // Merge the results we were given with the results we cached.
+  bool AddedResult = false;
+  unsigned InContexts  
+    = (Context.getKind() == CodeCompletionContext::CCC_Other? NormalContexts
+                                            : (1 << (Context.getKind() - 1)));
+
+  // Contains the set of names that are hidden by "local" completion results.
+  llvm::StringSet<> HiddenNames;
+  llvm::SmallVector<CodeCompletionString *, 4> StringsToDestroy;
+  typedef CodeCompletionResult Result;
+  llvm::SmallVector<Result, 8> AllResults;
+  for (ASTUnit::cached_completion_iterator 
+            C = AST.cached_completion_begin(),
+         CEnd = AST.cached_completion_end();
+       C != CEnd; ++C) {
+    // If the context we are in matches any of the contexts we are 
+    // interested in, we'll add this result.
+    if ((C->ShowInContexts & InContexts) == 0)
+      continue;
+    
+    // If we haven't added any results previously, do so now.
+    if (!AddedResult) {
+      CalculateHiddenNames(Context, Results, NumResults, S.Context, 
+                           HiddenNames);
+      AllResults.insert(AllResults.end(), Results, Results + NumResults);
+      AddedResult = true;
+    }
+    
+    // Determine whether this global completion result is hidden by a local
+    // completion result. If so, skip it.
+    if (C->Kind != CXCursor_MacroDefinition &&
+        HiddenNames.count(C->Completion->getTypedText()))
+      continue;
+    
+    // Adjust priority based on similar type classes.
+    unsigned Priority = C->Priority;
+    CXCursorKind CursorKind = C->Kind;
+    CodeCompletionString *Completion = C->Completion;
+    if (!Context.getPreferredType().isNull()) {
+      if (C->Kind == CXCursor_MacroDefinition) {
+        Priority = getMacroUsagePriority(C->Completion->getTypedText(),
+                               Context.getPreferredType()->isAnyPointerType());        
+      } else if (C->Type) {
+        CanQualType Expected
+          = S.Context.getCanonicalType(
+                               Context.getPreferredType().getUnqualifiedType());
+        SimplifiedTypeClass ExpectedSTC = getSimplifiedTypeClass(Expected);
+        if (ExpectedSTC == C->TypeClass) {
+          // We know this type is similar; check for an exact match.
+          llvm::StringMap<unsigned> &CachedCompletionTypes
+            = AST.getCachedCompletionTypes();
+          llvm::StringMap<unsigned>::iterator Pos
+            = CachedCompletionTypes.find(QualType(Expected).getAsString());
+          if (Pos != CachedCompletionTypes.end() && Pos->second == C->Type)
+            Priority /= CCF_ExactTypeMatch;
+          else
+            Priority /= CCF_SimilarTypeMatch;
+        }
+      }
+    }
+    
+    // Adjust the completion string, if required.
+    if (C->Kind == CXCursor_MacroDefinition &&
+        Context.getKind() == CodeCompletionContext::CCC_MacroNameUse) {
+      // Create a new code-completion string that just contains the
+      // macro name, without its arguments.
+      Completion = new CodeCompletionString;
+      Completion->AddTypedTextChunk(C->Completion->getTypedText());
+      StringsToDestroy.push_back(Completion);
+      CursorKind = CXCursor_NotImplemented;
+      Priority = CCP_CodePattern;
+    }
+    
+    AllResults.push_back(Result(Completion, Priority, CursorKind, 
+                                C->Availability));
+  }
+  
+  // If we did not add any cached completion results, just forward the
+  // results we were given to the next consumer.
+  if (!AddedResult) {
+    Next.ProcessCodeCompleteResults(S, Context, Results, NumResults);
+    return;
+  }
+  
+  Next.ProcessCodeCompleteResults(S, Context, AllResults.data(),
+                                  AllResults.size());
+  
+  for (unsigned I = 0, N = StringsToDestroy.size(); I != N; ++I)
+    delete StringsToDestroy[I];
+}
+
+
+
+void ASTUnit::CodeComplete(llvm::StringRef File, unsigned Line, unsigned Column,
+                           RemappedFile *RemappedFiles, 
+                           unsigned NumRemappedFiles,
+                           bool IncludeMacros, 
+                           bool IncludeCodePatterns,
+                           CodeCompleteConsumer &Consumer,
+                           Diagnostic &Diag, LangOptions &LangOpts,
+                           SourceManager &SourceMgr, FileManager &FileMgr,
+                   llvm::SmallVectorImpl<StoredDiagnostic> &StoredDiagnostics,
+             llvm::SmallVectorImpl<const llvm::MemoryBuffer *> &OwnedBuffers) {
+  if (!Invocation.get())
+    return;
+
+  llvm::Timer *CompletionTimer = 0;
+  if (TimerGroup.get()) {
+    llvm::SmallString<128> TimerName;
+    llvm::raw_svector_ostream TimerNameOut(TimerName);
+    TimerNameOut << "Code completion @ " << File << ":" << Line << ":" 
+                 << Column;
+    CompletionTimer = new llvm::Timer(TimerNameOut.str(), *TimerGroup);
+    CompletionTimer->startTimer();
+    Timers.push_back(CompletionTimer);
+  }
+
+  CompilerInvocation CCInvocation(*Invocation);
+  FrontendOptions &FrontendOpts = CCInvocation.getFrontendOpts();
+  PreprocessorOptions &PreprocessorOpts = CCInvocation.getPreprocessorOpts();
+
+  FrontendOpts.ShowMacrosInCodeCompletion
+    = IncludeMacros && CachedCompletionResults.empty();
+  FrontendOpts.ShowCodePatternsInCodeCompletion = IncludeCodePatterns;
+  FrontendOpts.ShowGlobalSymbolsInCodeCompletion
+    = CachedCompletionResults.empty();
+  FrontendOpts.CodeCompletionAt.FileName = File;
+  FrontendOpts.CodeCompletionAt.Line = Line;
+  FrontendOpts.CodeCompletionAt.Column = Column;
+
+  // Turn on spell-checking when performing code completion. It leads
+  // to better results.
+  unsigned SpellChecking = CCInvocation.getLangOpts().SpellChecking;
+  CCInvocation.getLangOpts().SpellChecking = 1;
+
+  // Set the language options appropriately.
+  LangOpts = CCInvocation.getLangOpts();
+
+  CompilerInstance Clang;
+  Clang.setInvocation(&CCInvocation);
+  OriginalSourceFile = Clang.getFrontendOpts().Inputs[0].second;
+    
+  // Set up diagnostics, capturing any diagnostics produced.
+  Clang.setDiagnostics(&Diag);
+  CaptureDroppedDiagnostics Capture(true, 
+                                    Clang.getDiagnostics(),
+                                    StoredDiagnostics);
+  
+  // Create the target instance.
+  Clang.setTarget(TargetInfo::CreateTargetInfo(Clang.getDiagnostics(),
+                                               Clang.getTargetOpts()));
+  if (!Clang.hasTarget()) {
+    Clang.takeInvocation();
+    CCInvocation.getLangOpts().SpellChecking = SpellChecking;
+    return;
+  }
+  
+  // Inform the target of the language options.
+  //
+  // FIXME: We shouldn't need to do this, the target should be immutable once
+  // created. This complexity should be lifted elsewhere.
+  Clang.getTarget().setForcedLangOptions(Clang.getLangOpts());
+  
+  assert(Clang.getFrontendOpts().Inputs.size() == 1 &&
+         "Invocation must have exactly one source file!");
+  assert(Clang.getFrontendOpts().Inputs[0].first != IK_AST &&
+         "FIXME: AST inputs not yet supported here!");
+  assert(Clang.getFrontendOpts().Inputs[0].first != IK_LLVM_IR &&
+         "IR inputs not support here!");
+
+  
+  // Use the source and file managers that we were given.
+  Clang.setFileManager(&FileMgr);
+  Clang.setSourceManager(&SourceMgr);
+
+  // Remap files.
+  PreprocessorOpts.clearRemappedFiles();
+  PreprocessorOpts.RetainRemappedFileBuffers = true;
+  for (unsigned I = 0; I != NumRemappedFiles; ++I) {
+    PreprocessorOpts.addRemappedFile(RemappedFiles[I].first,
+                                     RemappedFiles[I].second);
+    OwnedBuffers.push_back(RemappedFiles[I].second);
+  }
+  
+  // Use the code completion consumer we were given, but adding any cached
+  // code-completion results.
+  AugmentedCodeCompleteConsumer 
+  AugmentedConsumer(*this, Consumer, FrontendOpts.ShowMacrosInCodeCompletion,
+                    FrontendOpts.ShowCodePatternsInCodeCompletion,
+                    FrontendOpts.ShowGlobalSymbolsInCodeCompletion);
+  Clang.setCodeCompletionConsumer(&AugmentedConsumer);
+
+  // If we have a precompiled preamble, try to use it. We only allow
+  // the use of the precompiled preamble if we're if the completion
+  // point is within the main file, after the end of the precompiled
+  // preamble.
+  llvm::MemoryBuffer *OverrideMainBuffer = 0;
+  if (!PreambleFile.empty()) {
+    using llvm::sys::FileStatus;
+    llvm::sys::PathWithStatus CompleteFilePath(File);
+    llvm::sys::PathWithStatus MainPath(OriginalSourceFile);
+    if (const FileStatus *CompleteFileStatus = CompleteFilePath.getFileStatus())
+      if (const FileStatus *MainStatus = MainPath.getFileStatus())
+        if (CompleteFileStatus->getUniqueID() == MainStatus->getUniqueID())
+          OverrideMainBuffer
+            = getMainBufferWithPrecompiledPreamble(CCInvocation, false, 
+                                                   Line - 1);
+  }
+
+  // If the main file has been overridden due to the use of a preamble,
+  // make that override happen and introduce the preamble.
+  if (OverrideMainBuffer) {
+    PreprocessorOpts.addRemappedFile(OriginalSourceFile, OverrideMainBuffer);
+    PreprocessorOpts.PrecompiledPreambleBytes.first = Preamble.size();
+    PreprocessorOpts.PrecompiledPreambleBytes.second
+                                                    = PreambleEndsAtStartOfLine;
+    PreprocessorOpts.ImplicitPCHInclude = PreambleFile;
+    PreprocessorOpts.DisablePCHValidation = true;
+    
+    // The stored diagnostics have the old source manager. Copy them
+    // to our output set of stored diagnostics, updating the source
+    // manager to the one we were given.
+    for (unsigned I = 0, N = this->StoredDiagnostics.size(); I != N; ++I) {
+      StoredDiagnostics.push_back(this->StoredDiagnostics[I]);
+      FullSourceLoc Loc(StoredDiagnostics[I].getLocation(), SourceMgr);
+      StoredDiagnostics[I].setLocation(Loc);
+    }
+    
+    OwnedBuffers.push_back(OverrideMainBuffer);
+  } else {
+    PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
+    PreprocessorOpts.PrecompiledPreambleBytes.second = false;
+  }
+
+  llvm::OwningPtr<SyntaxOnlyAction> Act;
+  Act.reset(new SyntaxOnlyAction);
+  if (Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second,
+                           Clang.getFrontendOpts().Inputs[0].first)) {
+    Act->Execute();
+    Act->EndSourceFile();
+  }
+
+  if (CompletionTimer)
+    CompletionTimer->stopTimer();
+  
+  // Steal back our resources. 
+  Clang.takeFileManager();
+  Clang.takeSourceManager();
+  Clang.takeInvocation();
+  Clang.takeCodeCompletionConsumer();
+  CCInvocation.getLangOpts().SpellChecking = SpellChecking;
+}
+
+bool ASTUnit::Save(llvm::StringRef File) {
+  if (getDiagnostics().hasErrorOccurred())
+    return true;
+  
+  // FIXME: Can we somehow regenerate the stat cache here, or do we need to 
+  // unconditionally create a stat cache when we parse the file?
+  std::string ErrorInfo;
+  llvm::raw_fd_ostream Out(File.str().c_str(), ErrorInfo,
+                           llvm::raw_fd_ostream::F_Binary);
+  if (!ErrorInfo.empty() || Out.has_error())
+    return true;
+  
+  std::vector<unsigned char> Buffer;
+  llvm::BitstreamWriter Stream(Buffer);
+  ASTWriter Writer(Stream);
+  Writer.WriteAST(getSema(), 0, 0);
+  
+  // Write the generated bitstream to "Out".
+  if (!Buffer.empty())
+    Out.write((char *)&Buffer.front(), Buffer.size());  
+  Out.close();
+  return Out.has_error();
 }
diff --git a/lib/Frontend/CMakeLists.txt b/lib/Frontend/CMakeLists.txt
index 8757e2c9e37a..5a31495397ae 100644
--- a/lib/Frontend/CMakeLists.txt
+++ b/lib/Frontend/CMakeLists.txt
@@ -15,17 +15,9 @@ add_clang_library(clangFrontend
   FrontendAction.cpp
   FrontendActions.cpp
   FrontendOptions.cpp
-  GeneratePCH.cpp
   InitHeaderSearch.cpp
   InitPreprocessor.cpp
   LangStandards.cpp
-  PCHReader.cpp
-  PCHReaderDecl.cpp
-  PCHReaderStmt.cpp
-  PCHWriter.cpp
-  PCHWriterDecl.cpp
-  PCHWriterStmt.cpp
-  PrintParserCallbacks.cpp
   PrintPreprocessedOutput.cpp
   StmtXML.cpp
   TextDiagnosticBuffer.cpp
diff --git a/lib/Frontend/CacheTokens.cpp b/lib/Frontend/CacheTokens.cpp
index a5fcebe99411..53f7362ac276 100644
--- a/lib/Frontend/CacheTokens.cpp
+++ b/lib/Frontend/CacheTokens.cpp
@@ -311,14 +311,19 @@ PTHEntry PTHWriter::LexTokens(Lexer& L) {
       // the next token.
       assert(!ParsingPreprocessorDirective);
       Offset HashOff = (Offset) Out.tell();
-      EmitToken(Tok);
 
       // Get the next token.
-      L.LexFromRawLexer(Tok);
+      Token NextTok;
+      L.LexFromRawLexer(NextTok);
 
-      // If we see the start of line, then we had a null directive "#".
-      if (Tok.isAtStartOfLine())
+      // If we see the start of line, then we had a null directive "#".  In
+      // this case, discard both tokens.
+      if (NextTok.isAtStartOfLine())
         goto NextToken;
+      
+      // The token is the start of a directive.  Emit it.
+      EmitToken(Tok);
+      Tok = NextTok;
 
       // Did we see 'include'/'import'/'include_next'?
       if (Tok.isNot(tok::identifier)) {
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index 5037c83556aa..ce0b07243b05 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Sema/Sema.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Basic/Diagnostic.h"
@@ -20,11 +21,11 @@
 #include "clang/Lex/PTHManager.h"
 #include "clang/Frontend/ChainedDiagnosticClient.h"
 #include "clang/Frontend/FrontendAction.h"
-#include "clang/Frontend/PCHReader.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Frontend/VerifyDiagnosticsClient.h"
 #include "clang/Frontend/Utils.h"
+#include "clang/Serialization/ASTReader.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -37,7 +38,7 @@
 using namespace clang;
 
 CompilerInstance::CompilerInstance()
-  : Invocation(new CompilerInvocation()), Reader(0) {
+  : Invocation(new CompilerInvocation()) {
 }
 
 CompilerInstance::~CompilerInstance() {
@@ -55,10 +56,6 @@ void CompilerInstance::setDiagnostics(Diagnostic *Value) {
   Diagnostics = Value;
 }
 
-void CompilerInstance::setDiagnosticClient(DiagnosticClient *Value) {
-  DiagClient.reset(Value);
-}
-
 void CompilerInstance::setTarget(TargetInfo *Value) {
   Target.reset(Value);
 }
@@ -79,6 +76,10 @@ void CompilerInstance::setASTContext(ASTContext *Value) {
   Context.reset(Value);
 }
 
+void CompilerInstance::setSema(Sema *S) {
+  TheSema.reset(S);
+}
+
 void CompilerInstance::setASTConsumer(ASTConsumer *Value) {
   Consumer.reset(Value);
 }
@@ -126,14 +127,11 @@ static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
   // Chain in a diagnostic client which will log the diagnostics.
   DiagnosticClient *Logger =
     new TextDiagnosticPrinter(*OS.take(), DiagOpts, /*OwnsOutputStream=*/true);
-  Diags.setClient(new ChainedDiagnosticClient(Diags.getClient(), Logger));
+  Diags.setClient(new ChainedDiagnosticClient(Diags.takeClient(), Logger));
 }
 
 void CompilerInstance::createDiagnostics(int Argc, char **Argv) {
   Diagnostics = createDiagnostics(getDiagnosticOpts(), Argc, Argv);
-
-  if (Diagnostics)
-    DiagClient.reset(Diagnostics->getClient());
 }
 
 llvm::IntrusiveRefCntPtr<Diagnostic> 
@@ -150,22 +148,20 @@ CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
       // bit of a problem. So, just create a text diagnostic printer
       // to complain about this problem, and pretend that the user
       // didn't try to use binary output.
-      DiagClient.reset(new TextDiagnosticPrinter(llvm::errs(), Opts));
-      Diags->setClient(DiagClient.take());
+      Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
       Diags->Report(diag::err_fe_stderr_binary);
       return Diags;
     } else {
-      DiagClient.reset(new BinaryDiagnosticSerializer(llvm::errs()));
+      Diags->setClient(new BinaryDiagnosticSerializer(llvm::errs()));
     }
   } else {
-    DiagClient.reset(new TextDiagnosticPrinter(llvm::errs(), Opts));
+    Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
   }
 
   // Chain in -verify checker, if requested.
   if (Opts.VerifyDiagnostics)
-    DiagClient.reset(new VerifyDiagnosticsClient(*Diags, DiagClient.take()));
+    Diags->setClient(new VerifyDiagnosticsClient(*Diags, Diags->takeClient()));
 
-  Diags->setClient(DiagClient.take());
   if (!Opts.DumpBuildInformation.empty())
     SetUpBuildDumpLog(Opts, Argc, Argv, *Diags);
 
@@ -245,42 +241,48 @@ void CompilerInstance::createASTContext() {
   Context.reset(new ASTContext(getLangOpts(), PP.getSourceManager(),
                                getTarget(), PP.getIdentifierTable(),
                                PP.getSelectorTable(), PP.getBuiltinInfo(),
-                               /*FreeMemory=*/ !getFrontendOpts().DisableFree,
                                /*size_reserve=*/ 0));
 }
 
 // ExternalASTSource
 
-void CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path) {
+void CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
+                                                  bool DisablePCHValidation,
+                                                 void *DeserializationListener){
   llvm::OwningPtr<ExternalASTSource> Source;
   Source.reset(createPCHExternalASTSource(Path, getHeaderSearchOpts().Sysroot,
-                                          getPreprocessor(), getASTContext()));
-  // Remember the PCHReader, but in a non-owning way.
-  Reader = static_cast<PCHReader*>(Source.get());
+                                          DisablePCHValidation,
+                                          getPreprocessor(), getASTContext(),
+                                          DeserializationListener));
   getASTContext().setExternalSource(Source);
 }
 
 ExternalASTSource *
 CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
                                              const std::string &Sysroot,
+                                             bool DisablePCHValidation,
                                              Preprocessor &PP,
-                                             ASTContext &Context) {
-  llvm::OwningPtr<PCHReader> Reader;
-  Reader.reset(new PCHReader(PP, &Context,
-                             Sysroot.empty() ? 0 : Sysroot.c_str()));
-
-  switch (Reader->ReadPCH(Path)) {
-  case PCHReader::Success:
+                                             ASTContext &Context,
+                                             void *DeserializationListener) {
+  llvm::OwningPtr<ASTReader> Reader;
+  Reader.reset(new ASTReader(PP, &Context,
+                             Sysroot.empty() ? 0 : Sysroot.c_str(),
+                             DisablePCHValidation));
+
+  Reader->setDeserializationListener(
+            static_cast<ASTDeserializationListener *>(DeserializationListener));
+  switch (Reader->ReadAST(Path)) {
+  case ASTReader::Success:
     // Set the predefines buffer as suggested by the PCH reader. Typically, the
     // predefines buffer will be empty.
     PP.setPredefines(Reader->getSuggestedPredefines());
     return Reader.take();
 
-  case PCHReader::Failure:
+  case ASTReader::Failure:
     // Unrecoverable failure: don't even try to process the input file.
     break;
 
-  case PCHReader::IgnorePCH:
+  case ASTReader::IgnorePCH:
     // No suitable PCH file could be found. Return an error.
     break;
   }
@@ -290,17 +292,42 @@ CompilerInstance::createPCHExternalASTSource(llvm::StringRef Path,
 
 // Code Completion
 
+static bool EnableCodeCompletion(Preprocessor &PP, 
+                                 const std::string &Filename,
+                                 unsigned Line,
+                                 unsigned Column) {
+  // Tell the source manager to chop off the given file at a specific
+  // line and column.
+  const FileEntry *Entry = PP.getFileManager().getFile(Filename);
+  if (!Entry) {
+    PP.getDiagnostics().Report(diag::err_fe_invalid_code_complete_file)
+      << Filename;
+    return true;
+  }
+
+  // Truncate the named file at the given line/column.
+  PP.SetCodeCompletionPoint(Entry, Line, Column);
+  return false;
+}
+
 void CompilerInstance::createCodeCompletionConsumer() {
   const ParsedSourceLocation &Loc = getFrontendOpts().CodeCompletionAt;
-  CompletionConsumer.reset(
-    createCodeCompletionConsumer(getPreprocessor(),
-                                 Loc.FileName, Loc.Line, Loc.Column,
-                                 getFrontendOpts().DebugCodeCompletionPrinter,
-                                 getFrontendOpts().ShowMacrosInCodeCompletion,
+  if (!CompletionConsumer) {
+    CompletionConsumer.reset(
+      createCodeCompletionConsumer(getPreprocessor(),
+                                   Loc.FileName, Loc.Line, Loc.Column,
+                                   getFrontendOpts().DebugCodeCompletionPrinter,
+                                   getFrontendOpts().ShowMacrosInCodeCompletion,
                              getFrontendOpts().ShowCodePatternsInCodeCompletion,
-                                 llvm::outs()));
-  if (!CompletionConsumer)
+                           getFrontendOpts().ShowGlobalSymbolsInCodeCompletion,
+                                   llvm::outs()));
+    if (!CompletionConsumer)
+      return;
+  } else if (EnableCodeCompletion(getPreprocessor(), Loc.FileName,
+                                  Loc.Line, Loc.Column)) {
+    CompletionConsumer.reset();
     return;
+  }
 
   if (CompletionConsumer->isOutputBinary() &&
       llvm::sys::Program::ChangeStdoutToBinary()) {
@@ -321,24 +348,24 @@ CompilerInstance::createCodeCompletionConsumer(Preprocessor &PP,
                                                bool UseDebugPrinter,
                                                bool ShowMacros,
                                                bool ShowCodePatterns,
+                                               bool ShowGlobals,
                                                llvm::raw_ostream &OS) {
-  // Tell the source manager to chop off the given file at a specific
-  // line and column.
-  const FileEntry *Entry = PP.getFileManager().getFile(Filename);
-  if (!Entry) {
-    PP.getDiagnostics().Report(diag::err_fe_invalid_code_complete_file)
-      << Filename;
+  if (EnableCodeCompletion(PP, Filename, Line, Column))
     return 0;
-  }
-
-  // Truncate the named file at the given line/column.
-  PP.SetCodeCompletionPoint(Entry, Line, Column);
 
   // Set up the creation routine for code-completion.
   if (UseDebugPrinter)
-    return new PrintingCodeCompleteConsumer(ShowMacros, ShowCodePatterns, OS);
+    return new PrintingCodeCompleteConsumer(ShowMacros, ShowCodePatterns, 
+                                            ShowGlobals, OS);
   else
-    return new CIndexCodeCompleteConsumer(ShowMacros, ShowCodePatterns, OS);
+    return new CIndexCodeCompleteConsumer(ShowMacros, ShowCodePatterns, 
+                                          ShowGlobals, OS);
+}
+
+void CompilerInstance::createSema(bool CompleteTranslationUnit,
+                                  CodeCompleteConsumer *CompletionConsumer) {
+  TheSema.reset(new Sema(getPreprocessor(), getASTContext(), getASTConsumer(),
+                         CompleteTranslationUnit, CompletionConsumer));
 }
 
 // Output Files
@@ -437,7 +464,7 @@ bool CompilerInstance::InitializeSourceManager(llvm::StringRef InputFile,
   // Figure out where to get and map in the main file.
   if (InputFile != "-") {
     const FileEntry *File = FileMgr.getFile(InputFile);
-    if (File) SourceMgr.createMainFileID(File, SourceLocation());
+    if (File) SourceMgr.createMainFileID(File);
     if (SourceMgr.getMainFileID().isInvalid()) {
       Diags.Report(diag::err_fe_error_reading) << InputFile;
       return false;
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index 418d25b0d47d..8c644833b20e 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -18,11 +18,12 @@
 #include "clang/Driver/Option.h"
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/LangStandard.h"
-#include "clang/Frontend/PCHReader.h"
+#include "clang/Serialization/ASTReader.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/System/Host.h"
 #include "llvm/System/Path.h"
@@ -112,8 +113,8 @@ static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts,
     Res.push_back("-analyzer-experimental-checks");
   if (Opts.EnableExperimentalInternalChecks)
     Res.push_back("-analyzer-experimental-internal-checks");
-  if (Opts.EnableIdempotentOperationChecker)
-    Res.push_back("-analyzer-idempotent-operation");
+  if (Opts.IdempotentOps)
+      Res.push_back("-analyzer-check-idempotent-operations");
 }
 
 static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
@@ -148,9 +149,12 @@ static void CodeGenOptsToArgs(const CodeGenOptions &Opts,
   // SimplifyLibCalls is only derived.
   // TimePasses is only derived.
   // UnitAtATime is unused.
-  // UnrollLoops is only derived.
   // Inlining is only derived.
-
+  
+  // UnrollLoops is derived, but also accepts an option, no
+  // harm in pushing it back here.
+  if (Opts.UnrollLoops)
+    Res.push_back("-funroll-loops");
   if (Opts.DataSections)
     Res.push_back("-fdata-sections");
   if (Opts.FunctionSections)
@@ -241,6 +245,8 @@ static void DiagnosticOptsToArgs(const DiagnosticOptions &Opts,
     Res.push_back("-fno-diagnostics-fixit-info");
   if (Opts.ShowSourceRanges)
     Res.push_back("-fdiagnostics-print-source-range-info");
+  if (Opts.ShowParseableFixits)
+    Res.push_back("-fdiagnostics-parseable-fixits");
   if (Opts.ShowColors)
     Res.push_back("-fcolor-diagnostics");
   if (Opts.VerifyDiagnostics)
@@ -316,6 +322,7 @@ static const char *getActionName(frontend::ActionKind Kind) {
   case frontend::ASTPrintXML:            return "-ast-print-xml";
   case frontend::ASTView:                return "-ast-view";
   case frontend::BoostCon:               return "-boostcon";
+  case frontend::CreateModule:           return "-create-module";
   case frontend::DumpRawTokens:          return "-dump-raw-tokens";
   case frontend::DumpTokens:             return "-dump-tokens";
   case frontend::EmitAssembly:           return "-S";
@@ -329,10 +336,9 @@ static const char *getActionName(frontend::ActionKind Kind) {
   case frontend::GeneratePCH:            return "-emit-pch";
   case frontend::GeneratePTH:            return "-emit-pth";
   case frontend::InitOnly:               return "-init-only";
-  case frontend::ParseNoop:              return "-parse-noop";
-  case frontend::ParsePrintCallbacks:    return "-parse-print-callbacks";
   case frontend::ParseSyntaxOnly:        return "-fsyntax-only";
   case frontend::PrintDeclContext:       return "-print-decl-contexts";
+  case frontend::PrintPreamble:          return "-print-preamble";
   case frontend::PrintPreprocessedInput: return "-E";
   case frontend::RewriteMacros:          return "-rewrite-macros";
   case frontend::RewriteObjC:            return "-rewrite-objc";
@@ -361,12 +367,16 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-code-completion-macros");
   if (Opts.ShowCodePatternsInCodeCompletion)
     Res.push_back("-code-completion-patterns");
+  if (!Opts.ShowGlobalSymbolsInCodeCompletion)
+    Res.push_back("-no-code-completion-globals");
   if (Opts.ShowStats)
     Res.push_back("-print-stats");
   if (Opts.ShowTimers)
     Res.push_back("-ftime-report");
   if (Opts.ShowVersion)
     Res.push_back("-version");
+  if (Opts.FixWhatYouCan)
+    Res.push_back("-fix-what-you-can");
 
   bool NeedLang = false;
   for (unsigned i = 0, e = Opts.Inputs.size(); i != e; ++i)
@@ -416,6 +426,10 @@ static void FrontendOptsToArgs(const FrontendOptions &Opts,
     Res.push_back("-ast-merge");
     Res.push_back(Opts.ASTMergeFiles[i]);
   }
+  for (unsigned i = 0, e = Opts.Modules.size(); i != e; ++i) {
+    Res.push_back("-import-module");
+    Res.push_back(Opts.Modules[i]);
+  }
   for (unsigned i = 0, e = Opts.LLVMArgs.size(); i != e; ++i) {
     Res.push_back("-mllvm");
     Res.push_back(Opts.LLVMArgs[i]);
@@ -512,6 +526,8 @@ static void LangOptsToArgs(const LangOptions &Opts,
     Res.push_back("-fgnu-keywords");
   if (Opts.Microsoft)
     Res.push_back("-fms-extensions");
+  if (Opts.Borland)
+    Res.push_back("-fborland-extensions");
   if (Opts.ObjCNonFragileABI)
     Res.push_back("-fobjc-nonfragile-abi");
   if (Opts.ObjCNonFragileABI2)
@@ -676,6 +692,8 @@ static void PreprocessorOutputOptsToArgs(const PreprocessorOutputOptions &Opts,
   else if (!Opts.ShowCPP && Opts.ShowMacros)
     Res.push_back("-dM");
 
+  if (Opts.ShowHeaderIncludes)
+    Res.push_back("-H");
   if (!Opts.ShowLineMarkers)
     Res.push_back("-P");
   if (Opts.ShowComments)
@@ -696,8 +714,14 @@ static void TargetOptsToArgs(const TargetOptions &Opts,
     Res.push_back("-target-abi");
     Res.push_back(Opts.ABI);
   }
-  Res.push_back("-cxx-abi");
-  Res.push_back(Opts.CXXABI);
+  if (!Opts.LinkerVersion.empty()) {
+    Res.push_back("-target-linker-version");
+    Res.push_back(Opts.LinkerVersion);
+  }
+  if (!Opts.CXXABI.empty()) {
+    Res.push_back("-cxx-abi");
+    Res.push_back(Opts.CXXABI);
+  }
   for (unsigned i = 0, e = Opts.Features.size(); i != e; ++i) {
     Res.push_back("-target-feature");
     Res.push_back(Opts.Features[i]);
@@ -789,15 +813,15 @@ static void ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   Opts.PurgeDead = !Args.hasArg(OPT_analyzer_no_purge_dead);
   Opts.EagerlyAssume = Args.hasArg(OPT_analyzer_eagerly_assume);
   Opts.AnalyzeSpecificFunction = Args.getLastArgValue(OPT_analyze_function);
+  Opts.UnoptimizedCFG = Args.hasArg(OPT_analysis_UnoptimizedCFG);
   Opts.EnableExperimentalChecks = Args.hasArg(OPT_analyzer_experimental_checks);
   Opts.EnableExperimentalInternalChecks =
     Args.hasArg(OPT_analyzer_experimental_internal_checks);
-  Opts.EnableIdempotentOperationChecker =
-    Args.hasArg(OPT_analyzer_idempotent_operation);
   Opts.TrimGraph = Args.hasArg(OPT_trim_egraph);
   Opts.MaxNodes = Args.getLastArgIntValue(OPT_analyzer_max_nodes, 150000,Diags);
   Opts.MaxLoop = Args.getLastArgIntValue(OPT_analyzer_max_loop, 3, Diags);
   Opts.InlineCall = Args.hasArg(OPT_analyzer_inline_call);
+  Opts.IdempotentOps = Args.hasArg(OPT_analysis_WarnIdempotentOps);
 }
 
 static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
@@ -829,7 +853,8 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   Opts.OptimizeSize = Args.hasArg(OPT_Os);
   Opts.SimplifyLibCalls = !(Args.hasArg(OPT_fno_builtin) ||
                             Args.hasArg(OPT_ffreestanding));
-  Opts.UnrollLoops = (Opts.OptimizationLevel > 1 && !Opts.OptimizeSize);
+  Opts.UnrollLoops = Args.hasArg(OPT_funroll_loops) || 
+                     (Opts.OptimizationLevel > 1 && !Opts.OptimizeSize);
 
   Opts.AsmVerbose = Args.hasArg(OPT_masm_verbose);
   Opts.CXAAtExit = !Args.hasArg(OPT_fno_use_cxa_atexit);
@@ -838,6 +863,7 @@ static void ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   Opts.DebugPass = Args.getLastArgValue(OPT_mdebug_pass);
   Opts.DisableFPElim = Args.hasArg(OPT_mdisable_fp_elim);
   Opts.FloatABI = Args.getLastArgValue(OPT_mfloat_abi);
+  Opts.HiddenWeakVTables = Args.hasArg(OPT_fhidden_weak_vtables);
   Opts.LimitFloatPrecision = Args.getLastArgValue(OPT_mlimit_float_precision);
   Opts.NoZeroInitializedInBSS = Args.hasArg(OPT_mno_zero_initialized_in_bss);
   Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
@@ -916,6 +942,7 @@ static void ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
       << ShowCategory;
   
   Opts.ShowSourceRanges = Args.hasArg(OPT_fdiagnostics_print_source_range_info);
+  Opts.ShowParseableFixits = Args.hasArg(OPT_fdiagnostics_parseable_fixits);
   Opts.VerifyDiagnostics = Args.hasArg(OPT_verify);
   Opts.BinaryOutput = Args.hasArg(OPT_fdiagnostics_binary);
   Opts.ErrorLimit = Args.getLastArgIntValue(OPT_ferror_limit, 0, Diags);
@@ -985,14 +1012,12 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Opts.ProgramAction = frontend::GeneratePTH; break;
     case OPT_init_only:
       Opts.ProgramAction = frontend::InitOnly; break;
-    case OPT_parse_noop:
-      Opts.ProgramAction = frontend::ParseNoop; break;
-    case OPT_parse_print_callbacks:
-      Opts.ProgramAction = frontend::ParsePrintCallbacks; break;
     case OPT_fsyntax_only:
       Opts.ProgramAction = frontend::ParseSyntaxOnly; break;
     case OPT_print_decl_contexts:
       Opts.ProgramAction = frontend::PrintDeclContext; break;
+    case OPT_print_preamble:
+      Opts.ProgramAction = frontend::PrintPreamble; break;
     case OPT_E:
       Opts.ProgramAction = frontend::PrintPreprocessedInput; break;
     case OPT_rewrite_macros:
@@ -1005,6 +1030,8 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Opts.ProgramAction = frontend::RunAnalysis; break;
     case OPT_Eonly:
       Opts.ProgramAction = frontend::RunPreprocessorOnly; break;
+    case OPT_create_module:
+      Opts.ProgramAction = frontend::CreateModule; break;
     }
   }
 
@@ -1039,12 +1066,16 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.ShowMacrosInCodeCompletion = Args.hasArg(OPT_code_completion_macros);
   Opts.ShowCodePatternsInCodeCompletion
     = Args.hasArg(OPT_code_completion_patterns);
+  Opts.ShowGlobalSymbolsInCodeCompletion
+    = !Args.hasArg(OPT_no_code_completion_globals);
   Opts.ShowStats = Args.hasArg(OPT_print_stats);
   Opts.ShowTimers = Args.hasArg(OPT_ftime_report);
   Opts.ShowVersion = Args.hasArg(OPT_version);
   Opts.ViewClassInheritance = Args.getLastArgValue(OPT_cxx_inheritance_view);
   Opts.ASTMergeFiles = Args.getAllArgValues(OPT_ast_merge);
   Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
+  Opts.FixWhatYouCan = Args.hasArg(OPT_fix_what_you_can);
+  Opts.Modules = Args.getAllArgValues(OPT_import_module);
 
   InputKind DashX = IK_None;
   if (const Arg *A = Args.getLastArg(OPT_x)) {
@@ -1123,7 +1154,7 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   for (arg_iterator it = Args.filtered_begin(OPT_I, OPT_F),
          ie = Args.filtered_end(); it != ie; ++it)
     Opts.AddPath((*it)->getValue(Args), frontend::Angled, true,
-                 /*IsFramework=*/ (*it)->getOption().matches(OPT_F));
+                 /*IsFramework=*/ (*it)->getOption().matches(OPT_F), true);
 
   // Add -iprefix/-iwith-prefix/-iwithprefixbefore options.
   llvm::StringRef Prefix = ""; // FIXME: This isn't the correct default prefix.
@@ -1135,21 +1166,22 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
       Prefix = A->getValue(Args);
     else if (A->getOption().matches(OPT_iwithprefix))
       Opts.AddPath(Prefix.str() + A->getValue(Args),
-                   frontend::System, false, false);
+                   frontend::System, false, false, true);
     else
       Opts.AddPath(Prefix.str() + A->getValue(Args),
-                   frontend::Angled, false, false);
+                   frontend::Angled, false, false, true);
   }
 
   for (arg_iterator it = Args.filtered_begin(OPT_idirafter),
          ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::After, true, false);
+    Opts.AddPath((*it)->getValue(Args), frontend::After, true, false, true);
   for (arg_iterator it = Args.filtered_begin(OPT_iquote),
          ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::Quoted, true, false);
-  for (arg_iterator it = Args.filtered_begin(OPT_isystem),
+    Opts.AddPath((*it)->getValue(Args), frontend::Quoted, true, false, true);
+  for (arg_iterator it = Args.filtered_begin(OPT_isystem, OPT_iwithsysroot),
          ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::System, true, false);
+    Opts.AddPath((*it)->getValue(Args), frontend::System, true, false,
+                 (*it)->getOption().matches(OPT_iwithsysroot));
 
   // FIXME: Need options for the various environment variables!
 }
@@ -1287,6 +1319,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
                                    !Opts.AsmPreprocessor);
   Opts.PascalStrings = Args.hasArg(OPT_fpascal_strings);
   Opts.Microsoft = Args.hasArg(OPT_fms_extensions);
+  Opts.Borland = Args.hasArg(OPT_fborland_extensions);
   Opts.WritableStrings = Args.hasArg(OPT_fwritable_strings);
   Opts.ConstStrings = Args.hasArg(OPT_Wwrite_strings);
   if (Args.hasArg(OPT_fno_lax_vector_conversions))
@@ -1360,6 +1393,24 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
     Opts.TokenCache = Opts.ImplicitPTHInclude;
   Opts.UsePredefines = !Args.hasArg(OPT_undef);
   Opts.DetailedRecord = Args.hasArg(OPT_detailed_preprocessing_record);
+  Opts.DisablePCHValidation = Args.hasArg(OPT_fno_validate_pch);
+
+  if (const Arg *A = Args.getLastArg(OPT_preamble_bytes_EQ)) {
+    llvm::StringRef Value(A->getValue(Args));
+    size_t Comma = Value.find(',');
+    unsigned Bytes = 0;
+    unsigned EndOfLine = 0;
+    
+    if (Comma == llvm::StringRef::npos ||
+        Value.substr(0, Comma).getAsInteger(10, Bytes) ||
+        Value.substr(Comma + 1).getAsInteger(10, EndOfLine))
+      Diags.Report(diag::err_drv_preamble_format);
+    else {
+      Opts.PrecompiledPreambleBytes.first = Bytes;
+      Opts.PrecompiledPreambleBytes.second = (EndOfLine != 0);
+    }
+  }
+    
   // Add macros from the command line.
   for (arg_iterator it = Args.filtered_begin(OPT_D, OPT_U),
          ie = Args.filtered_end(); it != ie; ++it) {
@@ -1379,7 +1430,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
     // PCH is handled specially, we need to extra the original include path.
     if (A->getOption().matches(OPT_include_pch)) {
       std::string OriginalFile =
-        PCHReader::getOriginalSourceFile(A->getValue(Args), Diags);
+        ASTReader::getOriginalSourceFile(A->getValue(Args), Diags);
       if (OriginalFile.empty())
         continue;
 
@@ -1411,10 +1462,11 @@ static void ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
                                         ArgList &Args) {
   using namespace cc1options;
   Opts.ShowCPP = !Args.hasArg(OPT_dM);
-  Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
-  Opts.ShowLineMarkers = !Args.hasArg(OPT_P);
   Opts.ShowComments = Args.hasArg(OPT_C);
+  Opts.ShowHeaderIncludes = Args.hasArg(OPT_H);
+  Opts.ShowLineMarkers = !Args.hasArg(OPT_P);
   Opts.ShowMacroComments = Args.hasArg(OPT_CC);
+  Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
 }
 
 static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args) {
@@ -1422,16 +1474,13 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args) {
   Opts.ABI = Args.getLastArgValue(OPT_target_abi);
   Opts.CXXABI = Args.getLastArgValue(OPT_cxx_abi);
   Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
-  Opts.Triple = Args.getLastArgValue(OPT_triple);
   Opts.Features = Args.getAllArgValues(OPT_target_feature);
+  Opts.LinkerVersion = Args.getLastArgValue(OPT_target_linker_version);
+  Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
 
   // Use the host triple if unspecified.
   if (Opts.Triple.empty())
     Opts.Triple = llvm::sys::getHostTriple();
-
-  // Use the Itanium C++ ABI if unspecified.
-  if (Opts.CXXABI.empty())
-    Opts.CXXABI = "itanium";
 }
 
 //
diff --git a/lib/Frontend/DependencyFile.cpp b/lib/Frontend/DependencyFile.cpp
index 14aee3559c9e..cdff8077ee47 100644
--- a/lib/Frontend/DependencyFile.cpp
+++ b/lib/Frontend/DependencyFile.cpp
@@ -53,7 +53,6 @@ public:
 
   virtual void EndOfMainFile() {
     OutputDependencyFile();
-    OS->flush();
     delete OS;
     OS = 0;
   }
diff --git a/lib/Frontend/DiagChecker.cpp b/lib/Frontend/DiagChecker.cpp
index a50cc99ab79e..66d7ed7cf40a 100644
--- a/lib/Frontend/DiagChecker.cpp
+++ b/lib/Frontend/DiagChecker.cpp
@@ -13,7 +13,7 @@
 
 #include "clang/Frontend/Utils.h"
 #include "clang/Frontend/TextDiagnosticBuffer.h"
-#include "clang/Sema/ParseAST.h"
+#include "clang/Parse/ParseAST.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Preprocessor.h"
diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp
index dbbf69c8b12c..b244c5ce0225 100644
--- a/lib/Frontend/FrontendAction.cpp
+++ b/lib/Frontend/FrontendAction.cpp
@@ -8,13 +8,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/FrontendAction.h"
+#include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
-#include "clang/Sema/ParseAST.h"
+#include "clang/Parse/ParseAST.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -50,7 +51,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
 
     llvm::IntrusiveRefCntPtr<Diagnostic> Diags(&CI.getDiagnostics());
     std::string Error;
-    ASTUnit *AST = ASTUnit::LoadFromPCHFile(Filename, Diags);
+    ASTUnit *AST = ASTUnit::LoadFromASTFile(Filename, Diags);
     if (!AST)
       goto failure;
 
@@ -112,18 +113,21 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   if (!usesPreprocessorOnly()) {
     CI.createASTContext();
 
-    /// Use PCH? If so, we want the PCHReader active before the consumer
-    /// is created, because the consumer might be interested in the reader
-    /// (e.g. the PCH writer for chaining).
+    llvm::OwningPtr<ASTConsumer> Consumer(CreateASTConsumer(CI, Filename));
+
+    /// Use PCH?
     if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
       assert(hasPCHSupport() && "This action does not have PCH support!");
       CI.createPCHExternalASTSource(
-        CI.getPreprocessorOpts().ImplicitPCHInclude);
+                                CI.getPreprocessorOpts().ImplicitPCHInclude,
+                                CI.getPreprocessorOpts().DisablePCHValidation,
+                                CI.getInvocation().getFrontendOpts().ChainedPCH?
+                                 Consumer->GetASTDeserializationListener() : 0);
       if (!CI.getASTContext().getExternalSource())
         goto failure;
     }
 
-    CI.setASTConsumer(CreateASTConsumer(CI, Filename));
+    CI.setASTConsumer(Consumer.take());
     if (!CI.hasASTConsumer())
       goto failure;
   }
@@ -192,12 +196,16 @@ void FrontendAction::EndSourceFile() {
   // FIXME: There is more per-file stuff we could just drop here?
   if (CI.getFrontendOpts().DisableFree) {
     CI.takeASTConsumer();
-    if (!isCurrentFileAST())
+    if (!isCurrentFileAST()) {
+      CI.takeSema();
       CI.takeASTContext();
+    }
   } else {
-    CI.setASTConsumer(0);
-    if (!isCurrentFileAST())
+    if (!isCurrentFileAST()) {
+      CI.setSema(0);
       CI.setASTContext(0);
+    }
+    CI.setASTConsumer(0);
   }
 
   // Inform the preprocessor we are done.
@@ -221,6 +229,7 @@ void FrontendAction::EndSourceFile() {
   CI.getDiagnosticClient().EndSourceFile();
 
   if (isCurrentFileAST()) {
+    CI.takeSema();
     CI.takeASTContext();
     CI.takePreprocessor();
     CI.takeSourceManager();
@@ -249,9 +258,10 @@ void ASTFrontendAction::ExecuteAction() {
   if (CI.hasCodeCompletionConsumer())
     CompletionConsumer = &CI.getCodeCompletionConsumer();
 
-  ParseAST(CI.getPreprocessor(), &CI.getASTConsumer(), CI.getASTContext(),
-           CI.getFrontendOpts().ShowStats,
-           usesCompleteTranslationUnit(), CompletionConsumer);
+  if (!CI.hasSema())
+    CI.createSema(usesCompleteTranslationUnit(), CompletionConsumer);
+
+  ParseAST(CI.getSema(), CI.getFrontendOpts().ShowStats);
 }
 
 ASTConsumer *
diff --git a/lib/Frontend/FrontendActions.cpp b/lib/Frontend/FrontendActions.cpp
index 3a53dee80614..5bc6506e1fd9 100644
--- a/lib/Frontend/FrontendActions.cpp
+++ b/lib/Frontend/FrontendActions.cpp
@@ -18,7 +18,9 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/Utils.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
@@ -69,22 +71,35 @@ ASTConsumer *DeclContextPrintAction::CreateASTConsumer(CompilerInstance &CI,
 
 ASTConsumer *GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI,
                                                   llvm::StringRef InFile) {
-  const std::string &Sysroot = CI.getHeaderSearchOpts().Sysroot;
-  if (CI.getFrontendOpts().RelocatablePCH &&
-      Sysroot.empty()) {
-    CI.getDiagnostics().Report(diag::err_relocatable_without_without_isysroot);
+  std::string Sysroot;
+  llvm::raw_ostream *OS = 0;
+  bool Chaining;
+  if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OS, Chaining))
     return 0;
+
+  const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
+                             Sysroot.c_str() : 0;  
+  return new PCHGenerator(CI.getPreprocessor(), Chaining, isysroot, OS);
+}
+
+bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
+                                                    llvm::StringRef InFile,
+                                                    std::string &Sysroot,
+                                                    llvm::raw_ostream *&OS,
+                                                    bool &Chaining) {
+  Sysroot = CI.getHeaderSearchOpts().Sysroot;
+  if (CI.getFrontendOpts().RelocatablePCH && Sysroot.empty()) {
+    CI.getDiagnostics().Report(diag::err_relocatable_without_isysroot);
+    return true;
   }
 
-  llvm::raw_ostream *OS = CI.createDefaultOutputFile(true, InFile);
+  OS = CI.createDefaultOutputFile(true, InFile);
   if (!OS)
-    return 0;
+    return true;
 
-  PCHReader *Chain = CI.getInvocation().getFrontendOpts().ChainedPCH ?
-                               CI.getPCHReader() : 0;
-  const char *isysroot = CI.getFrontendOpts().RelocatablePCH ?
-                             Sysroot.c_str() : 0;
-  return CreatePCHGenerator(CI.getPreprocessor(), OS, Chain, isysroot);
+  Chaining = CI.getInvocation().getFrontendOpts().ChainedPCH &&
+             !CI.getPreprocessorOpts().ImplicitPCHInclude.empty();
+  return false;
 }
 
 ASTConsumer *InheritanceViewAction::CreateASTConsumer(CompilerInstance &CI,
@@ -146,15 +161,6 @@ void GeneratePTHAction::ExecuteAction() {
   CacheTokens(CI.getPreprocessor(), OS);
 }
 
-void ParseOnlyAction::ExecuteAction() {
-  Preprocessor &PP = getCompilerInstance().getPreprocessor();
-  llvm::OwningPtr<Action> PA(new MinimalAction(PP));
-
-  Parser P(PP, *PA);
-  PP.EnterMainSourceFile();
-  P.ParseTranslationUnit();
-}
-
 void PreprocessOnlyAction::ExecuteAction() {
   Preprocessor &PP = getCompilerInstance().getPreprocessor();
 
@@ -169,19 +175,6 @@ void PreprocessOnlyAction::ExecuteAction() {
   } while (Tok.isNot(tok::eof));
 }
 
-void PrintParseAction::ExecuteAction() {
-  CompilerInstance &CI = getCompilerInstance();
-  Preprocessor &PP = getCompilerInstance().getPreprocessor();
-  llvm::raw_ostream *OS = CI.createDefaultOutputFile(false, getCurrentFile());
-  if (!OS) return;
-
-  llvm::OwningPtr<Action> PA(CreatePrintParserActionsAction(PP, OS));
-
-  Parser P(PP, *PA);
-  PP.EnterMainSourceFile();
-  P.ParseTranslationUnit();
-}
-
 void PrintPreprocessedAction::ExecuteAction() {
   CompilerInstance &CI = getCompilerInstance();
   // Output file needs to be set to 'Binary', to avoid converting Unix style
@@ -192,3 +185,32 @@ void PrintPreprocessedAction::ExecuteAction() {
   DoPrintPreprocessedInput(CI.getPreprocessor(), OS,
                            CI.getPreprocessorOutputOpts());
 }
+
+void PrintPreambleAction::ExecuteAction() {
+  switch (getCurrentFileKind()) {
+  case IK_C:
+  case IK_CXX:
+  case IK_ObjC:
+  case IK_ObjCXX:
+  case IK_OpenCL:
+    break;
+      
+  case IK_None:
+  case IK_Asm:
+  case IK_PreprocessedC:
+  case IK_PreprocessedCXX:
+  case IK_PreprocessedObjC:
+  case IK_PreprocessedObjCXX:
+  case IK_AST:
+  case IK_LLVM_IR:
+    // We can't do anything with these.
+    return;
+  }
+  
+  llvm::MemoryBuffer *Buffer = llvm::MemoryBuffer::getFile(getCurrentFile());
+  if (Buffer) {
+    unsigned Preamble = Lexer::ComputePreamble(Buffer).first;
+    llvm::outs().write(Buffer->getBufferStart(), Preamble);
+    delete Buffer;
+  }
+}
diff --git a/lib/Frontend/InitHeaderSearch.cpp b/lib/Frontend/InitHeaderSearch.cpp
index d640d42492f1..df917136203f 100644
--- a/lib/Frontend/InitHeaderSearch.cpp
+++ b/lib/Frontend/InitHeaderSearch.cpp
@@ -1,4 +1,4 @@
-//===--- InitHeaderSearch.cpp - Initialize header search paths ----------*-===//
+//===--- InitHeaderSearch.cpp - Initialize header search paths ------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -195,6 +195,8 @@ void InitHeaderSearch::AddMinGWCPlusPlusIncludePaths(llvm::StringRef Base,
           System, true, false, false);
   AddPath(Base + "/" + Arch + "/" + Version + "/include/c++",
           System, true, false, false);
+  AddPath(Base + "/" + Arch + "/" + Version + "/include/c++/" + Arch,
+          System, true, false, false);
   AddPath(Base + "/" + Arch + "/" + Version + "/include/c++/backward",
           System, true, false, false);
 }
@@ -323,9 +325,19 @@ static bool getSystemRegistryString(const char*, const char*, char*, size_t) {
 
   // Get Visual Studio installation directory.
 static bool getVisualStudioDir(std::string &path) {
+  // First check the environment variables that vsvars32.bat sets.
+  const char* vcinstalldir = getenv("VCINSTALLDIR");
+  if(vcinstalldir) {
+    char *p = const_cast<char *>(strstr(vcinstalldir, "\\VC"));
+    if (p)
+      *p = '\0';
+    path = vcinstalldir;
+    return(true);
+  }
+
   char vsIDEInstallDir[256];
   char vsExpressIDEInstallDir[256];
-  // Try the Windows registry first.
+  // Then try the windows registry.
   bool hasVCDir = getSystemRegistryString(
     "HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\VisualStudio\\$VERSION",
     "InstallDir", vsIDEInstallDir, sizeof(vsIDEInstallDir) - 1);
@@ -440,7 +452,7 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
       if (getVisualStudioDir(VSDir)) {
         AddPath(VSDir + "\\VC\\include", System, false, false, false);
         if (getWindowsSDKDir(WindowsSDKDir))
-          AddPath(WindowsSDKDir, System, false, false, false);
+          AddPath(WindowsSDKDir + "\\include", System, false, false, false);
         else
           AddPath(VSDir + "\\VC\\PlatformSDK\\Include",
             System, false, false, false);
@@ -510,7 +522,7 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
     AddPath("/boot/develop/headers/glibc", System, true, false, false);
     AddPath("/boot/develop/headers/posix", System, true, false, false);
     AddPath("/boot/develop/headers",  System, true, false, false);
-  	break;
+    break;
   case llvm::Triple::MinGW64:
   case llvm::Triple::MinGW32:
     AddPath("c:/mingw/include", System, true, false, false);
@@ -549,12 +561,16 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
         System, true, false, false);
     break;
   case llvm::Triple::MinGW64:
+    // Try gcc 4.5.0
+    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw64", "4.5.0");
     // Try gcc 4.4.0
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw64", "4.4.0");
     // Try gcc 4.3.0
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw64", "4.3.0");
     // Fall through.
   case llvm::Triple::MinGW32:
+   // Try gcc 4.5.0
+    AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.5.0"); 
     // Try gcc 4.4.0
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.4.0");
     // Try gcc 4.3.0
@@ -716,6 +732,11 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
     AddGnuCPlusPlusIncludePaths(
         "/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.3/include/g++-v4",
         "x86_64-pc-linux-gnu", "32", "", triple);
+
+    // Gentoo amd64 llvm-gcc trunk
+    AddGnuCPlusPlusIncludePaths(
+        "/usr/lib/llvm-gcc-4.2-9999/include/c++/4.2.1",
+        "x86_64-pc-linux-gnu", "", "", triple);
     
     break;
   case llvm::Triple::FreeBSD:
@@ -723,6 +744,17 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple) {
     // FreeBSD 7.3
     AddGnuCPlusPlusIncludePaths("/usr/include/c++/4.2", "", "", "", triple);
     break;
+  case llvm::Triple::NetBSD:
+    AddGnuCPlusPlusIncludePaths("/usr/include/g++", "", "", "", triple);
+    break;
+  case llvm::Triple::OpenBSD: {
+    std::string t = triple.getTriple();
+    if (t.substr(0, 6) == "x86_64")
+      t.replace(0, 6, "amd64");
+    AddGnuCPlusPlusIncludePaths("/usr/include/g++",
+                                t, "", "", triple);
+    break;
+  }
   case llvm::Triple::Minix:
     AddGnuCPlusPlusIncludePaths("/usr/gnu/include/c++/4.4.3",
                                 "", "", "", triple);
@@ -889,7 +921,7 @@ void clang::ApplyHeaderSearchOptions(HeaderSearch &HS,
   for (unsigned i = 0, e = HSOpts.UserEntries.size(); i != e; ++i) {
     const HeaderSearchOptions::Entry &E = HSOpts.UserEntries[i];
     Init.AddPath(E.Path, E.Group, false, E.IsUserSupplied, E.IsFramework,
-                 false);
+                 !E.IsSysRootRelative);
   }
 
   // Add entries from CPATH and friends.
diff --git a/lib/Frontend/InitPreprocessor.cpp b/lib/Frontend/InitPreprocessor.cpp
index 889b6e52a456..0d07192b50e5 100644
--- a/lib/Frontend/InitPreprocessor.cpp
+++ b/lib/Frontend/InitPreprocessor.cpp
@@ -169,10 +169,11 @@ static void DefineTypeSize(llvm::StringRef MacroName, unsigned TypeWidth,
                            llvm::StringRef ValSuffix, bool isSigned,
                            MacroBuilder& Builder) {
   long long MaxVal;
-  if (isSigned)
-    MaxVal = (1LL << (TypeWidth - 1)) - 1;
-  else
-    MaxVal = ~0LL >> (64-TypeWidth);
+  if (isSigned) {
+    assert(TypeWidth != 1);
+    MaxVal = ~0ULL >> (65-TypeWidth);
+  } else
+    MaxVal = ~0ULL >> (64-TypeWidth);
 
   Builder.defineMacro(MacroName, llvm::Twine(MaxVal) + ValSuffix);
 }
@@ -318,7 +319,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
       Builder.defineMacro("__cplusplus");
     else
       // C++ [cpp.predefined]p1:
-      //   The name_ _cplusplusis defined to the value199711Lwhen compiling a
+      //   The name_ _cplusplusis defined to the value 199711L when compiling a
       //   C++ translation unit.
       Builder.defineMacro("__cplusplus", "199711L");
     Builder.defineMacro("__private_extern__", "extern");
@@ -339,9 +340,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
       // Since we define wchar_t in C++ mode.
       Builder.defineMacro("_WCHAR_T_DEFINED");
       Builder.defineMacro("_NATIVE_WCHAR_T_DEFINED");
-      // FIXME:  This should be temporary until we have a __pragma
-      // solution, to avoid some errors flagged in VC++ headers.
-      Builder.defineMacro("_CRT_SECURE_CPP_OVERLOAD_SECURE_NAMES", "0");
+      Builder.append("class type_info;");
     }
   }
 
@@ -477,7 +476,7 @@ static void InitializeFileRemapping(Diagnostic &Diags,
                                     FileManager &FileMgr,
                                     const PreprocessorOptions &InitOpts) {
   // Remap files in the source manager (with buffers).
-  for (PreprocessorOptions::remapped_file_buffer_iterator
+  for (PreprocessorOptions::const_remapped_file_buffer_iterator
          Remap = InitOpts.remapped_file_buffer_begin(),
          RemapEnd = InitOpts.remapped_file_buffer_end();
        Remap != RemapEnd;
@@ -489,19 +488,21 @@ static void InitializeFileRemapping(Diagnostic &Diags,
     if (!FromFile) {
       Diags.Report(diag::err_fe_remap_missing_from_file)
         << Remap->first;
-      delete Remap->second;
+      if (!InitOpts.RetainRemappedFileBuffers)
+        delete Remap->second;
       continue;
     }
 
     // Override the contents of the "from" file with the contents of
     // the "to" file.
-    SourceMgr.overrideFileContents(FromFile, Remap->second);
+    SourceMgr.overrideFileContents(FromFile, Remap->second,
+                                   InitOpts.RetainRemappedFileBuffers);
   }
 
   // Remap files in the source manager (with other files).
-  for (PreprocessorOptions::remapped_file_iterator
-       Remap = InitOpts.remapped_file_begin(),
-       RemapEnd = InitOpts.remapped_file_end();
+  for (PreprocessorOptions::const_remapped_file_iterator
+         Remap = InitOpts.remapped_file_begin(),
+         RemapEnd = InitOpts.remapped_file_end();
        Remap != RemapEnd;
        ++Remap) {
     // Find the file that we're mapping to.
@@ -596,6 +597,10 @@ void clang::InitializePreprocessor(Preprocessor &PP,
   if (!PP.getLangOptions().AsmPreprocessor)
     Builder.append("# 1 \"<built-in>\" 2");
 
+  // Instruct the preprocessor to skip the preamble.
+  PP.setSkipMainFilePreamble(InitOpts.PrecompiledPreambleBytes.first,
+                             InitOpts.PrecompiledPreambleBytes.second);
+                          
   // Copy PredefinedBuffer into the Preprocessor.
   PP.setPredefines(Predefines.str());
 
diff --git a/lib/Frontend/Makefile b/lib/Frontend/Makefile
index 3eb4bc95f4c3..3c13ad69cc67 100644
--- a/lib/Frontend/Makefile
+++ b/lib/Frontend/Makefile
@@ -9,7 +9,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangFrontend
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Frontend/PrintParserCallbacks.cpp b/lib/Frontend/PrintParserCallbacks.cpp
deleted file mode 100644
index 922067748336..000000000000
--- a/lib/Frontend/PrintParserCallbacks.cpp
+++ /dev/null
@@ -1,852 +0,0 @@
-//===--- PrintParserActions.cpp - Implement -parse-print-callbacks mode ---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This code simply runs the preprocessor on the input file and prints out the
-// result.  This is the traditional behavior of the -E option.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Frontend/Utils.h"
-#include "clang/Parse/Action.h"
-#include "clang/Parse/DeclSpec.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace clang;
-
-namespace {
-  class ParserPrintActions : public MinimalAction {
-  llvm::raw_ostream& Out;
-
-  public:
-    ParserPrintActions(Preprocessor &PP, llvm::raw_ostream& OS)
-      : MinimalAction(PP), Out(OS) {}
-
-    // Printing Functions which also must call MinimalAction
-
-    /// ActOnDeclarator - This callback is invoked when a declarator is parsed
-    /// and 'Init' specifies the initializer if any.  This is for things like:
-    /// "int X = 4" or "typedef int foo".
-    virtual DeclPtrTy ActOnDeclarator(Scope *S, Declarator &D) {
-      Out << __FUNCTION__ << " ";
-      if (IdentifierInfo *II = D.getIdentifier()) {
-        Out << "'" << II->getName() << "'";
-      } else {
-        Out << "<anon>";
-      }
-      Out << "\n";
-
-      // Pass up to EmptyActions so that the symbol table is maintained right.
-      return MinimalAction::ActOnDeclarator(S, D);
-    }
-    /// ActOnPopScope - This callback is called immediately before the specified
-    /// scope is popped and deleted.
-    virtual void ActOnPopScope(SourceLocation Loc, Scope *S) {
-      Out << __FUNCTION__ << "\n";
-      return MinimalAction::ActOnPopScope(Loc, S);
-    }
-
-    /// ActOnTranslationUnitScope - This callback is called once, immediately
-    /// after creating the translation unit scope (in Parser::Initialize).
-    virtual void ActOnTranslationUnitScope(SourceLocation Loc, Scope *S) {
-      Out << __FUNCTION__ << "\n";
-      MinimalAction::ActOnTranslationUnitScope(Loc, S);
-    }
-
-
-    Action::DeclPtrTy ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
-                                               IdentifierInfo *ClassName,
-                                               SourceLocation ClassLoc,
-                                               IdentifierInfo *SuperName,
-                                               SourceLocation SuperLoc,
-                                               const DeclPtrTy *ProtoRefs,
-                                               unsigned NumProtocols,
-                                               const SourceLocation *ProtoLocs,
-                                               SourceLocation EndProtoLoc,
-                                               AttributeList *AttrList) {
-      Out << __FUNCTION__ << "\n";
-      return MinimalAction::ActOnStartClassInterface(AtInterfaceLoc,
-                                                     ClassName, ClassLoc,
-                                                     SuperName, SuperLoc,
-                                                     ProtoRefs, NumProtocols,
-                                                     ProtoLocs, EndProtoLoc,
-                                                     AttrList);
-    }
-
-    /// ActOnForwardClassDeclaration -
-    /// Scope will always be top level file scope.
-    Action::DeclPtrTy ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
-                                                   IdentifierInfo **IdentList,
-                                                   SourceLocation *IdentLocs,
-                                                   unsigned NumElts) {
-      Out << __FUNCTION__ << "\n";
-      return MinimalAction::ActOnForwardClassDeclaration(AtClassLoc, IdentList,
-                                                         IdentLocs, NumElts);
-    }
-
-    // Pure Printing
-
-    /// ActOnParamDeclarator - This callback is invoked when a parameter
-    /// declarator is parsed. This callback only occurs for functions
-    /// with prototypes. S is the function prototype scope for the
-    /// parameters (C++ [basic.scope.proto]).
-    virtual DeclPtrTy ActOnParamDeclarator(Scope *S, Declarator &D) {
-      Out << __FUNCTION__ << " ";
-      if (IdentifierInfo *II = D.getIdentifier()) {
-        Out << "'" << II->getName() << "'";
-      } else {
-        Out << "<anon>";
-      }
-      Out << "\n";
-      return DeclPtrTy();
-    }
-
-    /// AddInitializerToDecl - This action is called immediately after
-    /// ParseDeclarator (when an initializer is present). The code is factored
-    /// this way to make sure we are able to handle the following:
-    ///   void func() { int xx = xx; }
-    /// This allows ActOnDeclarator to register "xx" prior to parsing the
-    /// initializer. The declaration above should still result in a warning,
-    /// since the reference to "xx" is uninitialized.
-    virtual void AddInitializerToDecl(DeclPtrTy Dcl, ExprArg Init) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    /// FinalizeDeclaratorGroup - After a sequence of declarators are parsed,
-    /// this gives the actions implementation a chance to process the group as
-    /// a whole.
-    virtual DeclGroupPtrTy FinalizeDeclaratorGroup(Scope *S, const DeclSpec& DS,
-                                                   DeclPtrTy *Group,
-                                                   unsigned NumDecls) {
-      Out << __FUNCTION__ << "\n";
-      return DeclGroupPtrTy();
-    }
-
-    /// ActOnStartOfFunctionDef - This is called at the start of a function
-    /// definition, instead of calling ActOnDeclarator.  The Declarator includes
-    /// information about formal arguments that are part of this function.
-    virtual DeclPtrTy ActOnStartOfFunctionDef(Scope *FnBodyScope,
-                                              Declarator &D){
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    /// ActOnStartOfFunctionDef - This is called at the start of a function
-    /// definition, after the FunctionDecl has already been created.
-    virtual DeclPtrTy ActOnStartOfFunctionDef(Scope *FnBodyScope, DeclPtrTy D) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual void ActOnStartOfObjCMethodDef(Scope *FnBodyScope, DeclPtrTy D) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    /// ActOnFunctionDefBody - This is called when a function body has completed
-    /// parsing.  Decl is the DeclTy returned by ParseStartOfFunctionDef.
-    virtual DeclPtrTy ActOnFinishFunctionBody(DeclPtrTy Decl, StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual DeclPtrTy ActOnFileScopeAsmDecl(SourceLocation Loc,
-                                            ExprArg AsmString) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    /// ParsedFreeStandingDeclSpec - This method is invoked when a declspec with
-    /// no declarator (e.g. "struct foo;") is parsed.
-    virtual DeclPtrTy ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
-                                                 DeclSpec &DS) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    /// ActOnLinkageSpec - Parsed a C++ linkage-specification that
-    /// contained braces. Lang/StrSize contains the language string that
-    /// was parsed at location Loc. Decls/NumDecls provides the
-    /// declarations parsed inside the linkage specification.
-    virtual DeclPtrTy ActOnLinkageSpec(SourceLocation Loc,
-                                       SourceLocation LBrace,
-                                       SourceLocation RBrace, const char *Lang,
-                                       unsigned StrSize,
-                                       DeclPtrTy *Decls, unsigned NumDecls) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    /// ActOnLinkageSpec - Parsed a C++ linkage-specification without
-    /// braces. Lang/StrSize contains the language string that was
-    /// parsed at location Loc. D is the declaration parsed.
-    virtual DeclPtrTy ActOnLinkageSpec(SourceLocation Loc, const char *Lang,
-                                       unsigned StrSize, DeclPtrTy D) {
-      return DeclPtrTy();
-    }
-
-    //===------------------------------------------------------------------===//
-    // Type Parsing Callbacks.
-    //===------------------------------------------------------------------===//
-
-    virtual TypeResult ActOnTypeName(Scope *S, Declarator &D) {
-      Out << __FUNCTION__ << "\n";
-      return TypeResult();
-    }
-
-    virtual DeclPtrTy ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
-                               SourceLocation KWLoc, CXXScopeSpec &SS,
-                               IdentifierInfo *Name, SourceLocation NameLoc,
-                               AttributeList *Attr, AccessSpecifier AS,
-                               MultiTemplateParamsArg TemplateParameterLists,
-                               bool &OwnedDecl, bool &IsDependent) {
-      // TagType is an instance of DeclSpec::TST, indicating what kind of tag this
-      // is (struct/union/enum/class).
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    /// Act on @defs() element found when parsing a structure.  ClassName is the
-    /// name of the referenced class.
-    virtual void ActOnDefs(Scope *S, DeclPtrTy TagD, SourceLocation DeclStart,
-                           IdentifierInfo *ClassName,
-                           llvm::SmallVectorImpl<DeclPtrTy> &Decls) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual DeclPtrTy ActOnField(Scope *S, DeclPtrTy TagD,
-                                 SourceLocation DeclStart,
-                                 Declarator &D, ExprTy *BitfieldWidth) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual DeclPtrTy ActOnIvar(Scope *S, SourceLocation DeclStart,
-                                DeclPtrTy IntfDecl,
-                                Declarator &D, ExprTy *BitfieldWidth,
-                                tok::ObjCKeywordKind visibility) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual void ActOnFields(Scope* S, SourceLocation RecLoc, DeclPtrTy TagDecl,
-                             DeclPtrTy *Fields, unsigned NumFields,
-                             SourceLocation LBrac, SourceLocation RBrac,
-                             AttributeList *AttrList) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual DeclPtrTy ActOnEnumConstant(Scope *S, DeclPtrTy EnumDecl,
-                                        DeclPtrTy LastEnumConstant,
-                                        SourceLocation IdLoc,IdentifierInfo *Id,
-                                        SourceLocation EqualLoc, ExprTy *Val) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual void ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
-                               SourceLocation RBraceLoc, DeclPtrTy EnumDecl,
-                               DeclPtrTy *Elements, unsigned NumElements,
-                               Scope *S, AttributeList *AttrList) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    //===------------------------------------------------------------------===//
-    // Statement Parsing Callbacks.
-    //===------------------------------------------------------------------===//
-
-    virtual OwningStmtResult ActOnNullStmt(SourceLocation SemiLoc) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnCompoundStmt(SourceLocation L,
-                                               SourceLocation R,
-                                               MultiStmtArg Elts,
-                                               bool isStmtExpr) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnDeclStmt(DeclGroupPtrTy Decl,
-                                           SourceLocation StartLoc,
-                                           SourceLocation EndLoc) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnExprStmt(FullExprArg Expr) {
-      Out << __FUNCTION__ << "\n";
-      return OwningStmtResult(*this, Expr->release());
-    }
-
-    /// ActOnCaseStmt - Note that this handles the GNU 'case 1 ... 4' extension,
-    /// which can specify an RHS value.
-    virtual OwningStmtResult ActOnCaseStmt(SourceLocation CaseLoc,
-                                           ExprArg LHSVal,
-                                           SourceLocation DotDotDotLoc,
-                                           ExprArg RHSVal,
-                                           SourceLocation ColonLoc) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnDefaultStmt(SourceLocation DefaultLoc,
-                                              SourceLocation ColonLoc,
-                                              StmtArg SubStmt, Scope *CurScope){
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnLabelStmt(SourceLocation IdentLoc,
-                                            IdentifierInfo *II,
-                                            SourceLocation ColonLoc,
-                                            StmtArg SubStmt) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnIfStmt(SourceLocation IfLoc,
-                                         FullExprArg CondVal, DeclPtrTy CondVar,
-                                         StmtArg ThenVal,
-                                         SourceLocation ElseLoc,
-                                         StmtArg ElseVal) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnStartOfSwitchStmt(SourceLocation SwitchLoc,
-                                                    ExprArg Cond, 
-                                                    DeclPtrTy CondVar) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnFinishSwitchStmt(SourceLocation SwitchLoc,
-                                                   StmtArg Switch,
-                                                   StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnWhileStmt(SourceLocation WhileLoc,
-                                            FullExprArg Cond, DeclPtrTy CondVar,
-                                            StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnDoStmt(SourceLocation DoLoc, StmtArg Body,
-                                         SourceLocation WhileLoc,
-                                         SourceLocation LPLoc, ExprArg Cond,
-                                         SourceLocation RPLoc){
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnForStmt(SourceLocation ForLoc,
-                                        SourceLocation LParenLoc,
-                                        StmtArg First, FullExprArg Second,
-                                        DeclPtrTy SecondVar,
-                                        FullExprArg Third, 
-                                        SourceLocation RParenLoc,
-                                        StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnObjCForCollectionStmt(
-                                       SourceLocation ForColLoc,
-                                       SourceLocation LParenLoc,
-                                       StmtArg First, ExprArg Second,
-                                       SourceLocation RParenLoc, StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnGotoStmt(SourceLocation GotoLoc,
-                                           SourceLocation LabelLoc,
-                                           IdentifierInfo *LabelII) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnIndirectGotoStmt(SourceLocation GotoLoc,
-                                                   SourceLocation StarLoc,
-                                                   ExprArg DestExp) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnContinueStmt(SourceLocation ContinueLoc,
-                                               Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnBreakStmt(SourceLocation GotoLoc,
-                                            Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnReturnStmt(SourceLocation ReturnLoc,
-                                             ExprArg RetValExp) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-    virtual OwningStmtResult ActOnAsmStmt(SourceLocation AsmLoc,
-                                          bool IsSimple,
-                                          bool IsVolatile,
-                                          unsigned NumOutputs,
-                                          unsigned NumInputs,
-                                          IdentifierInfo **Names,
-                                          MultiExprArg Constraints,
-                                          MultiExprArg Exprs,
-                                          ExprArg AsmString,
-                                          MultiExprArg Clobbers,
-                                          SourceLocation RParenLoc,
-                                          bool MSAsm) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    // Objective-c statements
-    virtual OwningStmtResult ActOnObjCAtCatchStmt(SourceLocation AtLoc,
-                                                  SourceLocation RParen,
-                                                  DeclPtrTy Parm,
-                                                  StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnObjCAtFinallyStmt(SourceLocation AtLoc,
-                                                    StmtArg Body) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnObjCAtTryStmt(SourceLocation AtLoc,
-                                                StmtArg Try,
-                                                MultiStmtArg CatchStmts,
-                                                StmtArg Finally) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc,
-                                                  ExprArg Throw,
-                                                  Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc,
-                                                         ExprArg SynchExpr,
-                                                         StmtArg SynchBody) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    // C++ Statements
-    virtual DeclPtrTy ActOnExceptionDeclarator(Scope *S, Declarator &D) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual OwningStmtResult ActOnCXXCatchBlock(SourceLocation CatchLoc,
-                                                DeclPtrTy ExceptionDecl,
-                                                StmtArg HandlerBlock) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    virtual OwningStmtResult ActOnCXXTryBlock(SourceLocation TryLoc,
-                                              StmtArg TryBlock,
-                                              MultiStmtArg Handlers) {
-      Out << __FUNCTION__ << "\n";
-      return StmtEmpty();
-    }
-
-    //===------------------------------------------------------------------===//
-    // Expression Parsing Callbacks.
-    //===------------------------------------------------------------------===//
-
-    // Primary Expressions.
-
-    /// ActOnIdentifierExpr - Parse an identifier in expression context.
-    /// 'HasTrailingLParen' indicates whether or not the identifier has a '('
-    /// token immediately after it.
-    virtual OwningExprResult ActOnIdentifierExpr(Scope *S, SourceLocation Loc,
-                                                 IdentifierInfo &II,
-                                                 bool HasTrailingLParen,
-                                                 const CXXScopeSpec *SS,
-                                                 bool isAddressOfOperand) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXOperatorFunctionIdExpr(
-                               Scope *S, SourceLocation OperatorLoc,
-                               OverloadedOperatorKind Op,
-                               bool HasTrailingLParen, const CXXScopeSpec &SS,
-                               bool isAddressOfOperand) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXConversionFunctionExpr(
-                               Scope *S, SourceLocation OperatorLoc,
-                               TypeTy *Type, bool HasTrailingLParen,
-                               const CXXScopeSpec &SS,bool isAddressOfOperand) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnPredefinedExpr(SourceLocation Loc,
-                                                 tok::TokenKind Kind) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCharacterConstant(const Token &) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnNumericConstant(const Token &) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    /// ActOnStringLiteral - The specified tokens were lexed as pasted string
-    /// fragments (e.g. "foo" "bar" L"baz").
-    virtual OwningExprResult ActOnStringLiteral(const Token *Toks,
-                                                unsigned NumToks) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnParenExpr(SourceLocation L, SourceLocation R,
-                                            ExprArg Val) {
-      Out << __FUNCTION__ << "\n";
-      return move(Val);  // Default impl returns operand.
-    }
-
-    // Postfix Expressions.
-    virtual OwningExprResult ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc,
-                                                 tok::TokenKind Kind,
-                                                 ExprArg Input) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-    virtual OwningExprResult ActOnArraySubscriptExpr(Scope *S, ExprArg Base,
-                                                     SourceLocation LLoc,
-                                                     ExprArg Idx,
-                                                     SourceLocation RLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-    virtual OwningExprResult ActOnMemberReferenceExpr(Scope *S, ExprArg Base,
-                                                      SourceLocation OpLoc,
-                                                      tok::TokenKind OpKind,
-                                                      SourceLocation MemberLoc,
-                                                      IdentifierInfo &Member,
-                                                      DeclPtrTy ImplDecl,
-                                                      const CXXScopeSpec *SS=0) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCallExpr(Scope *S, ExprArg Fn,
-                                           SourceLocation LParenLoc,
-                                           MultiExprArg Args,
-                                           SourceLocation *CommaLocs,
-                                           SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    // Unary Operators.  'Tok' is the token for the operator.
-    virtual OwningExprResult ActOnUnaryOp(Scope *S, SourceLocation OpLoc,
-                                          tok::TokenKind Op, ExprArg Input) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-    virtual OwningExprResult
-      ActOnSizeOfAlignOfExpr(SourceLocation OpLoc, bool isSizeof, bool isType,
-                             void *TyOrEx, const SourceRange &ArgRange) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCompoundLiteral(SourceLocation LParen,
-                                                  TypeTy *Ty,
-                                                  SourceLocation RParen,
-                                                  ExprArg Op) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-    virtual OwningExprResult ActOnInitList(SourceLocation LParenLoc,
-                                           MultiExprArg InitList,
-                                           SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-    virtual OwningExprResult ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
-                                           TypeTy *Ty, SourceLocation RParenLoc,
-                                           ExprArg Op) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnBinOp(Scope *S, SourceLocation TokLoc,
-                                        tok::TokenKind Kind,
-                                        ExprArg LHS, ExprArg RHS) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    /// ActOnConditionalOp - Parse a ?: operation.  Note that 'LHS' may be null
-    /// in the case of a the GNU conditional expr extension.
-    virtual OwningExprResult ActOnConditionalOp(SourceLocation QuestionLoc,
-                                                SourceLocation ColonLoc,
-                                                ExprArg Cond, ExprArg LHS,
-                                                ExprArg RHS) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    //===--------------------- GNU Extension Expressions ------------------===//
-
-    virtual OwningExprResult ActOnAddrLabel(SourceLocation OpLoc,
-                                            SourceLocation LabLoc,
-                                            IdentifierInfo *LabelII) {// "&&foo"
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnStmtExpr(SourceLocation LPLoc,
-                                           StmtArg SubStmt,
-                                           SourceLocation RPLoc) { // "({..})"
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnBuiltinOffsetOf(Scope *S,
-                                                  SourceLocation BuiltinLoc,
-                                                  SourceLocation TypeLoc,
-                                                  TypeTy *Arg1,
-                                                  OffsetOfComponent *CompPtr,
-                                                  unsigned NumComponents,
-                                                  SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    // __builtin_types_compatible_p(type1, type2)
-    virtual OwningExprResult ActOnTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                                                      TypeTy *arg1,TypeTy *arg2,
-                                                      SourceLocation RPLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-    // __builtin_choose_expr(constExpr, expr1, expr2)
-    virtual OwningExprResult ActOnChooseExpr(SourceLocation BuiltinLoc,
-                                             ExprArg cond, ExprArg expr1,
-                                             ExprArg expr2,
-                                             SourceLocation RPLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    // __builtin_va_arg(expr, type)
-    virtual OwningExprResult ActOnVAArg(SourceLocation BuiltinLoc,
-                                  ExprArg expr, TypeTy *type,
-                                  SourceLocation RPLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnGNUNullExpr(SourceLocation TokenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual void ActOnBlockStart(SourceLocation CaretLoc, Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void ActOnBlockError(SourceLocation CaretLoc, Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual OwningExprResult ActOnBlockStmtExpr(SourceLocation CaretLoc,
-                                                StmtArg Body,
-                                                Scope *CurScope) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual DeclPtrTy ActOnStartNamespaceDef(Scope *S, SourceLocation IdentLoc,
-                                             IdentifierInfo *Ident,
-                                             SourceLocation LBrace,
-                                             AttributeList *AttrList) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual void ActOnFinishNamespaceDef(DeclPtrTy Dcl, SourceLocation RBrace) {
-      Out << __FUNCTION__ << "\n";
-      return;
-    }
-
-#if 0
-    // FIXME: AttrList should be deleted by this function, but the definition
-    // would have to be available.
-    virtual DeclPtrTy ActOnUsingDirective(Scope *CurScope,
-                                          SourceLocation UsingLoc,
-                                          SourceLocation NamespcLoc,
-                                          const CXXScopeSpec &SS,
-                                          SourceLocation IdentLoc,
-                                          IdentifierInfo *NamespcName,
-                                          AttributeList *AttrList) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-#endif
-
-    virtual void ActOnParamDefaultArgument(DeclPtrTy param,
-                                           SourceLocation EqualLoc,
-                                           ExprArg defarg) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void ActOnParamUnparsedDefaultArgument(DeclPtrTy param,
-                                                   SourceLocation EqualLoc,
-                                                   SourceLocation ArgLoc) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void ActOnParamDefaultArgumentError(DeclPtrTy param) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void AddCXXDirectInitializerToDecl(DeclPtrTy Dcl,
-                                               SourceLocation LParenLoc,
-                                               MultiExprArg Exprs,
-                                               SourceLocation *CommaLocs,
-                                               SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return;
-    }
-
-    virtual void ActOnStartDelayedCXXMethodDeclaration(Scope *S,
-                                                       DeclPtrTy Method) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void ActOnDelayedCXXMethodParameter(Scope *S, DeclPtrTy Param) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual void ActOnFinishDelayedCXXMethodDeclaration(Scope *S,
-                                                        DeclPtrTy Method) {
-      Out << __FUNCTION__ << "\n";
-    }
-
-    virtual DeclPtrTy ActOnStaticAssertDeclaration(SourceLocation AssertLoc,
-                                                   ExprArg AssertExpr,
-                                                   ExprArg AssertMessageExpr) {
-      Out << __FUNCTION__ << "\n";
-      return DeclPtrTy();
-    }
-
-    virtual OwningExprResult ActOnCXXNamedCast(SourceLocation OpLoc,
-                                               tok::TokenKind Kind,
-                                               SourceLocation LAngleBracketLoc,
-                                               TypeTy *Ty,
-                                               SourceLocation RAngleBracketLoc,
-                                               SourceLocation LParenLoc,
-                                               ExprArg Op,
-                                               SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXTypeid(SourceLocation OpLoc,
-                                            SourceLocation LParenLoc,
-                                            bool isType, void *TyOrExpr,
-                                            SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXThis(SourceLocation ThisLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXBoolLiteral(SourceLocation OpLoc,
-                                                 tok::TokenKind Kind) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXThrow(SourceLocation OpLoc, ExprArg Op) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXTypeConstructExpr(SourceRange TypeRange,
-                                                     TypeTy *TypeRep,
-                                                     SourceLocation LParenLoc,
-                                                     MultiExprArg Exprs,
-                                                     SourceLocation *CommaLocs,
-                                                     SourceLocation RParenLoc) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXConditionDeclarationExpr(Scope *S,
-                                                        SourceLocation StartLoc,
-                                                        Declarator &D,
-                                                        SourceLocation EqualLoc,
-                                                        ExprArg AssignExprVal) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXNew(SourceLocation StartLoc,
-                                         bool UseGlobal,
-                                         SourceLocation PlacementLParen,
-                                         MultiExprArg PlacementArgs,
-                                         SourceLocation PlacementRParen,
-                                         SourceRange TypeIdParens, 
-                                         Declarator &D,
-                                         SourceLocation ConstructorLParen,
-                                         MultiExprArg ConstructorArgs,
-                                         SourceLocation ConstructorRParen) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnCXXDelete(SourceLocation StartLoc,
-                                            bool UseGlobal, bool ArrayForm,
-                                            ExprArg Operand) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-
-    virtual OwningExprResult ActOnUnaryTypeTrait(UnaryTypeTrait OTT,
-                                                 SourceLocation KWLoc,
-                                                 SourceLocation LParen,
-                                                 TypeTy *Ty,
-                                                 SourceLocation RParen) {
-      Out << __FUNCTION__ << "\n";
-      return ExprEmpty();
-    }
-  };
-}
-
-MinimalAction *clang::CreatePrintParserActionsAction(Preprocessor &PP,
-                                                     llvm::raw_ostream* OS) {
-  return new ParserPrintActions(PP, *OS);
-}
diff --git a/lib/Frontend/PrintPreprocessedOutput.cpp b/lib/Frontend/PrintPreprocessedOutput.cpp
index 73bca9a6caa1..cfaf8a23b118 100644
--- a/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -85,6 +85,10 @@ public:
   llvm::raw_ostream &OS;
 private:
   unsigned CurLine;
+
+  /// The current include nesting level, used by header include dumping (-H).
+  unsigned CurrentIncludeDepth;
+
   bool EmittedTokensOnThisLine;
   bool EmittedMacroOnThisLine;
   SrcMgr::CharacteristicKind FileType;
@@ -92,19 +96,22 @@ private:
   bool Initialized;
   bool DisableLineMarkers;
   bool DumpDefines;
+  bool DumpHeaderIncludes;
   bool UseLineDirective;
+  bool HasProcessedPredefines;
 public:
   PrintPPOutputPPCallbacks(Preprocessor &pp, llvm::raw_ostream &os,
-                           bool lineMarkers, bool defines)
+                           bool lineMarkers, bool defines, bool headers)
      : PP(pp), SM(PP.getSourceManager()),
        ConcatInfo(PP), OS(os), DisableLineMarkers(lineMarkers),
-       DumpDefines(defines) {
-    CurLine = 0;
+       DumpDefines(defines), DumpHeaderIncludes(headers) {
+    CurLine = CurrentIncludeDepth = 0;
     CurFilename += "<uninit>";
     EmittedTokensOnThisLine = false;
     EmittedMacroOnThisLine = false;
     FileType = SrcMgr::C_User;
     Initialized = false;
+    HasProcessedPredefines = false;
 
     // If we're in microsoft mode, use normal #line instead of line markers.
     UseLineDirective = PP.getLangOptions().Microsoft;
@@ -137,6 +144,9 @@ public:
   /// MacroDefined - This hook is called whenever a macro definition is seen.
   void MacroDefined(const IdentifierInfo *II, const MacroInfo *MI);
 
+  /// MacroUndefined - This hook is called whenever a macro #undef is seen.
+  void MacroUndefined(SourceLocation Loc, const IdentifierInfo *II,
+                      const MacroInfo *MI);
 };
 }  // end anonymous namespace
 
@@ -216,7 +226,7 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
   
   PresumedLoc UserLoc = SourceMgr.getPresumedLoc(Loc);
   unsigned NewLine = UserLoc.getLine();
-  
+
   if (Reason == PPCallbacks::EnterFile) {
     SourceLocation IncludeLoc = SourceMgr.getPresumedLoc(Loc).getIncludeLoc();
     if (IncludeLoc.isValid())
@@ -228,16 +238,41 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
     // directive and emits a bunch of spaces that aren't needed.  Emulate this
     // strange behavior.
   }
+
+  // Adjust the current include depth.
+  if (Reason == PPCallbacks::EnterFile) {
+    ++CurrentIncludeDepth;
+  } else {
+    if (CurrentIncludeDepth)
+      --CurrentIncludeDepth;
+
+    // We track when we are done with the predefines by watching for the first
+    // place where we drop back to a nesting depth of 0.
+    if (CurrentIncludeDepth == 0 && !HasProcessedPredefines)
+      HasProcessedPredefines = true;
+  }
   
   CurLine = NewLine;
 
-  if (DisableLineMarkers) return;
-
   CurFilename.clear();
   CurFilename += UserLoc.getFilename();
   Lexer::Stringify(CurFilename);
   FileType = NewFileType;
 
+  // Dump the header include information, if enabled and we are past the
+  // predefines buffer.
+  if (DumpHeaderIncludes && HasProcessedPredefines &&
+      Reason == PPCallbacks::EnterFile) {
+    llvm::SmallString<256> Msg;
+    llvm::raw_svector_ostream OS(Msg);
+    for (unsigned i = 0; i != CurrentIncludeDepth; ++i)
+      OS << '.';
+    OS << ' ' << CurFilename << '\n';
+    llvm::errs() << OS.str();
+  }
+
+  if (DisableLineMarkers) return;
+  
   if (!Initialized) {
     WriteLineInfo(CurLine);
     Initialized = true;
@@ -280,6 +315,16 @@ void PrintPPOutputPPCallbacks::MacroDefined(const IdentifierInfo *II,
   EmittedMacroOnThisLine = true;
 }
 
+void PrintPPOutputPPCallbacks::MacroUndefined(SourceLocation Loc,
+                                              const IdentifierInfo *II,
+                                              const MacroInfo *MI) {
+  // Only print out macro definitions in -dD mode.
+  if (!DumpDefines) return;
+
+  MoveToLine(Loc);
+  OS << "#undef " << II->getName();
+  EmittedMacroOnThisLine = true;
+}
 
 void PrintPPOutputPPCallbacks::PragmaComment(SourceLocation Loc,
                                              const IdentifierInfo *Kind,
@@ -516,7 +561,7 @@ void clang::DoPrintPreprocessedInput(Preprocessor &PP, llvm::raw_ostream *OS,
 
   PrintPPOutputPPCallbacks *Callbacks =
       new PrintPPOutputPPCallbacks(PP, *OS, !Opts.ShowLineMarkers,
-                                   Opts.ShowMacros);
+                                   Opts.ShowMacros, Opts.ShowHeaderIncludes);
   PP.AddPragmaHandler(new UnknownPragmaHandler("#pragma", Callbacks));
   PP.AddPragmaHandler("GCC", new UnknownPragmaHandler("#pragma GCC",
                                                       Callbacks));
diff --git a/lib/Frontend/StmtXML.cpp b/lib/Frontend/StmtXML.cpp
index 21dc0ba0a188..b6607349d76d 100644
--- a/lib/Frontend/StmtXML.cpp
+++ b/lib/Frontend/StmtXML.cpp
@@ -32,7 +32,8 @@ namespace  {
 
 
   void addSpecialAttribute(const char* pName, StringLiteral* Str) {
-    Doc.addAttribute(pName, Doc.escapeString(Str->getStrData(), Str->getByteLength()));
+    Doc.addAttribute(pName, Doc.escapeString(Str->getString().data(),
+                                             Str->getString().size()));
   }
 
   void addSpecialAttribute(const char* pName, SizeOfAlignOfExpr* S) {
@@ -261,7 +262,6 @@ const char *StmtXML::getOpcodeStr(UnaryOperator::Opcode Op) {
   case UnaryOperator::Real:    return "__real";
   case UnaryOperator::Imag:    return "__imag";
   case UnaryOperator::Extension: return "__extension__";
-  case UnaryOperator::OffsetOf: return "__builtin_offsetof";
   }
 }
 
diff --git a/lib/Frontend/TextDiagnosticPrinter.cpp b/lib/Frontend/TextDiagnosticPrinter.cpp
index 1b5b7e2ea863..1e453a08fdb9 100644
--- a/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -447,11 +447,11 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
   if (NumHints && DiagOpts->ShowFixits) {
     for (const FixItHint *Hint = Hints, *LastHint = Hints + NumHints;
          Hint != LastHint; ++Hint) {
-      if (Hint->InsertionLoc.isValid()) {
+      if (!Hint->CodeToInsert.empty()) {
         // We have an insertion hint. Determine whether the inserted
         // code is on the same line as the caret.
         std::pair<FileID, unsigned> HintLocInfo
-          = SM.getDecomposedInstantiationLoc(Hint->InsertionLoc);
+          = SM.getDecomposedInstantiationLoc(Hint->RemoveRange.getBegin());
         if (SM.getLineNumber(HintLocInfo.first, HintLocInfo.second) ==
               SM.getLineNumber(FID, FileOffset)) {
           // Insert the new code into the line just below the code
@@ -537,6 +537,48 @@ void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
     if (DiagOpts->ShowColors)
       OS.resetColor();
   }
+
+  if (DiagOpts->ShowParseableFixits) {
+
+    // We follow FixItRewriter's example in not (yet) handling
+    // fix-its in macros.
+    bool BadApples = false;
+    for (const FixItHint *Hint = Hints; Hint != Hints + NumHints; ++Hint) {
+      if (Hint->RemoveRange.isInvalid() ||
+          Hint->RemoveRange.getBegin().isMacroID() ||
+          Hint->RemoveRange.getEnd().isMacroID()) {
+        BadApples = true;
+        break;
+      }
+    }
+
+    if (!BadApples) {
+      for (const FixItHint *Hint = Hints; Hint != Hints + NumHints; ++Hint) {
+
+        SourceLocation B = Hint->RemoveRange.getBegin();
+        SourceLocation E = Hint->RemoveRange.getEnd();
+
+        std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
+        std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
+
+        // Adjust for token ranges.
+        if (Hint->RemoveRange.isTokenRange())
+          EInfo.second += Lexer::MeasureTokenLength(E, SM, *LangOpts);
+
+        // We specifically do not do word-wrapping or tab-expansion here,
+        // because this is supposed to be easy to parse.
+        OS << "fix-it:\"";
+        OS.write_escaped(SM.getPresumedLoc(B).getFilename());
+        OS << "\":{" << SM.getLineNumber(BInfo.first, BInfo.second)
+          << ':' << SM.getColumnNumber(BInfo.first, BInfo.second)
+          << '-' << SM.getLineNumber(EInfo.first, EInfo.second)
+          << ':' << SM.getColumnNumber(EInfo.first, EInfo.second)
+          << "}:\"";
+        OS.write_escaped(Hint->CodeToInsert);
+        OS << "\"\n";
+      }
+    }
+  }
 }
 
 /// \brief Skip over whitespace in the string, starting at the given
diff --git a/lib/Frontend/VerifyDiagnosticsClient.cpp b/lib/Frontend/VerifyDiagnosticsClient.cpp
index ae36481444da..31eb28f912ca 100644
--- a/lib/Frontend/VerifyDiagnosticsClient.cpp
+++ b/lib/Frontend/VerifyDiagnosticsClient.cpp
@@ -171,13 +171,12 @@ public:
     : Begin(Begin), End(End), C(Begin), P(Begin), PEnd(NULL) { }
 
   // Return true if string literal is next.
-  bool Next(const std::string &S) {
-    std::string::size_type LEN = S.length();
+  bool Next(llvm::StringRef S) {
     P = C;
-    PEnd = C + LEN;
+    PEnd = C + S.size();
     if (PEnd > End)
       return false;
-    return !memcmp(P, S.c_str(), LEN);
+    return !memcmp(P, S.data(), S.size());
   }
 
   // Return true if number is next.
@@ -198,9 +197,9 @@ public:
 
   // Return true if string literal is found.
   // When true, P marks begin-position of S in content.
-  bool Search(const std::string &S) {
+  bool Search(llvm::StringRef S) {
     P = std::search(C, End, S.begin(), S.end());
-    PEnd = P + S.length();
+    PEnd = P + S.size();
     return P != End;
   }
 
@@ -484,7 +483,7 @@ void VerifyDiagnosticsClient::CheckDiagnostics() {
   ExpectedData ED;
 
   // Ensure any diagnostics go to the primary client.
-  DiagnosticClient *CurClient = Diags.getClient();
+  DiagnosticClient *CurClient = Diags.takeClient();
   Diags.setClient(PrimaryClient.get());
 
   // If we have a preprocessor, scan the source for expected diagnostic
@@ -507,6 +506,7 @@ void VerifyDiagnosticsClient::CheckDiagnostics() {
                                "note", false));
   }
 
+  Diags.takeClient();
   Diags.setClient(CurClient);
 
   // Reset the buffer, we have processed all the diagnostics in it.
diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt
new file mode 100644
index 000000000000..26c9fc7b5106
--- /dev/null
+++ b/lib/FrontendTool/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LLVM_NO_RTTI 1)
+
+add_clang_library(clangFrontendTool
+  ExecuteCompilerInvocation.cpp
+  )
diff --git a/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
new file mode 100644
index 000000000000..63c628780762
--- /dev/null
+++ b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -0,0 +1,155 @@
+//===--- ExecuteCompilerInvocation.cpp ------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file holds ExecuteCompilerInvocation(). It is split into its own file to
+// minimize the impact of pulling in essentially everything else in Clang.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/FrontendTool/Utils.h"
+#include "clang/Checker/FrontendActions.h"
+#include "clang/CodeGen/CodeGenAction.h"
+#include "clang/Driver/CC1Options.h"
+#include "clang/Driver/OptTable.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/FrontendPluginRegistry.h"
+#include "clang/Rewrite/FrontendActions.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/System/DynamicLibrary.h"
+using namespace clang;
+
+static FrontendAction *CreateFrontendBaseAction(CompilerInstance &CI) {
+  using namespace clang::frontend;
+
+  switch (CI.getFrontendOpts().ProgramAction) {
+  default:
+    llvm_unreachable("Invalid program action!");
+
+  case ASTDump:                return new ASTDumpAction();
+  case ASTPrint:               return new ASTPrintAction();
+  case ASTPrintXML:            return new ASTPrintXMLAction();
+  case ASTView:                return new ASTViewAction();
+  case BoostCon:               return new BoostConAction();
+  case CreateModule:           return 0;
+  case DumpRawTokens:          return new DumpRawTokensAction();
+  case DumpTokens:             return new DumpTokensAction();
+  case EmitAssembly:           return new EmitAssemblyAction();
+  case EmitBC:                 return new EmitBCAction();
+  case EmitHTML:               return new HTMLPrintAction();
+  case EmitLLVM:               return new EmitLLVMAction();
+  case EmitLLVMOnly:           return new EmitLLVMOnlyAction();
+  case EmitCodeGenOnly:        return new EmitCodeGenOnlyAction();
+  case EmitObj:                return new EmitObjAction();
+  case FixIt:                  return new FixItAction();
+  case GeneratePCH:            return new GeneratePCHAction();
+  case GeneratePTH:            return new GeneratePTHAction();
+  case InheritanceView:        return new InheritanceViewAction();
+  case InitOnly:               return new InitOnlyAction();
+  case ParseSyntaxOnly:        return new SyntaxOnlyAction();
+
+  case PluginAction: {
+    for (FrontendPluginRegistry::iterator it =
+           FrontendPluginRegistry::begin(), ie = FrontendPluginRegistry::end();
+         it != ie; ++it) {
+      if (it->getName() == CI.getFrontendOpts().ActionName) {
+        llvm::OwningPtr<PluginASTAction> P(it->instantiate());
+        if (!P->ParseArgs(CI, CI.getFrontendOpts().PluginArgs))
+          return 0;
+        return P.take();
+      }
+    }
+
+    CI.getDiagnostics().Report(diag::err_fe_invalid_plugin_name)
+      << CI.getFrontendOpts().ActionName;
+    return 0;
+  }
+
+  case PrintDeclContext:       return new DeclContextPrintAction();
+  case PrintPreamble:          return new PrintPreambleAction();
+  case PrintPreprocessedInput: return new PrintPreprocessedAction();
+  case RewriteMacros:          return new RewriteMacrosAction();
+  case RewriteObjC:            return new RewriteObjCAction();
+  case RewriteTest:            return new RewriteTestAction();
+  case RunAnalysis:            return new AnalysisAction();
+  case RunPreprocessorOnly:    return new PreprocessOnlyAction();
+  }
+}
+
+static FrontendAction *CreateFrontendAction(CompilerInstance &CI) {
+  // Create the underlying action.
+  FrontendAction *Act = CreateFrontendBaseAction(CI);
+  if (!Act)
+    return 0;
+
+  // If there are any AST files to merge, create a frontend action
+  // adaptor to perform the merge.
+  if (!CI.getFrontendOpts().ASTMergeFiles.empty())
+    Act = new ASTMergeAction(Act, &CI.getFrontendOpts().ASTMergeFiles[0],
+                             CI.getFrontendOpts().ASTMergeFiles.size());
+
+  return Act;
+}
+
+bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
+  // Honor -help.
+  if (Clang->getFrontendOpts().ShowHelp) {
+    llvm::OwningPtr<driver::OptTable> Opts(driver::createCC1OptTable());
+    Opts->PrintHelp(llvm::outs(), "clang -cc1",
+                    "LLVM 'Clang' Compiler: http://clang.llvm.org");
+    return 0;
+  }
+
+  // Honor -version.
+  //
+  // FIXME: Use a better -version message?
+  if (Clang->getFrontendOpts().ShowVersion) {
+    llvm::cl::PrintVersionMessage();
+    return 0;
+  }
+
+  // Honor -mllvm.
+  //
+  // FIXME: Remove this, one day.
+  if (!Clang->getFrontendOpts().LLVMArgs.empty()) {
+    unsigned NumArgs = Clang->getFrontendOpts().LLVMArgs.size();
+    const char **Args = new const char*[NumArgs + 2];
+    Args[0] = "clang (LLVM option parsing)";
+    for (unsigned i = 0; i != NumArgs; ++i)
+      Args[i + 1] = Clang->getFrontendOpts().LLVMArgs[i].c_str();
+    Args[NumArgs + 1] = 0;
+    llvm::cl::ParseCommandLineOptions(NumArgs + 1, const_cast<char **>(Args));
+  }
+
+  // Load any requested plugins.
+  for (unsigned i = 0,
+         e = Clang->getFrontendOpts().Plugins.size(); i != e; ++i) {
+    const std::string &Path = Clang->getFrontendOpts().Plugins[i];
+    std::string Error;
+    if (llvm::sys::DynamicLibrary::LoadLibraryPermanently(Path.c_str(), &Error))
+      Clang->getDiagnostics().Report(diag::err_fe_unable_to_load_plugin)
+        << Path << Error;
+  }
+
+  // If there were errors in processing arguments, don't do anything else.
+  bool Success = false;
+  if (!Clang->getDiagnostics().getNumErrors()) {
+    // Create and execute the frontend action.
+    llvm::OwningPtr<FrontendAction> Act(CreateFrontendAction(*Clang));
+    if (Act) {
+      Success = Clang->ExecuteAction(*Act);
+      if (Clang->getFrontendOpts().DisableFree)
+        Act.take();
+    }
+  }
+
+  return Success;
+}
diff --git a/lib/FrontendTool/Makefile b/lib/FrontendTool/Makefile
new file mode 100644
index 000000000000..c43213ff99d0
--- /dev/null
+++ b/lib/FrontendTool/Makefile
@@ -0,0 +1,13 @@
+##===- clang/lib/FrontendTool/Makefile ---------------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../..
+LIBRARYNAME := clangFrontendTool
+
+include $(CLANG_LEVEL)/Makefile
diff --git a/lib/Headers/CMakeLists.txt b/lib/Headers/CMakeLists.txt
index 97a99d676e87..a1b5f50caa08 100644
--- a/lib/Headers/CMakeLists.txt
+++ b/lib/Headers/CMakeLists.txt
@@ -1,12 +1,15 @@
 set(files
   altivec.h
+  avxintrin.h
   emmintrin.h	
   float.h		
+  immintrin.h
   iso646.h	
   limits.h	
   mm_malloc.h	
   mmintrin.h	
   pmmintrin.h	
+  smmintrin.h
   stdarg.h	
   stdbool.h	
   stddef.h	
diff --git a/lib/Headers/Makefile b/lib/Headers/Makefile
index ebb83843cacd..d75b1a2e7cae 100644
--- a/lib/Headers/Makefile
+++ b/lib/Headers/Makefile
@@ -38,6 +38,7 @@ all-local:: $(OBJHEADERS)
 PROJ_headers := $(DESTDIR)$(PROJ_prefix)/lib/clang/$(CLANG_VERSION)/include
 
 INSTHEADERS := $(addprefix $(PROJ_headers)/, $(HEADERS))
+INSTHEADERS += $(PROJ_headers)/arm_neon.h
 
 $(PROJ_headers):
 	$(Verb) $(MKDIR) $@
diff --git a/lib/Headers/altivec.h b/lib/Headers/altivec.h
index d3d5ad90aef1..89bd259b6733 100644
--- a/lib/Headers/altivec.h
+++ b/lib/Headers/altivec.h
@@ -45,18 +45,30 @@ vec_perm(vector signed char a, vector signed char b, vector unsigned char c);
 static vector unsigned char __ATTRS_o_ai
 vec_perm(vector unsigned char a, vector unsigned char b, vector unsigned char c);
 
+static vector bool char __ATTRS_o_ai
+vec_perm(vector bool char a, vector bool char b, vector unsigned char c);
+
 static vector short __ATTRS_o_ai
 vec_perm(vector short a, vector short b, vector unsigned char c);
 
 static vector unsigned short __ATTRS_o_ai
 vec_perm(vector unsigned short a, vector unsigned short b, vector unsigned char c);
 
+static vector bool short __ATTRS_o_ai
+vec_perm(vector bool short a, vector bool short b, vector unsigned char c);
+
+static vector pixel __ATTRS_o_ai
+vec_perm(vector pixel a, vector pixel b, vector unsigned char c);
+
 static vector int __ATTRS_o_ai
 vec_perm(vector int a, vector int b, vector unsigned char c);
 
 static vector unsigned int __ATTRS_o_ai
 vec_perm(vector unsigned int a, vector unsigned int b, vector unsigned char c);
 
+static vector bool int __ATTRS_o_ai
+vec_perm(vector bool int a, vector bool int b, vector unsigned char c);
+
 static vector float __ATTRS_o_ai
 vec_perm(vector float a, vector float b, vector unsigned char c);
 
@@ -123,36 +135,108 @@ vec_add(vector signed char a, vector signed char b)
   return a + b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_add(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a + b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_add(vector signed char a, vector bool char b)
+{
+  return a + (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_add(vector unsigned char a, vector unsigned char b)
 {
   return a + b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_add(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a + b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_add(vector unsigned char a, vector bool char b)
+{
+  return a + (vector unsigned char)b;
+}
+
 static vector short __ATTRS_o_ai
 vec_add(vector short a, vector short b)
 {
   return a + b;
 }
 
+static vector short __ATTRS_o_ai
+vec_add(vector bool short a, vector short b)
+{
+  return (vector short)a + b;
+}
+
+static vector short __ATTRS_o_ai
+vec_add(vector short a, vector bool short b)
+{
+  return a + (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_add(vector unsigned short a, vector unsigned short b)
 {
   return a + b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_add(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a + b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_add(vector unsigned short a, vector bool short b)
+{
+  return a + (vector unsigned short)b;
+}
+
 static vector int __ATTRS_o_ai
 vec_add(vector int a, vector int b)
 {
   return a + b;
 }
 
+static vector int __ATTRS_o_ai
+vec_add(vector bool int a, vector int b)
+{
+  return (vector int)a + b;
+}
+
+static vector int __ATTRS_o_ai
+vec_add(vector int a, vector bool int b)
+{
+  return a + (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_add(vector unsigned int a, vector unsigned int b)
 {
   return a + b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_add(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a + b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_add(vector unsigned int a, vector bool int b)
+{
+  return a + (vector unsigned int)b;
+}
+
 static vector float __ATTRS_o_ai
 vec_add(vector float a, vector float b)
 {
@@ -169,12 +253,36 @@ vec_vaddubm(vector signed char a, vector signed char b)
   return a + b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vaddubm(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a + b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vaddubm(vector signed char a, vector bool char b)
+{
+  return a + (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vaddubm(vector unsigned char a, vector unsigned char b)
 {
   return a + b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vaddubm(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a + b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vaddubm(vector unsigned char a, vector bool char b)
+{
+  return a + (vector unsigned char)b;
+}
+
 /* vec_vadduhm */
 
 #define __builtin_altivec_vadduhm vec_vadduhm
@@ -185,12 +293,36 @@ vec_vadduhm(vector short a, vector short b)
   return a + b;
 }
 
+static vector short __ATTRS_o_ai
+vec_vadduhm(vector bool short a, vector short b)
+{
+  return (vector short)a + b;
+}
+
+static vector short __ATTRS_o_ai
+vec_vadduhm(vector short a, vector bool short b)
+{
+  return a + (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vadduhm(vector unsigned short a, vector unsigned short b)
 {
   return a + b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vadduhm(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a + b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vadduhm(vector unsigned short a, vector bool short b)
+{
+  return a + (vector unsigned short)b;
+}
+
 /* vec_vadduwm */
 
 #define __builtin_altivec_vadduwm vec_vadduwm
@@ -201,12 +333,36 @@ vec_vadduwm(vector int a, vector int b)
   return a + b;
 }
 
+static vector int __ATTRS_o_ai
+vec_vadduwm(vector bool int a, vector int b)
+{
+  return (vector int)a + b;
+}
+
+static vector int __ATTRS_o_ai
+vec_vadduwm(vector int a, vector bool int b)
+{
+  return a + (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vadduwm(vector unsigned int a, vector unsigned int b)
 {
   return a + b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vadduwm(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a + b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vadduwm(vector unsigned int a, vector bool int b)
+{
+  return a + (vector unsigned int)b;
+}
+
 /* vec_vaddfp */
 
 #define __builtin_altivec_vaddfp  vec_vaddfp
@@ -241,84 +397,228 @@ vec_adds(vector signed char a, vector signed char b)
   return __builtin_altivec_vaddsbs(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_adds(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vaddsbs((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_adds(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vaddsbs(a, (vector signed char)b);
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_adds(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vaddubs(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_adds(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vaddubs((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_adds(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vaddubs(a, (vector unsigned char)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_adds(vector short a, vector short b)
 {
   return __builtin_altivec_vaddshs(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_adds(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vaddshs((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_adds(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vaddshs(a, (vector short)b);
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_adds(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vadduhs(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_adds(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vadduhs((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_adds(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vadduhs(a, (vector unsigned short)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_adds(vector int a, vector int b)
 {
   return __builtin_altivec_vaddsws(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_adds(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vaddsws((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_adds(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vaddsws(a, (vector int)b);
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_adds(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vadduws(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_adds(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vadduws((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_adds(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vadduws(a, (vector unsigned int)b);
+}
+
 /* vec_vaddsbs */
 
-static vector signed char __attribute__((__always_inline__))
+static vector signed char __ATTRS_o_ai
 vec_vaddsbs(vector signed char a, vector signed char b)
 {
   return __builtin_altivec_vaddsbs(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vaddsbs(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vaddsbs((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vaddsbs(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vaddsbs(a, (vector signed char)b);
+}
+
 /* vec_vaddubs */
 
-static vector unsigned char __attribute__((__always_inline__))
+static vector unsigned char __ATTRS_o_ai
 vec_vaddubs(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vaddubs(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vaddubs(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vaddubs((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vaddubs(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vaddubs(a, (vector unsigned char)b);
+}
+
 /* vec_vaddshs */
 
-static vector short __attribute__((__always_inline__))
+static vector short __ATTRS_o_ai
 vec_vaddshs(vector short a, vector short b)
 {
   return __builtin_altivec_vaddshs(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_vaddshs(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vaddshs((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_vaddshs(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vaddshs(a, (vector short)b);
+}
+
 /* vec_vadduhs */
 
-static vector unsigned short __attribute__((__always_inline__))
+static vector unsigned short __ATTRS_o_ai
 vec_vadduhs(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vadduhs(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vadduhs(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vadduhs((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vadduhs(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vadduhs(a, (vector unsigned short)b);
+}
+
 /* vec_vaddsws */
 
-static vector int __attribute__((__always_inline__))
+static vector int __ATTRS_o_ai
 vec_vaddsws(vector int a, vector int b)
 {
   return __builtin_altivec_vaddsws(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_vaddsws(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vaddsws((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_vaddsws(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vaddsws(a, (vector int)b);
+}
+
 /* vec_vadduws */
 
-static vector unsigned int __attribute__((__always_inline__))
+static vector unsigned int __ATTRS_o_ai
 vec_vadduws(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vadduws(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vadduws(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vadduws((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vadduws(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vadduws(a, (vector unsigned int)b);
+}
+
 /* vec_and */
 
 #define __builtin_altivec_vand vec_and
@@ -329,36 +629,126 @@ vec_and(vector signed char a, vector signed char b)
   return a & b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_and(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a & b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_and(vector signed char a, vector bool char b)
+{
+  return a & (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_and(vector unsigned char a, vector unsigned char b)
 {
   return a & b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_and(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a & b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_and(vector unsigned char a, vector bool char b)
+{
+  return a & (vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_and(vector bool char a, vector bool char b)
+{
+  return a & b;
+}
+
 static vector short __ATTRS_o_ai
 vec_and(vector short a, vector short b)
 {
   return a & b;
 }
 
+static vector short __ATTRS_o_ai
+vec_and(vector bool short a, vector short b)
+{
+  return (vector short)a & b;
+}
+
+static vector short __ATTRS_o_ai
+vec_and(vector short a, vector bool short b)
+{
+  return a & (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_and(vector unsigned short a, vector unsigned short b)
 {
   return a & b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_and(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a & b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_and(vector unsigned short a, vector bool short b)
+{
+  return a & (vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_and(vector bool short a, vector bool short b)
+{
+  return a & b;
+}
+
 static vector int __ATTRS_o_ai
 vec_and(vector int a, vector int b)
 {
   return a & b;
 }
 
+static vector int __ATTRS_o_ai
+vec_and(vector bool int a, vector int b)
+{
+  return (vector int)a & b;
+}
+
+static vector int __ATTRS_o_ai
+vec_and(vector int a, vector bool int b)
+{
+  return a & (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_and(vector unsigned int a, vector unsigned int b)
 {
   return a & b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_and(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a & b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_and(vector unsigned int a, vector bool int b)
+{
+  return a & (vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_and(vector bool int a, vector bool int b)
+{
+  return a & b;
+}
+
 static vector float __ATTRS_o_ai
 vec_and(vector float a, vector float b)
 {
@@ -366,6 +756,20 @@ vec_and(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_and(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a & (vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_and(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a & (vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_vand */
 
 static vector signed char __ATTRS_o_ai
@@ -374,36 +778,126 @@ vec_vand(vector signed char a, vector signed char b)
   return a & b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vand(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a & b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vand(vector signed char a, vector bool char b)
+{
+  return a & (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vand(vector unsigned char a, vector unsigned char b)
 {
   return a & b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vand(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a & b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vand(vector unsigned char a, vector bool char b)
+{
+  return a & (vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vand(vector bool char a, vector bool char b)
+{
+  return a & b;
+}
+
 static vector short __ATTRS_o_ai
 vec_vand(vector short a, vector short b)
 {
   return a & b;
 }
 
+static vector short __ATTRS_o_ai
+vec_vand(vector bool short a, vector short b)
+{
+  return (vector short)a & b;
+}
+
+static vector short __ATTRS_o_ai
+vec_vand(vector short a, vector bool short b)
+{
+  return a & (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vand(vector unsigned short a, vector unsigned short b)
 {
   return a & b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vand(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a & b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vand(vector unsigned short a, vector bool short b)
+{
+  return a & (vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vand(vector bool short a, vector bool short b)
+{
+  return a & b;
+}
+
 static vector int __ATTRS_o_ai
 vec_vand(vector int a, vector int b)
 {
   return a & b;
 }
 
+static vector int __ATTRS_o_ai
+vec_vand(vector bool int a, vector int b)
+{
+  return (vector int)a & b;
+}
+
+static vector int __ATTRS_o_ai
+vec_vand(vector int a, vector bool int b)
+{
+  return a & (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vand(vector unsigned int a, vector unsigned int b)
 {
   return a & b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vand(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a & b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vand(vector unsigned int a, vector bool int b)
+{
+  return a & (vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vand(vector bool int a, vector bool int b)
+{
+  return a & b;
+}
+
 static vector float __ATTRS_o_ai
 vec_vand(vector float a, vector float b)
 {
@@ -411,6 +905,20 @@ vec_vand(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_vand(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a & (vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_vand(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a & (vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_andc */
 
 #define __builtin_altivec_vandc vec_andc
@@ -421,36 +929,126 @@ vec_andc(vector signed char a, vector signed char b)
   return a & ~b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_andc(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a & ~b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_andc(vector signed char a, vector bool char b)
+{
+  return a & ~(vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_andc(vector unsigned char a, vector unsigned char b)
 {
   return a & ~b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_andc(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a & ~b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_andc(vector unsigned char a, vector bool char b)
+{
+  return a & ~(vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_andc(vector bool char a, vector bool char b)
+{
+  return a & ~b;
+}
+
 static vector short __ATTRS_o_ai
 vec_andc(vector short a, vector short b)
 {
   return a & ~b;
 }
 
+static vector short __ATTRS_o_ai
+vec_andc(vector bool short a, vector short b)
+{
+  return (vector short)a & ~b;
+}
+
+static vector short __ATTRS_o_ai
+vec_andc(vector short a, vector bool short b)
+{
+  return a & ~(vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_andc(vector unsigned short a, vector unsigned short b)
 {
   return a & ~b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_andc(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a & ~b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_andc(vector unsigned short a, vector bool short b)
+{
+  return a & ~(vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_andc(vector bool short a, vector bool short b)
+{
+  return a & ~b;
+}
+
 static vector int __ATTRS_o_ai
 vec_andc(vector int a, vector int b)
 {
   return a & ~b;
 }
 
+static vector int __ATTRS_o_ai
+vec_andc(vector bool int a, vector int b)
+{
+  return (vector int)a & ~b;
+}
+
+static vector int __ATTRS_o_ai
+vec_andc(vector int a, vector bool int b)
+{
+  return a & ~(vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_andc(vector unsigned int a, vector unsigned int b)
 {
   return a & ~b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_andc(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a & ~b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_andc(vector unsigned int a, vector bool int b)
+{
+  return a & ~(vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_andc(vector bool int a, vector bool int b)
+{
+  return a & ~b;
+}
+
 static vector float __ATTRS_o_ai
 vec_andc(vector float a, vector float b)
 {
@@ -458,6 +1056,20 @@ vec_andc(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_andc(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a & ~(vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_andc(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a & ~(vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_vandc */
 
 static vector signed char __ATTRS_o_ai
@@ -466,36 +1078,126 @@ vec_vandc(vector signed char a, vector signed char b)
   return a & ~b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vandc(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a & ~b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vandc(vector signed char a, vector bool char b)
+{
+  return a & ~(vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vandc(vector unsigned char a, vector unsigned char b)
 {
   return a & ~b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vandc(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a & ~b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vandc(vector unsigned char a, vector bool char b)
+{
+  return a & ~(vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vandc(vector bool char a, vector bool char b)
+{
+  return a & ~b;
+}
+
 static vector short __ATTRS_o_ai
 vec_vandc(vector short a, vector short b)
 {
   return a & ~b;
 }
 
+static vector short __ATTRS_o_ai
+vec_vandc(vector bool short a, vector short b)
+{
+  return (vector short)a & ~b;
+}
+
+static vector short __ATTRS_o_ai
+vec_vandc(vector short a, vector bool short b)
+{
+  return a & ~(vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vandc(vector unsigned short a, vector unsigned short b)
 {
   return a & ~b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vandc(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a & ~b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vandc(vector unsigned short a, vector bool short b)
+{
+  return a & ~(vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vandc(vector bool short a, vector bool short b)
+{
+  return a & ~b;
+}
+
 static vector int __ATTRS_o_ai
 vec_vandc(vector int a, vector int b)
 {
   return a & ~b;
 }
 
+static vector int __ATTRS_o_ai
+vec_vandc(vector bool int a, vector int b)
+{
+  return (vector int)a & ~b;
+}
+
+static vector int __ATTRS_o_ai
+vec_vandc(vector int a, vector bool int b)
+{
+  return a & ~(vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vandc(vector unsigned int a, vector unsigned int b)
 {
   return a & ~b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vandc(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a & ~b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vandc(vector unsigned int a, vector bool int b)
+{
+  return a & ~(vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vandc(vector bool int a, vector bool int b)
+{
+  return a & ~b;
+}
+
 static vector float __ATTRS_o_ai
 vec_vandc(vector float a, vector float b)
 {
@@ -503,6 +1205,20 @@ vec_vandc(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_vandc(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a & ~(vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_vandc(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a & ~(vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_avg */
 
 static vector signed char __ATTRS_o_ai
@@ -623,214 +1339,218 @@ vec_vcmpbfp(vector float a, vector float b)
 
 /* vec_cmpeq */
 
-static vector /*bool*/ char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_cmpeq(vector signed char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpequb((vector char)a, (vector char)b);
+  return (vector bool char)
+    __builtin_altivec_vcmpequb((vector char)a, (vector char)b);
 }
 
-static vector /*bool*/ char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_cmpeq(vector unsigned char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpequb((vector char)a, (vector char)b);
+  return (vector bool char)
+    __builtin_altivec_vcmpequb((vector char)a, (vector char)b);
 }
 
-static vector /*bool*/ short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_cmpeq(vector short a, vector short b)
 {
-  return __builtin_altivec_vcmpequh(a, b);
+  return (vector bool short)__builtin_altivec_vcmpequh(a, b);
 }
 
-static vector /*bool*/ short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_cmpeq(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpequh((vector short)a, (vector short)b);
+  return (vector bool short)
+    __builtin_altivec_vcmpequh((vector short)a, (vector short)b);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmpeq(vector int a, vector int b)
 {
-  return __builtin_altivec_vcmpequw(a, b);
+  return (vector bool int)__builtin_altivec_vcmpequw(a, b);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmpeq(vector unsigned int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpequw((vector int)a, (vector int)b);
+  return (vector bool int)
+    __builtin_altivec_vcmpequw((vector int)a, (vector int)b);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmpeq(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpeqfp(a, b);
+  return (vector bool int)__builtin_altivec_vcmpeqfp(a, b);
 }
 
 /* vec_cmpge */
 
-static vector /*bool*/ int __attribute__((__always_inline__))
+static vector bool int __attribute__((__always_inline__))
 vec_cmpge(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpgefp(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgefp(a, b);
 }
 
 /* vec_vcmpgefp */
 
-static vector /*bool*/ int __attribute__((__always_inline__))
+static vector bool int __attribute__((__always_inline__))
 vec_vcmpgefp(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpgefp(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgefp(a, b);
 }
 
 /* vec_cmpgt */
 
-static vector /*bool*/ char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_cmpgt(vector signed char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtsb(a, b);
+  return (vector bool char)__builtin_altivec_vcmpgtsb(a, b);
 }
 
-static vector /*bool*/ char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_cmpgt(vector unsigned char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpgtub(a, b);
+  return (vector bool char)__builtin_altivec_vcmpgtub(a, b);
 }
 
-static vector /*bool*/ short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_cmpgt(vector short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtsh(a, b);
+  return (vector bool short)__builtin_altivec_vcmpgtsh(a, b);
 }
 
-static vector /*bool*/ short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_cmpgt(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpgtuh(a, b);
+  return (vector bool short)__builtin_altivec_vcmpgtuh(a, b);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmpgt(vector int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtsw(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgtsw(a, b);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmpgt(vector unsigned int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpgtuw(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgtuw(a, b);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmpgt(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpgtfp(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgtfp(a, b);
 }
 
 /* vec_vcmpgtsb */
 
-static vector /*bool*/ char __attribute__((__always_inline__))
+static vector bool char __attribute__((__always_inline__))
 vec_vcmpgtsb(vector signed char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtsb(a, b);
+  return (vector bool char)__builtin_altivec_vcmpgtsb(a, b);
 }
 
 /* vec_vcmpgtub */
 
-static vector /*bool*/ char __attribute__((__always_inline__))
+static vector bool char __attribute__((__always_inline__))
 vec_vcmpgtub(vector unsigned char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpgtub(a, b);
+  return (vector bool char)__builtin_altivec_vcmpgtub(a, b);
 }
 
 /* vec_vcmpgtsh */
 
-static vector /*bool*/ short __attribute__((__always_inline__))
+static vector bool short __attribute__((__always_inline__))
 vec_vcmpgtsh(vector short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtsh(a, b);
+  return (vector bool short)__builtin_altivec_vcmpgtsh(a, b);
 }
 
 /* vec_vcmpgtuh */
 
-static vector /*bool*/ short __attribute__((__always_inline__))
+static vector bool short __attribute__((__always_inline__))
 vec_vcmpgtuh(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpgtuh(a, b);
+  return (vector bool short)__builtin_altivec_vcmpgtuh(a, b);
 }
 
 /* vec_vcmpgtsw */
 
-static vector /*bool*/ int __attribute__((__always_inline__))
+static vector bool int __attribute__((__always_inline__))
 vec_vcmpgtsw(vector int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtsw(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgtsw(a, b);
 }
 
 /* vec_vcmpgtuw */
 
-static vector /*bool*/ int __attribute__((__always_inline__))
+static vector bool int __attribute__((__always_inline__))
 vec_vcmpgtuw(vector unsigned int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpgtuw(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgtuw(a, b);
 }
 
 /* vec_vcmpgtfp */
 
-static vector /*bool*/ int __attribute__((__always_inline__))
+static vector bool int __attribute__((__always_inline__))
 vec_vcmpgtfp(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpgtfp(a, b);
+  return (vector bool int)__builtin_altivec_vcmpgtfp(a, b);
 }
 
 /* vec_cmple */
 
-static vector /*bool*/ int __attribute__((__always_inline__))
+static vector bool int __attribute__((__always_inline__))
 vec_cmple(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpgefp(b, a);
+  return (vector bool int)__builtin_altivec_vcmpgefp(b, a);
 }
 
 /* vec_cmplt */
 
-static vector /*bool*/ char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_cmplt(vector signed char a, vector signed char b)
 {
-  return __builtin_altivec_vcmpgtsb(b, a);
+  return (vector bool char)__builtin_altivec_vcmpgtsb(b, a);
 }
 
-static vector /*bool*/ char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_cmplt(vector unsigned char a, vector unsigned char b)
 {
-  return __builtin_altivec_vcmpgtub(b, a);
+  return (vector bool char)__builtin_altivec_vcmpgtub(b, a);
 }
 
-static vector /*bool*/ short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_cmplt(vector short a, vector short b)
 {
-  return __builtin_altivec_vcmpgtsh(b, a);
+  return (vector bool short)__builtin_altivec_vcmpgtsh(b, a);
 }
 
-static vector /*bool*/ short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_cmplt(vector unsigned short a, vector unsigned short b)
 {
-  return __builtin_altivec_vcmpgtuh(b, a);
+  return (vector bool short)__builtin_altivec_vcmpgtuh(b, a);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmplt(vector int a, vector int b)
 {
-  return __builtin_altivec_vcmpgtsw(b, a);
+  return (vector bool int)__builtin_altivec_vcmpgtsw(b, a);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmplt(vector unsigned int a, vector unsigned int b)
 {
-  return __builtin_altivec_vcmpgtuw(b, a);
+  return (vector bool int)__builtin_altivec_vcmpgtuw(b, a);
 }
 
-static vector /*bool*/ int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_cmplt(vector float a, vector float b)
 {
-  return __builtin_altivec_vcmpgtfp(b, a);
+  return (vector bool int)__builtin_altivec_vcmpgtfp(b, a);
 }
 
 /* vec_ctf */
@@ -1001,6 +1721,12 @@ vec_ld(int a, unsigned char *b)
   return (vector unsigned char)__builtin_altivec_lvx(a, b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_ld(int a, vector bool char *b)
+{
+  return (vector bool char)__builtin_altivec_lvx(a, b);
+}
+
 static vector short __ATTRS_o_ai
 vec_ld(int a, vector short *b)
 {
@@ -1025,6 +1751,18 @@ vec_ld(int a, unsigned short *b)
   return (vector unsigned short)__builtin_altivec_lvx(a, b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_ld(int a, vector bool short *b)
+{
+  return (vector bool short)__builtin_altivec_lvx(a, b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_ld(int a, vector pixel *b)
+{
+  return (vector pixel)__builtin_altivec_lvx(a, b);
+}
+
 static vector int __ATTRS_o_ai
 vec_ld(int a, vector int *b)
 {
@@ -1049,6 +1787,12 @@ vec_ld(int a, unsigned int *b)
   return (vector unsigned int)__builtin_altivec_lvx(a, b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_ld(int a, vector bool int *b)
+{
+  return (vector bool int)__builtin_altivec_lvx(a, b);
+}
+
 static vector float __ATTRS_o_ai
 vec_ld(int a, vector float *b)
 {
@@ -1087,6 +1831,12 @@ vec_lvx(int a, unsigned char *b)
   return (vector unsigned char)__builtin_altivec_lvx(a, b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_lvx(int a, vector bool char *b)
+{
+  return (vector bool char)__builtin_altivec_lvx(a, b);
+}
+
 static vector short __ATTRS_o_ai
 vec_lvx(int a, vector short *b)
 {
@@ -1111,6 +1861,18 @@ vec_lvx(int a, unsigned short *b)
   return (vector unsigned short)__builtin_altivec_lvx(a, b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_lvx(int a, vector bool short *b)
+{
+  return (vector bool short)__builtin_altivec_lvx(a, b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_lvx(int a, vector pixel *b)
+{
+  return (vector pixel)__builtin_altivec_lvx(a, b);
+}
+
 static vector int __ATTRS_o_ai
 vec_lvx(int a, vector int *b)
 {
@@ -1135,6 +1897,12 @@ vec_lvx(int a, unsigned int *b)
   return (vector unsigned int)__builtin_altivec_lvx(a, b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_lvx(int a, vector bool int *b)
+{
+  return (vector bool int)__builtin_altivec_lvx(a, b);
+}
+
 static vector float __ATTRS_o_ai
 vec_lvx(int a, vector float *b)
 {
@@ -1265,6 +2033,12 @@ vec_ldl(int a, unsigned char *b)
   return (vector unsigned char)__builtin_altivec_lvxl(a, b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_ldl(int a, vector bool char *b)
+{
+  return (vector bool char)__builtin_altivec_lvxl(a, b);
+}
+
 static vector short __ATTRS_o_ai
 vec_ldl(int a, vector short *b)
 {
@@ -1289,6 +2063,18 @@ vec_ldl(int a, unsigned short *b)
   return (vector unsigned short)__builtin_altivec_lvxl(a, b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_ldl(int a, vector bool short *b)
+{
+  return (vector bool short)__builtin_altivec_lvxl(a, b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_ldl(int a, vector pixel *b)
+{
+  return (vector pixel short)__builtin_altivec_lvxl(a, b);
+}
+
 static vector int __ATTRS_o_ai
 vec_ldl(int a, vector int *b)
 {
@@ -1313,6 +2099,12 @@ vec_ldl(int a, unsigned int *b)
   return (vector unsigned int)__builtin_altivec_lvxl(a, b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_ldl(int a, vector bool int *b)
+{
+  return (vector bool int)__builtin_altivec_lvxl(a, b);
+}
+
 static vector float __ATTRS_o_ai
 vec_ldl(int a, vector float *b)
 {
@@ -1351,6 +2143,12 @@ vec_lvxl(int a, unsigned char *b)
   return (vector unsigned char)__builtin_altivec_lvxl(a, b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_lvxl(int a, vector bool char *b)
+{
+  return (vector bool char)__builtin_altivec_lvxl(a, b);
+}
+
 static vector short __ATTRS_o_ai
 vec_lvxl(int a, vector short *b)
 {
@@ -1375,6 +2173,18 @@ vec_lvxl(int a, unsigned short *b)
   return (vector unsigned short)__builtin_altivec_lvxl(a, b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_lvxl(int a, vector bool short *b)
+{
+  return (vector bool short)__builtin_altivec_lvxl(a, b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_lvxl(int a, vector pixel *b)
+{
+  return (vector pixel)__builtin_altivec_lvxl(a, b);
+}
+
 static vector int __ATTRS_o_ai
 vec_lvxl(int a, vector int *b)
 {
@@ -1399,6 +2209,12 @@ vec_lvxl(int a, unsigned int *b)
   return (vector unsigned int)__builtin_altivec_lvxl(a, b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_lvxl(int a, vector bool int *b)
+{
+  return (vector bool int)__builtin_altivec_lvxl(a, b);
+}
+
 static vector float __ATTRS_o_ai
 vec_lvxl(int a, vector float *b)
 {
@@ -1549,41 +2365,113 @@ vec_vmhaddshs(vector signed short a, vector signed short b, vector signed short
 /* vec_max */
 
 static vector signed char __ATTRS_o_ai
-vec_max(vector signed  char a, vector signed char b)
+vec_max(vector signed char a, vector signed char b)
 {
   return __builtin_altivec_vmaxsb(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_max(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vmaxsb((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_max(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vmaxsb(a, (vector signed char)b);
+}
+
 static vector unsigned char __ATTRS_o_ai
-vec_max(vector unsigned  char a, vector unsigned char b)
+vec_max(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vmaxub(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_max(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vmaxub((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_max(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vmaxub(a, (vector unsigned char)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_max(vector short a, vector short b)
 {
   return __builtin_altivec_vmaxsh(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_max(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vmaxsh((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_max(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vmaxsh(a, (vector short)b);
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_max(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vmaxuh(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_max(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vmaxuh((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_max(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vmaxuh(a, (vector unsigned short)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_max(vector int a, vector int b)
 {
   return __builtin_altivec_vmaxsw(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_max(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vmaxsw((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_max(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vmaxsw(a, (vector int)b);
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_max(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vmaxuw(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_max(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vmaxuw((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_max(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vmaxuw(a, (vector unsigned int)b);
+}
+
 static vector float __ATTRS_o_ai
 vec_max(vector float a, vector float b)
 {
@@ -1592,52 +2480,124 @@ vec_max(vector float a, vector float b)
 
 /* vec_vmaxsb */
 
-static vector signed char __attribute__((__always_inline__))
-vec_vmaxsb(vector signed  char a, vector signed char b)
+static vector signed char __ATTRS_o_ai
+vec_vmaxsb(vector signed char a, vector signed char b)
 {
   return __builtin_altivec_vmaxsb(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vmaxsb(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vmaxsb((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vmaxsb(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vmaxsb(a, (vector signed char)b);
+}
+
 /* vec_vmaxub */
 
-static vector unsigned char __attribute__((__always_inline__))
-vec_vmaxub(vector unsigned  char a, vector unsigned char b)
+static vector unsigned char __ATTRS_o_ai
+vec_vmaxub(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vmaxub(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vmaxub(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vmaxub((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vmaxub(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vmaxub(a, (vector unsigned char)b);
+}
+
 /* vec_vmaxsh */
 
-static vector short __attribute__((__always_inline__))
+static vector short __ATTRS_o_ai
 vec_vmaxsh(vector short a, vector short b)
 {
   return __builtin_altivec_vmaxsh(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_vmaxsh(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vmaxsh((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_vmaxsh(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vmaxsh(a, (vector short)b);
+}
+
 /* vec_vmaxuh */
 
-static vector unsigned short __attribute__((__always_inline__))
+static vector unsigned short __ATTRS_o_ai
 vec_vmaxuh(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vmaxuh(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vmaxuh(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vmaxuh((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vmaxuh(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vmaxuh(a, (vector unsigned short)b);
+}
+
 /* vec_vmaxsw */
 
-static vector int __attribute__((__always_inline__))
+static vector int __ATTRS_o_ai
 vec_vmaxsw(vector int a, vector int b)
 {
   return __builtin_altivec_vmaxsw(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_vmaxsw(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vmaxsw((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_vmaxsw(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vmaxsw(a, (vector int)b);
+}
+
 /* vec_vmaxuw */
 
-static vector unsigned int __attribute__((__always_inline__))
+static vector unsigned int __ATTRS_o_ai
 vec_vmaxuw(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vmaxuw(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vmaxuw(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vmaxuw((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vmaxuw(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vmaxuw(a, (vector unsigned int)b);
+}
+
 /* vec_vmaxfp */
 
 static vector float __attribute__((__always_inline__))
@@ -1664,6 +2624,14 @@ vec_mergeh(vector unsigned char a, vector unsigned char b)
      0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17));
 }
 
+static vector bool char __ATTRS_o_ai
+vec_mergeh(vector bool char a, vector bool char b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13, 
+     0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17));
+}
+
 static vector short __ATTRS_o_ai
 vec_mergeh(vector short a, vector short b)
 {
@@ -1680,6 +2648,22 @@ vec_mergeh(vector unsigned short a, vector unsigned short b)
      0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17));
 }
 
+static vector bool short __ATTRS_o_ai
+vec_mergeh(vector bool short a, vector bool short b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
+     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_mergeh(vector pixel a, vector pixel b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
+     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17));
+}
+
 static vector int __ATTRS_o_ai
 vec_mergeh(vector int a, vector int b)
 {
@@ -1696,6 +2680,14 @@ vec_mergeh(vector unsigned int a, vector unsigned int b)
      0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17));
 }
 
+static vector bool int __ATTRS_o_ai
+vec_mergeh(vector bool int a, vector bool int b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
+     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17));
+}
+
 static vector float __ATTRS_o_ai
 vec_mergeh(vector float a, vector float b)
 {
@@ -1724,6 +2716,14 @@ vec_vmrghb(vector unsigned char a, vector unsigned char b)
      0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17));
 }
 
+static vector bool char __ATTRS_o_ai
+vec_vmrghb(vector bool char a, vector bool char b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13, 
+     0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17));
+}
+
 /* vec_vmrghh */
 
 #define __builtin_altivec_vmrghh vec_vmrghh
@@ -1744,6 +2744,22 @@ vec_vmrghh(vector unsigned short a, vector unsigned short b)
      0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17));
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vmrghh(vector bool short a, vector bool short b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
+     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vmrghh(vector pixel a, vector pixel b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
+     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17));
+}
+
 /* vec_vmrghw */
 
 #define __builtin_altivec_vmrghw vec_vmrghw
@@ -1764,6 +2780,14 @@ vec_vmrghw(vector unsigned int a, vector unsigned int b)
      0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17));
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vmrghw(vector bool int a, vector bool int b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
+     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17));
+}
+
 static vector float __ATTRS_o_ai
 vec_vmrghw(vector float a, vector float b)
 {
@@ -1790,6 +2814,14 @@ vec_mergel(vector unsigned char a, vector unsigned char b)
      0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F));
 }
 
+static vector bool char __ATTRS_o_ai
+vec_mergel(vector bool char a, vector bool char b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B, 
+     0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F));
+}
+
 static vector short __ATTRS_o_ai
 vec_mergel(vector short a, vector short b)
 {
@@ -1806,6 +2838,22 @@ vec_mergel(vector unsigned short a, vector unsigned short b)
      0x0C, 0x0D, 0x1C, 0x1D, 0x0E, 0x0F, 0x1E, 0x1F));
 }
 
+static vector bool short __ATTRS_o_ai
+vec_mergel(vector bool short a, vector bool short b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x09, 0x18, 0x19, 0x0A, 0x0B, 0x1A, 0x1B,
+     0x0C, 0x0D, 0x1C, 0x1D, 0x0E, 0x0F, 0x1E, 0x1F));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_mergel(vector pixel a, vector pixel b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x09, 0x18, 0x19, 0x0A, 0x0B, 0x1A, 0x1B,
+     0x0C, 0x0D, 0x1C, 0x1D, 0x0E, 0x0F, 0x1E, 0x1F));
+}
+
 static vector int __ATTRS_o_ai
 vec_mergel(vector int a, vector int b)
 {
@@ -1822,6 +2870,14 @@ vec_mergel(vector unsigned int a, vector unsigned int b)
      0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F));
 }
 
+static vector bool int __ATTRS_o_ai
+vec_mergel(vector bool int a, vector bool int b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B,
+     0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F));
+}
+
 static vector float __ATTRS_o_ai
 vec_mergel(vector float a, vector float b)
 {
@@ -1850,6 +2906,14 @@ vec_vmrglb(vector unsigned char a, vector unsigned char b)
      0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F));
 }
 
+static vector bool char __ATTRS_o_ai
+vec_vmrglb(vector bool char a, vector bool char b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B, 
+     0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F));
+}
+
 /* vec_vmrglh */
 
 #define __builtin_altivec_vmrglh vec_vmrglh
@@ -1870,6 +2934,22 @@ vec_vmrglh(vector unsigned short a, vector unsigned short b)
      0x0C, 0x0D, 0x1C, 0x1D, 0x0E, 0x0F, 0x1E, 0x1F));
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vmrglh(vector bool short a, vector bool short b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x09, 0x18, 0x19, 0x0A, 0x0B, 0x1A, 0x1B,
+     0x0C, 0x0D, 0x1C, 0x1D, 0x0E, 0x0F, 0x1E, 0x1F));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vmrglh(vector pixel a, vector pixel b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x09, 0x18, 0x19, 0x0A, 0x0B, 0x1A, 0x1B,
+     0x0C, 0x0D, 0x1C, 0x1D, 0x0E, 0x0F, 0x1E, 0x1F));
+}
+
 /* vec_vmrglw */
 
 #define __builtin_altivec_vmrglw vec_vmrglw
@@ -1890,6 +2970,14 @@ vec_vmrglw(vector unsigned int a, vector unsigned int b)
      0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F));
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vmrglw(vector bool int a, vector bool int b)
+{
+  return vec_perm(a, b, (vector unsigned char)
+    (0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B,
+     0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F));
+}
+
 static vector float __ATTRS_o_ai
 vec_vmrglw(vector float a, vector float b)
 {
@@ -1909,41 +2997,113 @@ vec_mfvscr(void)
 /* vec_min */
 
 static vector signed char __ATTRS_o_ai
-vec_min(vector signed  char a, vector signed char b)
+vec_min(vector signed char a, vector signed char b)
 {
   return __builtin_altivec_vminsb(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_min(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vminsb((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_min(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vminsb(a, (vector signed char)b);
+}
+
 static vector unsigned char __ATTRS_o_ai
-vec_min(vector unsigned  char a, vector unsigned char b)
+vec_min(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vminub(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_min(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vminub((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_min(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vminub(a, (vector unsigned char)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_min(vector short a, vector short b)
 {
   return __builtin_altivec_vminsh(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_min(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vminsh((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_min(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vminsh(a, (vector short)b);
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_min(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vminuh(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_min(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vminuh((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_min(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vminuh(a, (vector unsigned short)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_min(vector int a, vector int b)
 {
   return __builtin_altivec_vminsw(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_min(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vminsw((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_min(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vminsw(a, (vector int)b);
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_min(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vminuw(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_min(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vminuw((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_min(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vminuw(a, (vector unsigned int)b);
+}
+
 static vector float __ATTRS_o_ai
 vec_min(vector float a, vector float b)
 {
@@ -1952,52 +3112,124 @@ vec_min(vector float a, vector float b)
 
 /* vec_vminsb */
 
-static vector signed char __attribute__((__always_inline__))
-vec_vminsb(vector signed  char a, vector signed char b)
+static vector signed char __ATTRS_o_ai
+vec_vminsb(vector signed char a, vector signed char b)
 {
   return __builtin_altivec_vminsb(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vminsb(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vminsb((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vminsb(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vminsb(a, (vector signed char)b);
+}
+
 /* vec_vminub */
 
-static vector unsigned char __attribute__((__always_inline__))
-vec_vminub(vector unsigned  char a, vector unsigned char b)
+static vector unsigned char __ATTRS_o_ai
+vec_vminub(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vminub(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vminub(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vminub((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vminub(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vminub(a, (vector unsigned char)b);
+}
+
 /* vec_vminsh */
 
-static vector short __attribute__((__always_inline__))
+static vector short __ATTRS_o_ai
 vec_vminsh(vector short a, vector short b)
 {
   return __builtin_altivec_vminsh(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_vminsh(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vminsh((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_vminsh(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vminsh(a, (vector short)b);
+}
+
 /* vec_vminuh */
 
-static vector unsigned short __attribute__((__always_inline__))
+static vector unsigned short __ATTRS_o_ai
 vec_vminuh(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vminuh(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vminuh(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vminuh((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vminuh(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vminuh(a, (vector unsigned short)b);
+}
+
 /* vec_vminsw */
 
-static vector int __attribute__((__always_inline__))
+static vector int __ATTRS_o_ai
 vec_vminsw(vector int a, vector int b)
 {
   return __builtin_altivec_vminsw(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_vminsw(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vminsw((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_vminsw(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vminsw(a, (vector int)b);
+}
+
 /* vec_vminuw */
 
-static vector unsigned int __attribute__((__always_inline__))
+static vector unsigned int __ATTRS_o_ai
 vec_vminuw(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vminuw(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vminuw(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vminuw((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vminuw(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vminuw(a, (vector unsigned int)b);
+}
+
 /* vec_vminfp */
 
 static vector float __attribute__((__always_inline__))
@@ -2179,6 +3411,12 @@ vec_mtvscr(vector unsigned char a)
 }
 
 static void __ATTRS_o_ai
+vec_mtvscr(vector bool char a)
+{
+  __builtin_altivec_mtvscr((vector int)a);
+}
+
+static void __ATTRS_o_ai
 vec_mtvscr(vector short a)
 {
   __builtin_altivec_mtvscr((vector int)a);
@@ -2191,6 +3429,18 @@ vec_mtvscr(vector unsigned short a)
 }
 
 static void __ATTRS_o_ai
+vec_mtvscr(vector bool short a)
+{
+  __builtin_altivec_mtvscr((vector int)a);
+}
+
+static void __ATTRS_o_ai
+vec_mtvscr(vector pixel a)
+{
+  __builtin_altivec_mtvscr((vector int)a);
+}
+
+static void __ATTRS_o_ai
 vec_mtvscr(vector int a)
 {
   __builtin_altivec_mtvscr((vector int)a);
@@ -2203,6 +3453,12 @@ vec_mtvscr(vector unsigned int a)
 }
 
 static void __ATTRS_o_ai
+vec_mtvscr(vector bool int a)
+{
+  __builtin_altivec_mtvscr((vector int)a);
+}
+
+static void __ATTRS_o_ai
 vec_mtvscr(vector float a)
 {
   __builtin_altivec_mtvscr((vector int)a);
@@ -2356,6 +3612,12 @@ vec_nor(vector unsigned char a, vector unsigned char b)
   return ~(a | b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_nor(vector bool char a, vector bool char b)
+{
+  return ~(a | b);
+}
+
 static vector short __ATTRS_o_ai
 vec_nor(vector short a, vector short b)
 {
@@ -2368,6 +3630,12 @@ vec_nor(vector unsigned short a, vector unsigned short b)
   return ~(a | b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_nor(vector bool short a, vector bool short b)
+{
+  return ~(a | b);
+}
+
 static vector int __ATTRS_o_ai
 vec_nor(vector int a, vector int b)
 {
@@ -2380,6 +3648,12 @@ vec_nor(vector unsigned int a, vector unsigned int b)
   return ~(a | b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_nor(vector bool int a, vector bool int b)
+{
+  return ~(a | b);
+}
+
 static vector float __ATTRS_o_ai
 vec_nor(vector float a, vector float b)
 {
@@ -2401,6 +3675,12 @@ vec_vnor(vector unsigned char a, vector unsigned char b)
   return ~(a | b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_vnor(vector bool char a, vector bool char b)
+{
+  return ~(a | b);
+}
+
 static vector short __ATTRS_o_ai
 vec_vnor(vector short a, vector short b)
 {
@@ -2413,6 +3693,12 @@ vec_vnor(vector unsigned short a, vector unsigned short b)
   return ~(a | b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vnor(vector bool short a, vector bool short b)
+{
+  return ~(a | b);
+}
+
 static vector int __ATTRS_o_ai
 vec_vnor(vector int a, vector int b)
 {
@@ -2425,6 +3711,12 @@ vec_vnor(vector unsigned int a, vector unsigned int b)
   return ~(a | b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vnor(vector bool int a, vector bool int b)
+{
+  return ~(a | b);
+}
+
 static vector float __ATTRS_o_ai
 vec_vnor(vector float a, vector float b)
 {
@@ -2442,36 +3734,126 @@ vec_or(vector signed char a, vector signed char b)
   return a | b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_or(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a | b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_or(vector signed char a, vector bool char b)
+{
+  return a | (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_or(vector unsigned char a, vector unsigned char b)
 {
   return a | b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_or(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a | b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_or(vector unsigned char a, vector bool char b)
+{
+  return a | (vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_or(vector bool char a, vector bool char b)
+{
+  return a | b;
+}
+
 static vector short __ATTRS_o_ai
 vec_or(vector short a, vector short b)
 {
   return a | b;
 }
 
+static vector short __ATTRS_o_ai
+vec_or(vector bool short a, vector short b)
+{
+  return (vector short)a | b;
+}
+
+static vector short __ATTRS_o_ai
+vec_or(vector short a, vector bool short b)
+{
+  return a | (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_or(vector unsigned short a, vector unsigned short b)
 {
   return a | b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_or(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a | b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_or(vector unsigned short a, vector bool short b)
+{
+  return a | (vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_or(vector bool short a, vector bool short b)
+{
+  return a | b;
+}
+
 static vector int __ATTRS_o_ai
 vec_or(vector int a, vector int b)
 {
   return a | b;
 }
 
+static vector int __ATTRS_o_ai
+vec_or(vector bool int a, vector int b)
+{
+  return (vector int)a | b;
+}
+
+static vector int __ATTRS_o_ai
+vec_or(vector int a, vector bool int b)
+{
+  return a | (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_or(vector unsigned int a, vector unsigned int b)
 {
   return a | b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_or(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a | b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_or(vector unsigned int a, vector bool int b)
+{
+  return a | (vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_or(vector bool int a, vector bool int b)
+{
+  return a | b;
+}
+
 static vector float __ATTRS_o_ai
 vec_or(vector float a, vector float b)
 {
@@ -2479,6 +3861,20 @@ vec_or(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_or(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a | (vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_or(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a | (vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_vor */
 
 static vector signed char __ATTRS_o_ai
@@ -2487,36 +3883,126 @@ vec_vor(vector signed char a, vector signed char b)
   return a | b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vor(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a | b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vor(vector signed char a, vector bool char b)
+{
+  return a | (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vor(vector unsigned char a, vector unsigned char b)
 {
   return a | b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vor(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a | b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vor(vector unsigned char a, vector bool char b)
+{
+  return a | (vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vor(vector bool char a, vector bool char b)
+{
+  return a | b;
+}
+
 static vector short __ATTRS_o_ai
 vec_vor(vector short a, vector short b)
 {
   return a | b;
 }
 
+static vector short __ATTRS_o_ai
+vec_vor(vector bool short a, vector short b)
+{
+  return (vector short)a | b;
+}
+
+static vector short __ATTRS_o_ai
+vec_vor(vector short a, vector bool short b)
+{
+  return a | (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vor(vector unsigned short a, vector unsigned short b)
 {
   return a | b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vor(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a | b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vor(vector unsigned short a, vector bool short b)
+{
+  return a | (vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vor(vector bool short a, vector bool short b)
+{
+  return a | b;
+}
+
 static vector int __ATTRS_o_ai
 vec_vor(vector int a, vector int b)
 {
   return a | b;
 }
 
+static vector int __ATTRS_o_ai
+vec_vor(vector bool int a, vector int b)
+{
+  return (vector int)a | b;
+}
+
+static vector int __ATTRS_o_ai
+vec_vor(vector int a, vector bool int b)
+{
+  return a | (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vor(vector unsigned int a, vector unsigned int b)
 {
   return a | b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vor(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a | b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vor(vector unsigned int a, vector bool int b)
+{
+  return a | (vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vor(vector bool int a, vector bool int b)
+{
+  return a | b;
+}
+
 static vector float __ATTRS_o_ai
 vec_vor(vector float a, vector float b)
 {
@@ -2524,6 +4010,20 @@ vec_vor(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_vor(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a | (vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_vor(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a | (vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_pack */
 
 static vector signed char __ATTRS_o_ai
@@ -2542,6 +4042,14 @@ vec_pack(vector unsigned short a, vector unsigned short b)
      0x11, 0x13, 0x15, 0x17, 0x19, 0x1B, 0x1D, 0x1F));
 }
 
+static vector bool char __ATTRS_o_ai
+vec_pack(vector bool short a, vector bool short b)
+{
+  return (vector bool char)vec_perm(a, b, (vector unsigned char)
+    (0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F,
+     0x11, 0x13, 0x15, 0x17, 0x19, 0x1B, 0x1D, 0x1F));
+}
+
 static vector short __ATTRS_o_ai
 vec_pack(vector int a, vector int b)
 {
@@ -2558,6 +4066,14 @@ vec_pack(vector unsigned int a, vector unsigned int b)
      0x12, 0x13, 0x16, 0x17, 0x1A, 0x1B, 0x1E, 0x1F));
 }
 
+static vector bool short __ATTRS_o_ai
+vec_pack(vector bool int a, vector bool int b)
+{
+  return (vector bool short)vec_perm(a, b, (vector unsigned char)
+    (0x02, 0x03, 0x06, 0x07, 0x0A, 0x0B, 0x0E, 0x0F,
+     0x12, 0x13, 0x16, 0x17, 0x1A, 0x1B, 0x1E, 0x1F));
+}
+
 /* vec_vpkuhum */
 
 #define __builtin_altivec_vpkuhum vec_vpkuhum
@@ -2578,6 +4094,14 @@ vec_vpkuhum(vector unsigned short a, vector unsigned short b)
      0x11, 0x13, 0x15, 0x17, 0x19, 0x1B, 0x1D, 0x1F));
 }
 
+static vector bool char __ATTRS_o_ai
+vec_vpkuhum(vector bool short a, vector bool short b)
+{
+  return (vector bool char)vec_perm(a, b, (vector unsigned char)
+    (0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F,
+     0x11, 0x13, 0x15, 0x17, 0x19, 0x1B, 0x1D, 0x1F));
+}
+
 /* vec_vpkuwum */
 
 #define __builtin_altivec_vpkuwum vec_vpkuwum
@@ -2598,6 +4122,14 @@ vec_vpkuwum(vector unsigned int a, vector unsigned int b)
      0x12, 0x13, 0x16, 0x17, 0x1A, 0x1B, 0x1E, 0x1F));
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vpkuwum(vector bool int a, vector bool int b)
+{
+  return (vector bool short)vec_perm(a, b, (vector unsigned char)
+    (0x02, 0x03, 0x06, 0x07, 0x0A, 0x0B, 0x0E, 0x0F,
+     0x12, 0x13, 0x16, 0x17, 0x1A, 0x1B, 0x1E, 0x1F));
+}
+
 /* vec_packpx */
 
 static vector pixel __attribute__((__always_inline__))
@@ -2740,6 +4272,12 @@ vec_perm(vector unsigned char a, vector unsigned char b, vector unsigned char c)
   return (vector unsigned char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
+vector bool char __ATTRS_o_ai
+vec_perm(vector bool char a, vector bool char b, vector unsigned char c)
+{
+  return (vector bool char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
 vector short __ATTRS_o_ai
 vec_perm(vector short a, vector short b, vector unsigned char c)
 {
@@ -2752,6 +4290,18 @@ vec_perm(vector unsigned short a, vector unsigned short b, vector unsigned char
   return (vector unsigned short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
+vector bool short __ATTRS_o_ai
+vec_perm(vector bool short a, vector bool short b, vector unsigned char c)
+{
+  return (vector bool short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
+vector pixel __ATTRS_o_ai
+vec_perm(vector pixel a, vector pixel b, vector unsigned char c)
+{
+  return (vector pixel)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
 vector int __ATTRS_o_ai
 vec_perm(vector int a, vector int b, vector unsigned char c)
 {
@@ -2764,6 +4314,12 @@ vec_perm(vector unsigned int a, vector unsigned int b, vector unsigned char c)
   return (vector unsigned int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
+vector bool int __ATTRS_o_ai
+vec_perm(vector bool int a, vector bool int b, vector unsigned char c)
+{
+  return (vector bool int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
 vector float __ATTRS_o_ai
 vec_perm(vector float a, vector float b, vector unsigned char c)
 {
@@ -2784,6 +4340,12 @@ vec_vperm(vector unsigned char a, vector unsigned char b, vector unsigned char c
   return (vector unsigned char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
+vector bool char __ATTRS_o_ai
+vec_vperm(vector bool char a, vector bool char b, vector unsigned char c)
+{
+  return (vector bool char)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
 vector short __ATTRS_o_ai
 vec_vperm(vector short a, vector short b, vector unsigned char c)
 {
@@ -2796,6 +4358,18 @@ vec_vperm(vector unsigned short a, vector unsigned short b, vector unsigned char
   return (vector unsigned short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
+vector bool short __ATTRS_o_ai
+vec_vperm(vector bool short a, vector bool short b, vector unsigned char c)
+{
+  return (vector bool short)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
+vector pixel __ATTRS_o_ai
+vec_vperm(vector pixel a, vector pixel b, vector unsigned char c)
+{
+  return (vector pixel)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
 vector int __ATTRS_o_ai
 vec_vperm(vector int a, vector int b, vector unsigned char c)
 {
@@ -2808,6 +4382,12 @@ vec_vperm(vector unsigned int a, vector unsigned int b, vector unsigned char c)
   return (vector unsigned int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
+vector bool int __ATTRS_o_ai
+vec_vperm(vector bool int a, vector bool int b, vector unsigned char c)
+{
+  return (vector bool int)__builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
+}
+
 vector float __ATTRS_o_ai
 vec_vperm(vector float a, vector float b, vector unsigned char c)
 {
@@ -2952,36 +4532,108 @@ vec_sel(vector signed char a, vector signed char b, vector unsigned char c)
   return (a & ~(vector signed char)c) | (b & (vector signed char)c);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_sel(vector signed char a, vector signed char b, vector bool char c)
+{
+  return (a & ~(vector signed char)c) | (b & (vector signed char)c);
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_sel(vector unsigned char a, vector unsigned char b, vector unsigned char c)
 {
   return (a & ~c) | (b & c);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_sel(vector unsigned char a, vector unsigned char b, vector bool char c)
+{
+  return (a & ~(vector unsigned char)c) | (b & (vector unsigned char)c);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_sel(vector bool char a, vector bool char b, vector unsigned char c)
+{
+  return (a & ~(vector bool char)c) | (b & (vector bool char)c);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_sel(vector bool char a, vector bool char b, vector bool char c)
+{
+  return (a & ~c) | (b & c);
+}
+
 static vector short __ATTRS_o_ai
 vec_sel(vector short a, vector short b, vector unsigned short c)
 {
   return (a & ~(vector short)c) | (b & (vector short)c);
 }
 
+static vector short __ATTRS_o_ai
+vec_sel(vector short a, vector short b, vector bool short c)
+{
+  return (a & ~(vector short)c) | (b & (vector short)c);
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_sel(vector unsigned short a, vector unsigned short b, vector unsigned short c)
 {
   return (a & ~c) | (b & c);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_sel(vector unsigned short a, vector unsigned short b, vector bool short c)
+{
+  return (a & ~(vector unsigned short)c) | (b & (vector unsigned short)c);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_sel(vector bool short a, vector bool short b, vector unsigned short c)
+{
+  return (a & ~(vector bool short)c) | (b & (vector bool short)c);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_sel(vector bool short a, vector bool short b, vector bool short c)
+{
+  return (a & ~c) | (b & c);
+}
+
 static vector int __ATTRS_o_ai
 vec_sel(vector int a, vector int b, vector unsigned int c)
 {
   return (a & ~(vector int)c) | (b & (vector int)c);
 }
 
+static vector int __ATTRS_o_ai
+vec_sel(vector int a, vector int b, vector bool int c)
+{
+  return (a & ~(vector int)c) | (b & (vector int)c);
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_sel(vector unsigned int a, vector unsigned int b, vector unsigned int c)
 {
   return (a & ~c) | (b & c);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_sel(vector unsigned int a, vector unsigned int b, vector bool int c)
+{
+  return (a & ~(vector unsigned int)c) | (b & (vector unsigned int)c);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_sel(vector bool int a, vector bool int b, vector unsigned int c)
+{
+  return (a & ~(vector bool int)c) | (b & (vector bool int)c);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_sel(vector bool int a, vector bool int b, vector bool int c)
+{
+  return (a & ~c) | (b & c);
+}
+
 static vector float __ATTRS_o_ai
 vec_sel(vector float a, vector float b, vector unsigned int c)
 {
@@ -2989,6 +4641,13 @@ vec_sel(vector float a, vector float b, vector unsigned int c)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_sel(vector float a, vector float b, vector bool int c)
+{
+  vector int res = ((vector int)a & ~(vector int)c) | ((vector int)b & (vector int)c);
+  return (vector float)res;
+}
+
 /* vec_vsel */
 
 static vector signed char __ATTRS_o_ai
@@ -2997,36 +4656,108 @@ vec_vsel(vector signed char a, vector signed char b, vector unsigned char c)
   return (a & ~(vector signed char)c) | (b & (vector signed char)c);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vsel(vector signed char a, vector signed char b, vector bool char c)
+{
+  return (a & ~(vector signed char)c) | (b & (vector signed char)c);
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vsel(vector unsigned char a, vector unsigned char b, vector unsigned char c)
 {
   return (a & ~c) | (b & c);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vsel(vector unsigned char a, vector unsigned char b, vector bool char c)
+{
+  return (a & ~(vector unsigned char)c) | (b & (vector unsigned char)c);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vsel(vector bool char a, vector bool char b, vector unsigned char c)
+{
+  return (a & ~(vector bool char)c) | (b & (vector bool char)c);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vsel(vector bool char a, vector bool char b, vector bool char c)
+{
+  return (a & ~c) | (b & c);
+}
+
 static vector short __ATTRS_o_ai
 vec_vsel(vector short a, vector short b, vector unsigned short c)
 {
   return (a & ~(vector short)c) | (b & (vector short)c);
 }
 
+static vector short __ATTRS_o_ai
+vec_vsel(vector short a, vector short b, vector bool short c)
+{
+  return (a & ~(vector short)c) | (b & (vector short)c);
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vsel(vector unsigned short a, vector unsigned short b, vector unsigned short c)
 {
   return (a & ~c) | (b & c);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vsel(vector unsigned short a, vector unsigned short b, vector bool short c)
+{
+  return (a & ~(vector unsigned short)c) | (b & (vector unsigned short)c);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsel(vector bool short a, vector bool short b, vector unsigned short c)
+{
+  return (a & ~(vector bool short)c) | (b & (vector bool short)c);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsel(vector bool short a, vector bool short b, vector bool short c)
+{
+  return (a & ~c) | (b & c);
+}
+
 static vector int __ATTRS_o_ai
 vec_vsel(vector int a, vector int b, vector unsigned int c)
 {
   return (a & ~(vector int)c) | (b & (vector int)c);
 }
 
+static vector int __ATTRS_o_ai
+vec_vsel(vector int a, vector int b, vector bool int c)
+{
+  return (a & ~(vector int)c) | (b & (vector int)c);
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vsel(vector unsigned int a, vector unsigned int b, vector unsigned int c)
 {
   return (a & ~c) | (b & c);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vsel(vector unsigned int a, vector unsigned int b, vector bool int c)
+{
+  return (a & ~(vector unsigned int)c) | (b & (vector unsigned int)c);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vsel(vector bool int a, vector bool int b, vector unsigned int c)
+{
+  return (a & ~(vector bool int)c) | (b & (vector bool int)c);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vsel(vector bool int a, vector bool int b, vector bool int c)
+{
+  return (a & ~c) | (b & c);
+}
+
 static vector float __ATTRS_o_ai
 vec_vsel(vector float a, vector float b, vector unsigned int c)
 {
@@ -3034,6 +4765,13 @@ vec_vsel(vector float a, vector float b, vector unsigned int c)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_vsel(vector float a, vector float b, vector bool int c)
+{
+  vector int res = ((vector int)a & ~(vector int)c) | ((vector int)b & (vector int)c);
+  return (vector float)res;
+}
+
 /* vec_sl */
 
 static vector signed char __ATTRS_o_ai
@@ -3127,7 +4865,7 @@ vec_vslw(vector unsigned int a, vector unsigned int b)
 static vector signed char __ATTRS_o_ai
 vec_sld(vector signed char a, vector signed char b, unsigned char c)
 {
-  return (vector signed char)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3135,7 +4873,7 @@ vec_sld(vector signed char a, vector signed char b, unsigned char c)
 static vector unsigned char __ATTRS_o_ai
 vec_sld(vector unsigned char a, vector unsigned char b, unsigned char c)
 {
-  return (vector unsigned char)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3143,7 +4881,7 @@ vec_sld(vector unsigned char a, vector unsigned char b, unsigned char c)
 static vector short __ATTRS_o_ai
 vec_sld(vector short a, vector short b, unsigned char c)
 {
-  return (vector short)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3151,7 +4889,15 @@ vec_sld(vector short a, vector short b, unsigned char c)
 static vector unsigned short __ATTRS_o_ai
 vec_sld(vector unsigned short a, vector unsigned short b, unsigned char c)
 {
-  return (vector unsigned short)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
+    (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
+     c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_sld(vector pixel a, vector pixel b, unsigned char c)
+{
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3167,7 +4913,7 @@ vec_sld(vector int a, vector int b, unsigned char c)
 static vector unsigned int __ATTRS_o_ai
 vec_sld(vector unsigned int a, vector unsigned int b, unsigned char c)
 {
-  return (vector unsigned int)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3175,7 +4921,7 @@ vec_sld(vector unsigned int a, vector unsigned int b, unsigned char c)
 static vector float __ATTRS_o_ai
 vec_sld(vector float a, vector float b, unsigned char c)
 {
-  return (vector float)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3185,7 +4931,7 @@ vec_sld(vector float a, vector float b, unsigned char c)
 static vector signed char __ATTRS_o_ai
 vec_vsldoi(vector signed char a, vector signed char b, unsigned char c)
 {
-  return (vector signed char)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3193,7 +4939,7 @@ vec_vsldoi(vector signed char a, vector signed char b, unsigned char c)
 static vector unsigned char __ATTRS_o_ai
 vec_vsldoi(vector unsigned char a, vector unsigned char b, unsigned char c)
 {
-  return (vector unsigned char)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3201,7 +4947,7 @@ vec_vsldoi(vector unsigned char a, vector unsigned char b, unsigned char c)
 static vector short __ATTRS_o_ai
 vec_vsldoi(vector short a, vector short b, unsigned char c)
 {
-  return (vector short)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3209,7 +4955,15 @@ vec_vsldoi(vector short a, vector short b, unsigned char c)
 static vector unsigned short __ATTRS_o_ai
 vec_vsldoi(vector unsigned short a, vector unsigned short b, unsigned char c)
 {
-  return (vector unsigned short)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
+    (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
+     c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsldoi(vector pixel a, vector pixel b, unsigned char c)
+{
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3225,7 +4979,7 @@ vec_vsldoi(vector int a, vector int b, unsigned char c)
 static vector unsigned int __ATTRS_o_ai
 vec_vsldoi(vector unsigned int a, vector unsigned int b, unsigned char c)
 {
-  return (vector unsigned int)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3233,7 +4987,7 @@ vec_vsldoi(vector unsigned int a, vector unsigned int b, unsigned char c)
 static vector float __ATTRS_o_ai
 vec_vsldoi(vector float a, vector float b, unsigned char c)
 {
-  return (vector float)vec_perm(a, b, (vector unsigned char)
+  return vec_perm(a, b, (vector unsigned char)
     (c,   c+1, c+2,  c+3,  c+4,  c+5,  c+6,  c+7, 
      c+8, c+9, c+10, c+11, c+12, c+13, c+14, c+15));
 }
@@ -3276,6 +5030,24 @@ vec_sll(vector unsigned char a, vector unsigned int b)
   return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_sll(vector bool char a, vector unsigned char b)
+{
+  return (vector bool char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_sll(vector bool char a, vector unsigned short b)
+{
+  return (vector bool char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_sll(vector bool char a, vector unsigned int b)
+{
+  return (vector bool char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_sll(vector short a, vector unsigned char b)
 {
@@ -3312,6 +5084,42 @@ vec_sll(vector unsigned short a, vector unsigned int b)
   return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_sll(vector bool short a, vector unsigned char b)
+{
+  return (vector bool short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_sll(vector bool short a, vector unsigned short b)
+{
+  return (vector bool short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_sll(vector bool short a, vector unsigned int b)
+{
+  return (vector bool short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_sll(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_sll(vector pixel a, vector unsigned short b)
+{
+  return (vector pixel)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_sll(vector pixel a, vector unsigned int b)
+{
+  return (vector pixel)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_sll(vector int a, vector unsigned char b)
 {
@@ -3348,6 +5156,24 @@ vec_sll(vector unsigned int a, vector unsigned int b)
   return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_sll(vector bool int a, vector unsigned char b)
+{
+  return (vector bool int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_sll(vector bool int a, vector unsigned short b)
+{
+  return (vector bool int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_sll(vector bool int a, vector unsigned int b)
+{
+  return (vector bool int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
 /* vec_vsl */
 
 static vector signed char __ATTRS_o_ai
@@ -3386,6 +5212,24 @@ vec_vsl(vector unsigned char a, vector unsigned int b)
   return (vector unsigned char)__builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_vsl(vector bool char a, vector unsigned char b)
+{
+  return (vector bool char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vsl(vector bool char a, vector unsigned short b)
+{
+  return (vector bool char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vsl(vector bool char a, vector unsigned int b)
+{
+  return (vector bool char)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_vsl(vector short a, vector unsigned char b)
 {
@@ -3422,6 +5266,42 @@ vec_vsl(vector unsigned short a, vector unsigned int b)
   return (vector unsigned short)__builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vsl(vector bool short a, vector unsigned char b)
+{
+  return (vector bool short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsl(vector bool short a, vector unsigned short b)
+{
+  return (vector bool short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsl(vector bool short a, vector unsigned int b)
+{
+  return (vector bool short)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsl(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsl(vector pixel a, vector unsigned short b)
+{
+  return (vector pixel)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsl(vector pixel a, vector unsigned int b)
+{
+  return (vector pixel)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_vsl(vector int a, vector unsigned char b)
 {
@@ -3458,6 +5338,24 @@ vec_vsl(vector unsigned int a, vector unsigned int b)
   return (vector unsigned int)__builtin_altivec_vsl((vector int)a, (vector int)b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vsl(vector bool int a, vector unsigned char b)
+{
+  return (vector bool int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vsl(vector bool int a, vector unsigned short b)
+{
+  return (vector bool int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vsl(vector bool int a, vector unsigned int b)
+{
+  return (vector bool int)__builtin_altivec_vsl((vector int)a, (vector int)b);
+}
+
 /* vec_slo */
 
 static vector signed char __ATTRS_o_ai
@@ -3508,6 +5406,18 @@ vec_slo(vector unsigned short a, vector unsigned char b)
   return (vector unsigned short)__builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
+static vector pixel __ATTRS_o_ai
+vec_slo(vector pixel a, vector signed char b)
+{
+  return (vector pixel)__builtin_altivec_vslo((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_slo(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vslo((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_slo(vector int a, vector signed char b)
 {
@@ -3594,6 +5504,18 @@ vec_vslo(vector unsigned short a, vector unsigned char b)
   return (vector unsigned short)__builtin_altivec_vslo((vector int)a, (vector int)b);
 }
 
+static vector pixel __ATTRS_o_ai
+vec_vslo(vector pixel a, vector signed char b)
+{
+  return (vector pixel)__builtin_altivec_vslo((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vslo(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vslo((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_vslo(vector int a, vector signed char b)
 {
@@ -3635,20 +5557,26 @@ vec_vslo(vector float a, vector unsigned char b)
 static vector signed char __ATTRS_o_ai
 vec_splat(vector signed char a, unsigned char b)
 {
-  return (vector signed char)vec_perm(a, a, (vector unsigned char)(b));
+  return vec_perm(a, a, (vector unsigned char)(b));
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_splat(vector unsigned char a, unsigned char b)
 {
-  return (vector unsigned char)vec_perm(a, a, (vector unsigned char)(b));
+  return vec_perm(a, a, (vector unsigned char)(b));
+}
+
+static vector bool char __ATTRS_o_ai
+vec_splat(vector bool char a, unsigned char b)
+{
+  return vec_perm(a, a, (vector unsigned char)(b));
 }
 
 static vector short __ATTRS_o_ai
 vec_splat(vector short a, unsigned char b)
 { 
   b *= 2;
-  return (vector short)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
 }
 
@@ -3656,7 +5584,23 @@ static vector unsigned short __ATTRS_o_ai
 vec_splat(vector unsigned short a, unsigned char b)
 { 
   b *= 2;
-  return (vector unsigned short)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
+    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+}
+
+static vector bool short __ATTRS_o_ai
+vec_splat(vector bool short a, unsigned char b)
+{ 
+  b *= 2;
+  return vec_perm(a, a, (vector unsigned char)
+    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_splat(vector pixel a, unsigned char b)
+{ 
+  b *= 2;
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
 }
 
@@ -3672,7 +5616,15 @@ static vector unsigned int __ATTRS_o_ai
 vec_splat(vector unsigned int a, unsigned char b)
 { 
   b *= 4;
-  return (vector unsigned int)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
+    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+}
+
+static vector bool int __ATTRS_o_ai
+vec_splat(vector bool int a, unsigned char b)
+{ 
+  b *= 4;
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
 }
 
@@ -3680,7 +5632,7 @@ static vector float __ATTRS_o_ai
 vec_splat(vector float a, unsigned char b)
 { 
   b *= 4;
-  return (vector float)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
 }
 
@@ -3691,13 +5643,19 @@ vec_splat(vector float a, unsigned char b)
 static vector signed char __ATTRS_o_ai
 vec_vspltb(vector signed char a, unsigned char b)
 {
-  return (vector signed char)vec_perm(a, a, (vector unsigned char)(b));
+  return vec_perm(a, a, (vector unsigned char)(b));
 }
 
 static vector unsigned char __ATTRS_o_ai
 vec_vspltb(vector unsigned char a, unsigned char b)
 {
-  return (vector unsigned char)vec_perm(a, a, (vector unsigned char)(b));
+  return vec_perm(a, a, (vector unsigned char)(b));
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vspltb(vector bool char a, unsigned char b)
+{
+  return vec_perm(a, a, (vector unsigned char)(b));
 }
 
 /* vec_vsplth */
@@ -3708,7 +5666,7 @@ static vector short __ATTRS_o_ai
 vec_vsplth(vector short a, unsigned char b)
 {
   b *= 2;
-  return (vector short)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
 }
 
@@ -3716,7 +5674,23 @@ static vector unsigned short __ATTRS_o_ai
 vec_vsplth(vector unsigned short a, unsigned char b)
 {
   b *= 2;
-  return (vector unsigned short)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
+    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsplth(vector bool short a, unsigned char b)
+{
+  b *= 2;
+  return vec_perm(a, a, (vector unsigned char)
+    (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsplth(vector pixel a, unsigned char b)
+{
+  b *= 2;
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1, b, b+1));
 }
 
@@ -3728,7 +5702,7 @@ static vector int __ATTRS_o_ai
 vec_vspltw(vector int a, unsigned char b)
 {
   b *= 4;
-  return (vector int)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
 }
 
@@ -3736,7 +5710,15 @@ static vector unsigned int __ATTRS_o_ai
 vec_vspltw(vector unsigned int a, unsigned char b)
 {
   b *= 4;
-  return (vector unsigned int)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
+    (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vspltw(vector bool int a, unsigned char b)
+{
+  b *= 4;
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
 }
 
@@ -3744,7 +5726,7 @@ static vector float __ATTRS_o_ai
 vec_vspltw(vector float a, unsigned char b)
 {
   b *= 4;
-  return (vector float)vec_perm(a, a, (vector unsigned char)
+  return vec_perm(a, a, (vector unsigned char)
     (b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3, b, b+1, b+2, b+3));
 }
 
@@ -4039,6 +6021,24 @@ vec_srl(vector unsigned char a, vector unsigned int b)
   return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_srl(vector bool char a, vector unsigned char b)
+{
+  return (vector bool char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_srl(vector bool char a, vector unsigned short b)
+{
+  return (vector bool char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_srl(vector bool char a, vector unsigned int b)
+{
+  return (vector bool char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_srl(vector short a, vector unsigned char b)
 {
@@ -4075,6 +6075,42 @@ vec_srl(vector unsigned short a, vector unsigned int b)
   return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_srl(vector bool short a, vector unsigned char b)
+{
+  return (vector bool short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_srl(vector bool short a, vector unsigned short b)
+{
+  return (vector bool short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_srl(vector bool short a, vector unsigned int b)
+{
+  return (vector bool short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_srl(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_srl(vector pixel a, vector unsigned short b)
+{
+  return (vector pixel)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_srl(vector pixel a, vector unsigned int b)
+{
+  return (vector pixel)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_srl(vector int a, vector unsigned char b)
 {
@@ -4111,6 +6147,24 @@ vec_srl(vector unsigned int a, vector unsigned int b)
   return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_srl(vector bool int a, vector unsigned char b)
+{
+  return (vector bool int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_srl(vector bool int a, vector unsigned short b)
+{
+  return (vector bool int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_srl(vector bool int a, vector unsigned int b)
+{
+  return (vector bool int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
 /* vec_vsr */
 
 static vector signed char __ATTRS_o_ai
@@ -4149,6 +6203,24 @@ vec_vsr(vector unsigned char a, vector unsigned int b)
   return (vector unsigned char)__builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
+static vector bool char __ATTRS_o_ai
+vec_vsr(vector bool char a, vector unsigned char b)
+{
+  return (vector bool char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vsr(vector bool char a, vector unsigned short b)
+{
+  return (vector bool char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vsr(vector bool char a, vector unsigned int b)
+{
+  return (vector bool char)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_vsr(vector short a, vector unsigned char b)
 {
@@ -4185,6 +6257,42 @@ vec_vsr(vector unsigned short a, vector unsigned int b)
   return (vector unsigned short)__builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vsr(vector bool short a, vector unsigned char b)
+{
+  return (vector bool short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsr(vector bool short a, vector unsigned short b)
+{
+  return (vector bool short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vsr(vector bool short a, vector unsigned int b)
+{
+  return (vector bool short)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsr(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsr(vector pixel a, vector unsigned short b)
+{
+  return (vector pixel)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsr(vector pixel a, vector unsigned int b)
+{
+  return (vector pixel)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_vsr(vector int a, vector unsigned char b)
 {
@@ -4221,6 +6329,24 @@ vec_vsr(vector unsigned int a, vector unsigned int b)
   return (vector unsigned int)__builtin_altivec_vsr((vector int)a, (vector int)b);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vsr(vector bool int a, vector unsigned char b)
+{
+  return (vector bool int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vsr(vector bool int a, vector unsigned short b)
+{
+  return (vector bool int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vsr(vector bool int a, vector unsigned int b)
+{
+  return (vector bool int)__builtin_altivec_vsr((vector int)a, (vector int)b);
+}
+
 /* vec_sro */
 
 static vector signed char __ATTRS_o_ai
@@ -4271,6 +6397,18 @@ vec_sro(vector unsigned short a, vector unsigned char b)
   return (vector unsigned short)__builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
+static vector pixel __ATTRS_o_ai
+vec_sro(vector pixel a, vector signed char b)
+{
+  return (vector pixel)__builtin_altivec_vsro((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_sro(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vsro((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_sro(vector int a, vector signed char b)
 {
@@ -4357,6 +6495,18 @@ vec_vsro(vector unsigned short a, vector unsigned char b)
   return (vector unsigned short)__builtin_altivec_vsro((vector int)a, (vector int)b);
 }
 
+static vector pixel __ATTRS_o_ai
+vec_vsro(vector pixel a, vector signed char b)
+{
+  return (vector pixel)__builtin_altivec_vsro((vector int)a, (vector int)b);
+}
+
+static vector pixel __ATTRS_o_ai
+vec_vsro(vector pixel a, vector unsigned char b)
+{
+  return (vector pixel)__builtin_altivec_vsro((vector int)a, (vector int)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_vsro(vector int a, vector signed char b)
 {
@@ -4420,6 +6570,24 @@ vec_st(vector unsigned char a, int b, unsigned char *c)
 }
 
 static void __ATTRS_o_ai
+vec_st(vector bool char a, int b, signed char *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector bool char a, int b, unsigned char *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector bool char a, int b, vector bool char *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_st(vector short a, int b, vector short *c)
 {
   __builtin_altivec_stvx((vector int)a, b, c);
@@ -4444,6 +6612,42 @@ vec_st(vector unsigned short a, int b, unsigned short *c)
 }
 
 static void __ATTRS_o_ai
+vec_st(vector bool short a, int b, short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector bool short a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector bool short a, int b, vector bool short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector pixel a, int b, short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector pixel a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector pixel a, int b, vector pixel *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_st(vector int a, int b, vector int *c)
 {
   __builtin_altivec_stvx(a, b, c);
@@ -4468,6 +6672,24 @@ vec_st(vector unsigned int a, int b, unsigned int *c)
 }
 
 static void __ATTRS_o_ai
+vec_st(vector bool int a, int b, int *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector bool int a, int b, unsigned int *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_st(vector bool int a, int b, vector bool int *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_st(vector float a, int b, vector float *c)
 {
   __builtin_altivec_stvx((vector int)a, b, c);
@@ -4506,6 +6728,24 @@ vec_stvx(vector unsigned char a, int b, unsigned char *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvx(vector bool char a, int b, signed char *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector bool char a, int b, unsigned char *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector bool char a, int b, vector bool char *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvx(vector short a, int b, vector short *c)
 {
   __builtin_altivec_stvx((vector int)a, b, c);
@@ -4530,6 +6770,42 @@ vec_stvx(vector unsigned short a, int b, unsigned short *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvx(vector bool short a, int b, short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector bool short a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector bool short a, int b, vector bool short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector pixel a, int b, short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector pixel a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector pixel a, int b, vector pixel *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvx(vector int a, int b, vector int *c)
 {
   __builtin_altivec_stvx(a, b, c);
@@ -4554,6 +6830,24 @@ vec_stvx(vector unsigned int a, int b, unsigned int *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvx(vector bool int a, int b, int *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector bool int a, int b, unsigned int *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvx(vector bool int a, int b, vector bool int *c)
+{
+  __builtin_altivec_stvx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvx(vector float a, int b, vector float *c)
 {
   __builtin_altivec_stvx((vector int)a, b, c);
@@ -4580,6 +6874,18 @@ vec_ste(vector unsigned char a, int b, unsigned char *c)
 }
 
 static void __ATTRS_o_ai
+vec_ste(vector bool char a, int b, signed char *c)
+{
+  __builtin_altivec_stvebx((vector char)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_ste(vector bool char a, int b, unsigned char *c)
+{
+  __builtin_altivec_stvebx((vector char)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_ste(vector short a, int b, short *c)
 {
   __builtin_altivec_stvehx(a, b, c);
@@ -4592,6 +6898,30 @@ vec_ste(vector unsigned short a, int b, unsigned short *c)
 }
 
 static void __ATTRS_o_ai
+vec_ste(vector bool short a, int b, short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_ste(vector bool short a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_ste(vector pixel a, int b, short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_ste(vector pixel a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_ste(vector int a, int b, int *c)
 {
   __builtin_altivec_stvewx(a, b, c);
@@ -4604,6 +6934,18 @@ vec_ste(vector unsigned int a, int b, unsigned int *c)
 }
 
 static void __ATTRS_o_ai
+vec_ste(vector bool int a, int b, int *c)
+{
+  __builtin_altivec_stvewx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_ste(vector bool int a, int b, unsigned int *c)
+{
+  __builtin_altivec_stvewx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_ste(vector float a, int b, float *c)
 {
   __builtin_altivec_stvewx((vector int)a, b, c);
@@ -4623,6 +6965,18 @@ vec_stvebx(vector unsigned char a, int b, unsigned char *c)
   __builtin_altivec_stvebx((vector char)a, b, c);
 }
 
+static void __ATTRS_o_ai
+vec_stvebx(vector bool char a, int b, signed char *c)
+{
+  __builtin_altivec_stvebx((vector char)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvebx(vector bool char a, int b, unsigned char *c)
+{
+  __builtin_altivec_stvebx((vector char)a, b, c);
+}
+
 /* vec_stvehx */
 
 static void __ATTRS_o_ai
@@ -4637,6 +6991,30 @@ vec_stvehx(vector unsigned short a, int b, unsigned short *c)
   __builtin_altivec_stvehx((vector short)a, b, c);
 }
 
+static void __ATTRS_o_ai
+vec_stvehx(vector bool short a, int b, short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvehx(vector bool short a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvehx(vector pixel a, int b, short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvehx(vector pixel a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvehx((vector short)a, b, c);
+}
+
 /* vec_stvewx */
 
 static void __ATTRS_o_ai
@@ -4652,6 +7030,18 @@ vec_stvewx(vector unsigned int a, int b, unsigned int *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvewx(vector bool int a, int b, int *c)
+{
+  __builtin_altivec_stvewx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvewx(vector bool int a, int b, unsigned int *c)
+{
+  __builtin_altivec_stvewx((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvewx(vector float a, int b, float *c)
 {
   __builtin_altivec_stvewx((vector int)a, b, c);
@@ -4684,6 +7074,24 @@ vec_stl(vector unsigned char a, int b, unsigned char *c)
 }
 
 static void __ATTRS_o_ai
+vec_stl(vector bool char a, int b, signed char *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector bool char a, int b, unsigned char *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector bool char a, int b, vector bool char *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stl(vector short a, int b, vector short *c)
 {
   __builtin_altivec_stvxl((vector int)a, b, c);
@@ -4708,6 +7116,42 @@ vec_stl(vector unsigned short a, int b, unsigned short *c)
 }
 
 static void __ATTRS_o_ai
+vec_stl(vector bool short a, int b, short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector bool short a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector bool short a, int b, vector bool short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector pixel a, int b, short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector pixel a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector pixel a, int b, vector pixel *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stl(vector int a, int b, vector int *c)
 {
   __builtin_altivec_stvxl(a, b, c);
@@ -4732,6 +7176,24 @@ vec_stl(vector unsigned int a, int b, unsigned int *c)
 }
 
 static void __ATTRS_o_ai
+vec_stl(vector bool int a, int b, int *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector bool int a, int b, unsigned int *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stl(vector bool int a, int b, vector bool int *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stl(vector float a, int b, vector float *c)
 {
   __builtin_altivec_stvxl((vector int)a, b, c);
@@ -4770,6 +7232,24 @@ vec_stvxl(vector unsigned char a, int b, unsigned char *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvxl(vector bool char a, int b, signed char *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector bool char a, int b, unsigned char *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector bool char a, int b, vector bool char *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvxl(vector short a, int b, vector short *c)
 {
   __builtin_altivec_stvxl((vector int)a, b, c);
@@ -4794,6 +7274,42 @@ vec_stvxl(vector unsigned short a, int b, unsigned short *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvxl(vector bool short a, int b, short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector bool short a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector bool short a, int b, vector bool short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector pixel a, int b, short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector pixel a, int b, unsigned short *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector pixel a, int b, vector pixel *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvxl(vector int a, int b, vector int *c)
 {
   __builtin_altivec_stvxl(a, b, c);
@@ -4818,6 +7334,24 @@ vec_stvxl(vector unsigned int a, int b, unsigned int *c)
 }
 
 static void __ATTRS_o_ai
+vec_stvxl(vector bool int a, int b, int *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector bool int a, int b, unsigned int *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
+vec_stvxl(vector bool int a, int b, vector bool int *c)
+{
+  __builtin_altivec_stvxl((vector int)a, b, c);
+}
+
+static void __ATTRS_o_ai
 vec_stvxl(vector float a, int b, vector float *c)
 {
   __builtin_altivec_stvxl((vector int)a, b, c);
@@ -4837,36 +7371,108 @@ vec_sub(vector signed char a, vector signed char b)
   return a - b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_sub(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a - b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_sub(vector signed char a, vector bool char b)
+{
+  return a - (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_sub(vector unsigned char a, vector unsigned char b)
 {
   return a - b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_sub(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a - b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_sub(vector unsigned char a, vector bool char b)
+{
+  return a - (vector unsigned char)b;
+}
+
 static vector short __ATTRS_o_ai
 vec_sub(vector short a, vector short b)
 {
   return a - b;
 }
 
+static vector short __ATTRS_o_ai
+vec_sub(vector bool short a, vector short b)
+{
+  return (vector short)a - b;
+}
+
+static vector short __ATTRS_o_ai
+vec_sub(vector short a, vector bool short b)
+{
+  return a - (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_sub(vector unsigned short a, vector unsigned short b)
 {
   return a - b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_sub(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a - b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_sub(vector unsigned short a, vector bool short b)
+{
+  return a - (vector unsigned short)b;
+}
+
 static vector int __ATTRS_o_ai
 vec_sub(vector int a, vector int b)
 {
   return a - b;
 }
 
+static vector int __ATTRS_o_ai
+vec_sub(vector bool int a, vector int b)
+{
+  return (vector int)a - b;
+}
+
+static vector int __ATTRS_o_ai
+vec_sub(vector int a, vector bool int b)
+{
+  return a - (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_sub(vector unsigned int a, vector unsigned int b)
 {
   return a - b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_sub(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a - b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_sub(vector unsigned int a, vector bool int b)
+{
+  return a - (vector unsigned int)b;
+}
+
 static vector float __ATTRS_o_ai
 vec_sub(vector float a, vector float b)
 {
@@ -4883,12 +7489,36 @@ vec_vsububm(vector signed char a, vector signed char b)
   return a - b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vsububm(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a - b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vsububm(vector signed char a, vector bool char b)
+{
+  return a - (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vsububm(vector unsigned char a, vector unsigned char b)
 {
   return a - b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vsububm(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a - b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vsububm(vector unsigned char a, vector bool char b)
+{
+  return a - (vector unsigned char)b;
+}
+
 /* vec_vsubuhm */
 
 #define __builtin_altivec_vsubuhm vec_vsubuhm
@@ -4899,12 +7529,36 @@ vec_vsubuhm(vector short a, vector short b)
   return a - b;
 }
 
+static vector short __ATTRS_o_ai
+vec_vsubuhm(vector bool short a, vector short b)
+{
+  return (vector short)a - b;
+}
+
+static vector short __ATTRS_o_ai
+vec_vsubuhm(vector short a, vector bool short b)
+{
+  return a - (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vsubuhm(vector unsigned short a, vector unsigned short b)
 {
   return a - b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vsubuhm(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a - b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vsubuhm(vector unsigned short a, vector bool short b)
+{
+  return a - (vector unsigned short)b;
+}
+
 /* vec_vsubuwm */
 
 #define __builtin_altivec_vsubuwm vec_vsubuwm
@@ -4915,12 +7569,36 @@ vec_vsubuwm(vector int a, vector int b)
   return a - b;
 }
 
+static vector int __ATTRS_o_ai
+vec_vsubuwm(vector bool int a, vector int b)
+{
+  return (vector int)a - b;
+}
+
+static vector int __ATTRS_o_ai
+vec_vsubuwm(vector int a, vector bool int b)
+{
+  return a - (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vsubuwm(vector unsigned int a, vector unsigned int b)
 {
   return a - b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vsubuwm(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a - b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vsubuwm(vector unsigned int a, vector bool int b)
+{
+  return a - (vector unsigned int)b;
+}
+
 /* vec_vsubfp */
 
 #define __builtin_altivec_vsubfp vec_vsubfp
@@ -4955,84 +7633,228 @@ vec_subs(vector signed char a, vector signed char b)
   return __builtin_altivec_vsubsbs(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_subs(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vsubsbs((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_subs(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vsubsbs(a, (vector signed char)b);
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_subs(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vsububs(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_subs(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vsububs((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_subs(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vsububs(a, (vector unsigned char)b);
+}
+
 static vector short __ATTRS_o_ai
 vec_subs(vector short a, vector short b)
 {
   return __builtin_altivec_vsubshs(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_subs(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vsubshs((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_subs(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vsubshs(a, (vector short)b);
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_subs(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vsubuhs(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_subs(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vsubuhs((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_subs(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vsubuhs(a, (vector unsigned short)b);
+}
+
 static vector int __ATTRS_o_ai
 vec_subs(vector int a, vector int b)
 {
   return __builtin_altivec_vsubsws(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_subs(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vsubsws((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_subs(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vsubsws(a, (vector int)b);
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_subs(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vsubuws(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_subs(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vsubuws((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_subs(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vsubuws(a, (vector unsigned int)b);
+}
+
 /* vec_vsubsbs */
 
-static vector signed char __attribute__((__always_inline__))
+static vector signed char __ATTRS_o_ai
 vec_vsubsbs(vector signed char a, vector signed char b)
 {
   return __builtin_altivec_vsubsbs(a, b);
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vsubsbs(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vsubsbs((vector signed char)a, b);
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vsubsbs(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vsubsbs(a, (vector signed char)b);
+}
+
 /* vec_vsububs */
 
-static vector unsigned char __attribute__((__always_inline__))
+static vector unsigned char __ATTRS_o_ai
 vec_vsububs(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vsububs(a, b);
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vsububs(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vsububs((vector unsigned char)a, b);
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vsububs(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vsububs(a, (vector unsigned char)b);
+}
+
 /* vec_vsubshs */
 
-static vector short __attribute__((__always_inline__))
+static vector short __ATTRS_o_ai
 vec_vsubshs(vector short a, vector short b)
 {
   return __builtin_altivec_vsubshs(a, b);
 }
 
+static vector short __ATTRS_o_ai
+vec_vsubshs(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vsubshs((vector short)a, b);
+}
+
+static vector short __ATTRS_o_ai
+vec_vsubshs(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vsubshs(a, (vector short)b);
+}
+
 /* vec_vsubuhs */
 
-static vector unsigned short __attribute__((__always_inline__))
+static vector unsigned short __ATTRS_o_ai
 vec_vsubuhs(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vsubuhs(a, b);
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vsubuhs(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vsubuhs((vector unsigned short)a, b);
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vsubuhs(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vsubuhs(a, (vector unsigned short)b);
+}
+
 /* vec_vsubsws */
 
-static vector int __attribute__((__always_inline__))
+static vector int __ATTRS_o_ai
 vec_vsubsws(vector int a, vector int b)
 {
   return __builtin_altivec_vsubsws(a, b);
 }
 
+static vector int __ATTRS_o_ai
+vec_vsubsws(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vsubsws((vector int)a, b);
+}
+
+static vector int __ATTRS_o_ai
+vec_vsubsws(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vsubsws(a, (vector int)b);
+}
+
 /* vec_vsubuws */
 
-static vector unsigned int __attribute__((__always_inline__))
+static vector unsigned int __ATTRS_o_ai
 vec_vsubuws(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vsubuws(a, b);
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vsubuws(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vsubuws((vector unsigned int)a, b);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vsubuws(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vsubuws(a, (vector unsigned int)b);
+}
+
 /* vec_sum4s */
 
 static vector int __ATTRS_o_ai
@@ -5133,28 +7955,64 @@ vec_unpackh(vector signed char a)
   return __builtin_altivec_vupkhsb((vector char)a);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_unpackh(vector bool char a)
+{
+  return (vector bool short)__builtin_altivec_vupkhsb((vector char)a);
+}
+
 static vector int __ATTRS_o_ai
 vec_unpackh(vector short a)
 {
   return __builtin_altivec_vupkhsh(a);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_unpackh(vector bool short a)
+{
+  return (vector bool int)__builtin_altivec_vupkhsh((vector short)a);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_unpackh(vector pixel a)
+{
+  return (vector unsigned int)__builtin_altivec_vupkhsh((vector short)a);
+}
+
 /* vec_vupkhsb */
 
-static vector short __attribute__((__always_inline__))
+static vector short __ATTRS_o_ai
 vec_vupkhsb(vector signed char a)
 {
   return __builtin_altivec_vupkhsb((vector char)a);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vupkhsb(vector bool char a)
+{
+  return (vector bool short)__builtin_altivec_vupkhsb((vector char)a);
+}
+
 /* vec_vupkhsh */
 
-static vector int __attribute__((__always_inline__))
+static vector int __ATTRS_o_ai
 vec_vupkhsh(vector short a)
 {
   return __builtin_altivec_vupkhsh(a);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vupkhsh(vector bool short a)
+{
+  return (vector bool int)__builtin_altivec_vupkhsh((vector short)a);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vupkhsh(vector pixel a)
+{
+  return (vector unsigned int)__builtin_altivec_vupkhsh((vector short)a);
+}
+
 /* vec_unpackl */
 
 static vector short __ATTRS_o_ai
@@ -5163,28 +8021,64 @@ vec_unpackl(vector signed char a)
   return __builtin_altivec_vupklsb((vector char)a);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_unpackl(vector bool char a)
+{
+  return (vector bool short)__builtin_altivec_vupklsb((vector char)a);
+}
+
 static vector int __ATTRS_o_ai
 vec_unpackl(vector short a)
 {
   return __builtin_altivec_vupklsh(a);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_unpackl(vector bool short a)
+{
+  return (vector bool int)__builtin_altivec_vupklsh((vector short)a);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_unpackl(vector pixel a)
+{
+  return (vector unsigned int)__builtin_altivec_vupklsh((vector short)a);
+}
+
 /* vec_vupklsb */
 
-static vector short __attribute__((__always_inline__))
+static vector short __ATTRS_o_ai
 vec_vupklsb(vector signed char a)
 {
   return __builtin_altivec_vupklsb((vector char)a);
 }
 
+static vector bool short __ATTRS_o_ai
+vec_vupklsb(vector bool char a)
+{
+  return (vector bool short)__builtin_altivec_vupklsb((vector char)a);
+}
+
 /* vec_vupklsh */
 
-static vector int __attribute__((__always_inline__))
+static vector int __ATTRS_o_ai
 vec_vupklsh(vector short a)
 {
   return __builtin_altivec_vupklsh(a);
 }
 
+static vector bool int __ATTRS_o_ai
+vec_vupklsh(vector bool short a)
+{
+  return (vector bool int)__builtin_altivec_vupklsh((vector short)a);
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vupklsh(vector pixel a)
+{
+  return (vector unsigned int)__builtin_altivec_vupklsh((vector short)a);
+}
+
 /* vec_xor */
 
 #define __builtin_altivec_vxor vec_xor
@@ -5195,36 +8089,126 @@ vec_xor(vector signed char a, vector signed char b)
   return a ^ b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_xor(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a ^ b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_xor(vector signed char a, vector bool char b)
+{
+  return a ^ (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_xor(vector unsigned char a, vector unsigned char b)
 {
   return a ^ b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_xor(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a ^ b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_xor(vector unsigned char a, vector bool char b)
+{
+  return a ^ (vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_xor(vector bool char a, vector bool char b)
+{
+  return a ^ b;
+}
+
 static vector short __ATTRS_o_ai
 vec_xor(vector short a, vector short b)
 {
   return a ^ b;
 }
 
+static vector short __ATTRS_o_ai
+vec_xor(vector bool short a, vector short b)
+{
+  return (vector short)a ^ b;
+}
+
+static vector short __ATTRS_o_ai
+vec_xor(vector short a, vector bool short b)
+{
+  return a ^ (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_xor(vector unsigned short a, vector unsigned short b)
 {
   return a ^ b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_xor(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a ^ b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_xor(vector unsigned short a, vector bool short b)
+{
+  return a ^ (vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_xor(vector bool short a, vector bool short b)
+{
+  return a ^ b;
+}
+
 static vector int __ATTRS_o_ai
 vec_xor(vector int a, vector int b)
 {
   return a ^ b;
 }
 
+static vector int __ATTRS_o_ai
+vec_xor(vector bool int a, vector int b)
+{
+  return (vector int)a ^ b;
+}
+
+static vector int __ATTRS_o_ai
+vec_xor(vector int a, vector bool int b)
+{
+  return a ^ (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_xor(vector unsigned int a, vector unsigned int b)
 {
   return a ^ b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_xor(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a ^ b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_xor(vector unsigned int a, vector bool int b)
+{
+  return a ^ (vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_xor(vector bool int a, vector bool int b)
+{
+  return a ^ b;
+}
+
 static vector float __ATTRS_o_ai
 vec_xor(vector float a, vector float b)
 {
@@ -5232,6 +8216,20 @@ vec_xor(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_xor(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a ^ (vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_xor(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a ^ (vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* vec_vxor */
 
 static vector signed char __ATTRS_o_ai
@@ -5240,36 +8238,126 @@ vec_vxor(vector signed char a, vector signed char b)
   return a ^ b;
 }
 
+static vector signed char __ATTRS_o_ai
+vec_vxor(vector bool char a, vector signed char b)
+{
+  return (vector signed char)a ^ b;
+}
+
+static vector signed char __ATTRS_o_ai
+vec_vxor(vector signed char a, vector bool char b)
+{
+  return a ^ (vector signed char)b;
+}
+
 static vector unsigned char __ATTRS_o_ai
 vec_vxor(vector unsigned char a, vector unsigned char b)
 {
   return a ^ b;
 }
 
+static vector unsigned char __ATTRS_o_ai
+vec_vxor(vector bool char a, vector unsigned char b)
+{
+  return (vector unsigned char)a ^ b;
+}
+
+static vector unsigned char __ATTRS_o_ai
+vec_vxor(vector unsigned char a, vector bool char b)
+{
+  return a ^ (vector unsigned char)b;
+}
+
+static vector bool char __ATTRS_o_ai
+vec_vxor(vector bool char a, vector bool char b)
+{
+  return a ^ b;
+}
+
 static vector short __ATTRS_o_ai
 vec_vxor(vector short a, vector short b)
 {
   return a ^ b;
 }
 
+static vector short __ATTRS_o_ai
+vec_vxor(vector bool short a, vector short b)
+{
+  return (vector short)a ^ b;
+}
+
+static vector short __ATTRS_o_ai
+vec_vxor(vector short a, vector bool short b)
+{
+  return a ^ (vector short)b;
+}
+
 static vector unsigned short __ATTRS_o_ai
 vec_vxor(vector unsigned short a, vector unsigned short b)
 {
   return a ^ b;
 }
 
+static vector unsigned short __ATTRS_o_ai
+vec_vxor(vector bool short a, vector unsigned short b)
+{
+  return (vector unsigned short)a ^ b;
+}
+
+static vector unsigned short __ATTRS_o_ai
+vec_vxor(vector unsigned short a, vector bool short b)
+{
+  return a ^ (vector unsigned short)b;
+}
+
+static vector bool short __ATTRS_o_ai
+vec_vxor(vector bool short a, vector bool short b)
+{
+  return a ^ b;
+}
+
 static vector int __ATTRS_o_ai
 vec_vxor(vector int a, vector int b)
 {
   return a ^ b;
 }
 
+static vector int __ATTRS_o_ai
+vec_vxor(vector bool int a, vector int b)
+{
+  return (vector int)a ^ b;
+}
+
+static vector int __ATTRS_o_ai
+vec_vxor(vector int a, vector bool int b)
+{
+  return a ^ (vector int)b;
+}
+
 static vector unsigned int __ATTRS_o_ai
 vec_vxor(vector unsigned int a, vector unsigned int b)
 {
   return a ^ b;
 }
 
+static vector unsigned int __ATTRS_o_ai
+vec_vxor(vector bool int a, vector unsigned int b)
+{
+  return (vector unsigned int)a ^ b;
+}
+
+static vector unsigned int __ATTRS_o_ai
+vec_vxor(vector unsigned int a, vector bool int b)
+{
+  return a ^ (vector unsigned int)b;
+}
+
+static vector bool int __ATTRS_o_ai
+vec_vxor(vector bool int a, vector bool int b)
+{
+  return a ^ b;
+}
+
 static vector float __ATTRS_o_ai
 vec_vxor(vector float a, vector float b)
 {
@@ -5277,6 +8365,20 @@ vec_vxor(vector float a, vector float b)
   return (vector float)res;
 }
 
+static vector float __ATTRS_o_ai
+vec_vxor(vector bool int a, vector float b)
+{
+  vector unsigned int res = (vector unsigned int)a ^ (vector unsigned int)b;
+  return (vector float)res;
+}
+
+static vector float __ATTRS_o_ai
+vec_vxor(vector float a, vector bool int b)
+{
+  vector unsigned int res = (vector unsigned int)a ^ (vector unsigned int)b;
+  return (vector float)res;
+}
+
 /* ------------------------------ predicates ------------------------------------ */
 
 /* vec_all_eq */
@@ -5288,36 +8390,132 @@ vec_all_eq(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_all_eq(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_eq(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpequb_p(__CR6_LT, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
+vec_all_eq(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_eq(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_eq(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_eq(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
+vec_all_eq(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector pixel a, vector pixel b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_eq(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_eq(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_eq(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_LT, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
+vec_all_eq(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_eq(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_eq(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpeqfp_p(__CR6_LT, a, b);
@@ -5332,36 +8530,132 @@ vec_all_ge(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_all_ge(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_EQ, (vector signed char)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_all_ge(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_EQ, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_ge(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)b, 
+                                                (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, b, (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)b, 
+                                                (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
 vec_all_ge(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_EQ, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_ge(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_EQ, (vector short)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_all_ge(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_ge(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)b, 
+                                                (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, b, (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)b, 
+                                                (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
 vec_all_ge(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_EQ, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_ge(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_EQ, (vector int)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_all_ge(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_ge(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)b, 
+                                                (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, b, (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_ge(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)b, 
+                                                (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
 vec_all_ge(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgefp_p(__CR6_LT, a, b);
@@ -5376,36 +8670,132 @@ vec_all_gt(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_all_gt(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_LT, a, (vector signed char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_gt(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_gt(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, a, (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)a, 
+                                                (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)a, 
+                                                (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_gt(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_gt(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_LT, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_gt(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_gt(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, a, (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)a, 
+                                                (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)a, 
+                                                (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_gt(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_gt(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_LT, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_gt(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_LT, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_gt(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, a, (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)a, 
+                                                (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_all_gt(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)a, 
+                                                (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_gt(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgtfp_p(__CR6_LT, a, b);
@@ -5428,36 +8818,132 @@ vec_all_le(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_all_le(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_EQ, a, (vector signed char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_le(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_le(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, a, (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)a, 
+                                                (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ, (vector unsigned char)a, 
+                                                (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_le(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_le(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_EQ, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_le(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_le(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, a, (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)a, 
+                                                (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ, (vector unsigned short)a, 
+                                                (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_le(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_le(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_EQ, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_le(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_le(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, a, (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)a, 
+                                                (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_all_le(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ, (vector unsigned int)a, 
+                                                (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_le(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgefp_p(__CR6_LT, b, a);
@@ -5472,36 +8958,132 @@ vec_all_lt(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_all_lt(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_LT, (vector signed char)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_all_lt(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_LT, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_lt(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)b, 
+                                                (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, b, (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT, (vector unsigned char)b, 
+                                                (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
 vec_all_lt(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_LT, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_lt(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_LT, (vector short)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_all_lt(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_LT, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_lt(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)b, 
+                                                (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, b, (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT, (vector unsigned short)b, 
+                                                (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
 vec_all_lt(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_LT, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_lt(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_LT, (vector int)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_all_lt(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_LT, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_all_lt(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)b, 
+                                                (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, b, (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_all_lt(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT, (vector unsigned int)b, 
+                                                (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
 vec_all_lt(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgtfp_p(__CR6_LT, b, a);
@@ -5524,36 +9106,132 @@ vec_all_ne(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_all_ne(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_ne(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpequb_p(__CR6_EQ, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
+vec_all_ne(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_ne(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_ne(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_ne(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
+vec_all_ne(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector pixel a, vector pixel b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_ne(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_EQ, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_all_ne(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_ne(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_EQ, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
+vec_all_ne(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_all_ne(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_all_ne(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpeqfp_p(__CR6_EQ, a, b);
@@ -5608,36 +9286,132 @@ vec_any_eq(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_any_eq(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_eq(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
+vec_any_eq(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_EQ_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_eq(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_eq(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_eq(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
+vec_any_eq(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector pixel a, vector pixel b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_EQ_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_eq(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_eq(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_eq(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
+vec_any_eq(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_eq(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_EQ_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_eq(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpeqfp_p(__CR6_EQ_REV, a, b);
@@ -5652,36 +9426,133 @@ vec_any_ge(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_any_ge(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_LT_REV, (vector signed char)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_any_ge(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_ge(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)b, 
+                                                    (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, b, (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)b, 
+                                                    (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
 vec_any_ge(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_LT_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_ge(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_LT_REV, (vector short)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_any_ge(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_ge(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, 
+                                                    (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool short a, vector unsigned short b)
+{
+  return 
+    __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, b, (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)b, 
+                                                    (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
 vec_any_ge(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_LT_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_ge(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_LT_REV, (vector int)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_any_ge(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_ge(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)b, 
+                                                    (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, b, (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_ge(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)b, 
+                                                    (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
 vec_any_ge(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgefp_p(__CR6_EQ_REV, a, b);
@@ -5696,36 +9567,135 @@ vec_any_gt(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_any_gt(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_EQ_REV, a, (vector signed char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_gt(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_gt(vector unsigned char a, vector bool char b)
+{
+  return 
+    __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, a, (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)a,
+                                                    (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool char a, vector unsigned char b)
+{
+  return 
+    __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)a,
+                                                    (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_gt(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_gt(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_EQ_REV, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_gt(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_gt(vector unsigned short a, vector bool short b)
+{
+  return 
+    __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, a, (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a,
+                                                    (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)a, 
+                                                    (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_gt(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_gt(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_EQ_REV, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_gt(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_gt(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, a, (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)a,
+                                                    (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_any_gt(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)a,
+                                                    (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_gt(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgtfp_p(__CR6_EQ_REV, a, b);
@@ -5740,36 +9710,136 @@ vec_any_le(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_any_le(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_LT_REV, a, (vector signed char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_le(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_le(vector unsigned char a, vector bool char b)
+{
+  return 
+    __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, a, (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)a,
+                                                    (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool char a, vector unsigned char b)
+{
+  return 
+    __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_LT_REV, (vector unsigned char)a,
+                                                    (vector unsigned char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_le(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_le(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_LT_REV, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_le(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_le(vector unsigned short a, vector bool short b)
+{
+  return 
+    __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, a, (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)a,
+                                                    (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool short a, vector unsigned short b)
+{
+  return 
+    __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_LT_REV, (vector unsigned short)a,
+                                                    (vector unsigned short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_le(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_le(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_LT_REV, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_le(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_le(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, a, (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)a,
+                                                    (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)a, b);
+}
+
+static int __ATTRS_o_ai
+vec_any_le(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_LT_REV, (vector unsigned int)a,
+                                                    (vector unsigned int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_le(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgefp_p(__CR6_EQ_REV, b, a);
@@ -5784,36 +9854,136 @@ vec_any_lt(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_any_lt(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtsb_p(__CR6_EQ_REV, (vector signed char)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_any_lt(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_lt(vector unsigned char a, vector bool char b)
+{
+  return 
+    __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)b,
+                                                    (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool char a, vector unsigned char b)
+{
+  return 
+    __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, b, (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpgtub_p(__CR6_EQ_REV, (vector unsigned char)b,
+                                                    (vector unsigned char)a);
+}
+
+static int __ATTRS_o_ai
 vec_any_lt(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpgtsh_p(__CR6_EQ_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_lt(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtsh_p(__CR6_EQ_REV, (vector short)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_any_lt(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_lt(vector unsigned short a, vector bool short b)
+{
+  return 
+    __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)b,
+                                                    (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool short a, vector unsigned short b)
+{
+  return 
+    __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, b, (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpgtuh_p(__CR6_EQ_REV, (vector unsigned short)b,
+                                                    (vector unsigned short)a);
+}
+
+static int __ATTRS_o_ai
 vec_any_lt(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpgtsw_p(__CR6_EQ_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_lt(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtsw_p(__CR6_EQ_REV, (vector int)b, a);
+}
+
+static int __ATTRS_o_ai
 vec_any_lt(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, b, a);
 }
 
 static int __ATTRS_o_ai
+vec_any_lt(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)b, a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)b,
+                                                    (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, b, (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
+vec_any_lt(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpgtuw_p(__CR6_EQ_REV, (vector unsigned int)b,
+                                                    (vector unsigned int)a);
+}
+
+static int __ATTRS_o_ai
 vec_any_lt(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpgtfp_p(__CR6_EQ_REV, b, a);
@@ -5836,36 +10006,132 @@ vec_any_ne(vector signed char a, vector signed char b)
 }
 
 static int __ATTRS_o_ai
+vec_any_ne(vector signed char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_ne(vector unsigned char a, vector unsigned char b)
 {
   return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
 }
 
 static int __ATTRS_o_ai
+vec_any_ne(vector unsigned char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool char a, vector signed char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool char a, vector unsigned char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool char a, vector bool char b)
+{
+  return __builtin_altivec_vcmpequb_p(__CR6_LT_REV, (vector char)a, (vector char)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_ne(vector short a, vector short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_ne(vector short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_ne(vector unsigned short a, vector unsigned short b)
 {
   return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
 }
 
 static int __ATTRS_o_ai
+vec_any_ne(vector unsigned short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool short a, vector short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool short a, vector unsigned short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool short a, vector bool short b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector pixel a, vector pixel b)
+{
+  return __builtin_altivec_vcmpequh_p(__CR6_LT_REV, (vector short)a, (vector short)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_ne(vector int a, vector int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, a, b);
 }
 
 static int __ATTRS_o_ai
+vec_any_ne(vector int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_ne(vector unsigned int a, vector unsigned int b)
 {
   return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
 }
 
 static int __ATTRS_o_ai
+vec_any_ne(vector unsigned int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool int a, vector int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool int a, vector unsigned int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
+vec_any_ne(vector bool int a, vector bool int b)
+{
+  return __builtin_altivec_vcmpequw_p(__CR6_LT_REV, (vector int)a, (vector int)b);
+}
+
+static int __ATTRS_o_ai
 vec_any_ne(vector float a, vector float b)
 {
   return __builtin_altivec_vcmpeqfp_p(__CR6_LT_REV, a, b);
diff --git a/lib/Headers/avxintrin.h b/lib/Headers/avxintrin.h
new file mode 100644
index 000000000000..884d31cb8917
--- /dev/null
+++ b/lib/Headers/avxintrin.h
@@ -0,0 +1,1156 @@
+/*===---- avxintrin.h - AVX intrinsics -------------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <avxintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+typedef double __v4df __attribute__ ((__vector_size__ (32)));
+typedef float __v8sf __attribute__ ((__vector_size__ (32)));
+typedef long long __v4di __attribute__ ((__vector_size__ (32)));
+typedef int __v8si __attribute__ ((__vector_size__ (32)));
+typedef short __v16hi __attribute__ ((__vector_size__ (32)));
+typedef char __v32qi __attribute__ ((__vector_size__ (32)));
+
+typedef float __m256 __attribute__ ((__vector_size__ (32)));
+typedef double __m256d __attribute__((__vector_size__(32)));
+typedef long long __m256i __attribute__((__vector_size__(32)));
+
+/* Arithmetic */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_add_pd(__m256d a, __m256d b)
+{
+  return a+b;
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_add_ps(__m256 a, __m256 b)
+{
+  return a+b;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_sub_pd(__m256d a, __m256d b)
+{
+  return a-b;
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_sub_ps(__m256 a, __m256 b)
+{
+  return a-b;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_addsub_pd(__m256d a, __m256d b)
+{
+  return (__m256d)__builtin_ia32_addsubpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_addsub_ps(__m256 a, __m256 b)
+{
+  return (__m256)__builtin_ia32_addsubps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_div_pd(__m256d a, __m256d b)
+{
+  return a / b;
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_div_ps(__m256 a, __m256 b)
+{
+  return a / b;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_max_pd(__m256d a, __m256d b)
+{
+  return (__m256d)__builtin_ia32_maxpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_max_ps(__m256 a, __m256 b)
+{
+  return (__m256)__builtin_ia32_maxps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_min_pd(__m256d a, __m256d b)
+{
+  return (__m256d)__builtin_ia32_minpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_min_ps(__m256 a, __m256 b)
+{
+  return (__m256)__builtin_ia32_minps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_mul_pd(__m256d a, __m256d b)
+{
+  return a * b;
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_mul_ps(__m256 a, __m256 b)
+{
+  return a * b;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_sqrt_pd(__m256d a)
+{
+  return (__m256d)__builtin_ia32_sqrtpd256((__v4df)a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_sqrt_ps(__m256 a)
+{
+  return (__m256)__builtin_ia32_sqrtps256((__v8sf)a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_rsqrt_ps(__m256 a)
+{
+  return (__m256)__builtin_ia32_rsqrtps256((__v8sf)a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_rcp_ps(__m256 a)
+{
+  return (__m256)__builtin_ia32_rcpps256((__v8sf)a);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_round_pd(__m256d v, const int m)
+{
+  return (__m256d)__builtin_ia32_roundpd256((__v4df)v, m);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_round_ps(__m256 v, const int m)
+{
+  return (__m256)__builtin_ia32_roundps256((__v8sf)v, m);
+}
+
+#define _mm256_ceil_pd(V)  _mm256_round_pd((V), _MM_FROUND_CEIL)
+#define _mm256_floor_pd(V) _mm256_round_pd((V), _MM_FROUND_FLOOR)
+#define _mm256_ceil_ps(V)  _mm256_round_ps((V), _MM_FROUND_CEIL)
+#define _mm256_floor_ps(V) _mm256_round_ps((V), _MM_FROUND_FLOOR)
+
+/* Logical */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_and_pd(__m256d a, __m256d b)
+{
+  return (__m256d)((__v4di)a & (__v4di)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_and_ps(__m256 a, __m256 b)
+{
+  return (__m256)((__v8si)a & (__v8si)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_andnot_pd(__m256d a, __m256d b)
+{
+  return (__m256d)(~(__v4di)a & (__v4di)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_andnot_ps(__m256 a, __m256 b)
+{
+  return (__m256)(~(__v8si)a & (__v8si)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_or_pd(__m256d a, __m256d b)
+{
+  return (__m256d)((__v4di)a | (__v4di)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_or_ps(__m256 a, __m256 b)
+{
+  return (__m256)((__v8si)a | (__v8si)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_xor_pd(__m256d a, __m256d b)
+{
+  return (__m256d)((__v4di)a ^ (__v4di)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_xor_ps(__m256 a, __m256 b)
+{
+  return (__m256)((__v8si)a ^ (__v8si)b);
+}
+
+/* Horizontal arithmetic */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_hadd_pd(__m256d a, __m256d b)
+{
+  return (__m256d)__builtin_ia32_haddpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_hadd_ps(__m256 a, __m256 b)
+{
+  return (__m256)__builtin_ia32_haddps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_hsub_pd(__m256d a, __m256d b)
+{
+  return (__m256d)__builtin_ia32_hsubpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_hsub_ps(__m256 a, __m256 b)
+{
+  return (__m256)__builtin_ia32_hsubps256((__v8sf)a, (__v8sf)b);
+}
+
+/* Vector permutations */
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm_permutevar_pd(__m128d a, __m128i c)
+{
+  return (__m128d)__builtin_ia32_vpermilvarpd((__v2df)a, (__v2di)c);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_permutevar_pd(__m256d a, __m256i c)
+{
+  return (__m256d)__builtin_ia32_vpermilvarpd256((__v4df)a, (__v4di)c);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_permutevar_ps(__m128 a, __m128i c)
+{
+  return (__m128)__builtin_ia32_vpermilvarps((__v4sf)a, (__v4si)c);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_permutevar_ps(__m256 a, __m256i c)
+{
+  return (__m256)__builtin_ia32_vpermilvarps256((__v8sf)a,
+						  (__v8si)c);
+}
+
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm_permute_pd(__m128d a, const int c)
+{
+  return (__m128d)__builtin_ia32_vpermilpd((__v2df)a, c);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_permute_pd(__m256d a, const int c)
+{
+  return (__m256d)__builtin_ia32_vpermilpd256((__v4df)a, c);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_permute_ps(__m128 a, const int c)
+{
+  return (__m128)__builtin_ia32_vpermilps((__v4sf)a, c);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_permute_ps(__m256 a, const int c)
+{
+  return (__m256)__builtin_ia32_vpermilps256((__v8sf)a, c);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_permute2f128_pd(__m256d a, __m256d b, const int c)
+{
+  return (__m256d)__builtin_ia32_vperm2f128_pd256((__v4df)a, (__v4df)b, c);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_permute2f128_ps(__m256 a, __m256 b, const int c)
+{
+  return (__m256)__builtin_ia32_vperm2f128_ps256((__v8sf)a, (__v8sf)b, c);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_permute2f128_si256(__m256i a, __m256i b, const int c)
+{
+  return (__m256i)__builtin_ia32_vperm2f128_si256((__v8si)a, (__v8si)b, c);
+}
+
+/* Vector Blend */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_blend_pd(__m256d a, __m256d b, const int c)
+{
+  return (__m256d)__builtin_ia32_blendpd256((__v4df)a, (__v4df)b, c);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_blend_ps(__m256 a, __m256 b, const int c)
+{
+  return (__m256)__builtin_ia32_blendps256((__v8sf)a, (__v8sf)b, c);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_blendv_pd(__m256d a, __m256d b, __m256d c)
+{
+  return (__m256d)__builtin_ia32_blendvpd256((__v4df)a, (__v4df)b, (__v4df)c);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_blendv_ps(__m256 a, __m256 b, __m256 c)
+{
+  return (__m256)__builtin_ia32_blendvps256((__v8sf)a, (__v8sf)b, (__v8sf)c);
+}
+
+/* Vector Dot Product */
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_dp_ps(__m256 a, __m256 b, const int c)
+{
+  return (__m256)__builtin_ia32_dpps256((__v8sf)a, (__v8sf)b, c);
+}
+
+/* Vector shuffle */
+#define _mm256_shuffle_ps(a, b, mask) \
+        (__builtin_shufflevector((__v8sf)(a), (__v8sf)(b), \
+        (mask) & 0x3,                ((mask) & 0xc) >> 2, \
+        (((mask) & 0x30) >> 4) + 8,  (((mask) & 0xc0) >> 6) + 8, \
+        (mask) & 0x3 + 4,            (((mask) & 0xc) >> 2) + 4, \
+        (((mask) & 0x30) >> 4) + 12, (((mask) & 0xc0) >> 6) + 12))
+
+#define _mm256_shuffle_pd(a, b, mask) \
+        (__builtin_shufflevector((__v4df)(a), (__v4df)(b), \
+        (mask) & 0x1, \
+        (((mask) & 0x2) >> 1) + 4, \
+        (((mask) & 0x4) >> 2) + 2, \
+        (((mask) & 0x8) >> 3) + 6))
+
+/* Compare */
+#define _CMP_EQ_OQ    0x00 /* Equal (ordered, non-signaling)  */
+#define _CMP_LT_OS    0x01 /* Less-than (ordered, signaling)  */
+#define _CMP_LE_OS    0x02 /* Less-than-or-equal (ordered, signaling)  */
+#define _CMP_UNORD_Q  0x03 /* Unordered (non-signaling)  */
+#define _CMP_NEQ_UQ   0x04 /* Not-equal (unordered, non-signaling)  */
+#define _CMP_NLT_US   0x05 /* Not-less-than (unordered, signaling)  */
+#define _CMP_NLE_US   0x06 /* Not-less-than-or-equal (unordered, signaling)  */
+#define _CMP_ORD_Q    0x07 /* Ordered (nonsignaling)   */
+#define _CMP_EQ_UQ    0x08 /* Equal (unordered, non-signaling)  */
+#define _CMP_NGE_US   0x09 /* Not-greater-than-or-equal (unord, signaling)  */
+#define _CMP_NGT_US   0x0a /* Not-greater-than (unordered, signaling)  */
+#define _CMP_FALSE_OQ 0x0b /* False (ordered, non-signaling)  */
+#define _CMP_NEQ_OQ   0x0c /* Not-equal (ordered, non-signaling)  */
+#define _CMP_GE_OS    0x0d /* Greater-than-or-equal (ordered, signaling)  */
+#define _CMP_GT_OS    0x0e /* Greater-than (ordered, signaling)  */
+#define _CMP_TRUE_UQ  0x0f /* True (unordered, non-signaling)  */
+#define _CMP_EQ_OS    0x10 /* Equal (ordered, signaling)  */
+#define _CMP_LT_OQ    0x11 /* Less-than (ordered, non-signaling)  */
+#define _CMP_LE_OQ    0x12 /* Less-than-or-equal (ordered, non-signaling)  */
+#define _CMP_UNORD_S  0x13 /* Unordered (signaling)  */
+#define _CMP_NEQ_US   0x14 /* Not-equal (unordered, signaling)  */
+#define _CMP_NLT_UQ   0x15 /* Not-less-than (unordered, non-signaling)  */
+#define _CMP_NLE_UQ   0x16 /* Not-less-than-or-equal (unord, non-signaling)  */
+#define _CMP_ORD_S    0x17 /* Ordered (signaling)  */
+#define _CMP_EQ_US    0x18 /* Equal (unordered, signaling)  */
+#define _CMP_NGE_UQ   0x19 /* Not-greater-than-or-equal (unord, non-sign)  */
+#define _CMP_NGT_UQ   0x1a /* Not-greater-than (unordered, non-signaling)  */
+#define _CMP_FALSE_OS 0x1b /* False (ordered, signaling)  */
+#define _CMP_NEQ_OS   0x1c /* Not-equal (ordered, signaling)  */
+#define _CMP_GE_OQ    0x1d /* Greater-than-or-equal (ordered, non-signaling)  */
+#define _CMP_GT_OQ    0x1e /* Greater-than (ordered, non-signaling)  */
+#define _CMP_TRUE_US  0x1f /* True (unordered, signaling)  */
+
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm_cmp_pd(__m128d a, __m128d b, const int c)
+{
+  return (__m128d)__builtin_ia32_cmppd((__v2df)a, (__v2df)b, c);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_cmp_ps(__m128 a, __m128 b, const int c)
+{
+  return (__m128)__builtin_ia32_cmpps((__v4sf)a, (__v4sf)b, c);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_cmp_pd(__m256d a, __m256d b, const int c)
+{
+  return (__m256d)__builtin_ia32_cmppd256((__v4df)a, (__v4df)b, c);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_cmp_ps(__m256 a, __m256 b, const int c)
+{
+  return (__m256)__builtin_ia32_cmpps256((__v8sf)a, (__v8sf)b, c);
+}
+
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm_cmp_sd(__m128d a, __m128d b, const int c)
+{
+  return (__m128d)__builtin_ia32_cmpsd((__v2df)a, (__v2df)b, c);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_cmp_ss(__m128 a, __m128 b, const int c)
+{
+  return (__m128)__builtin_ia32_cmpss((__v4sf)a, (__v4sf)b, c);
+}
+
+/* Vector extract */
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm256_extractf128_pd(__m256d a, const int o)
+{
+  return (__m128d)__builtin_ia32_vextractf128_pd256((__v4df)a, o);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm256_extractf128_ps(__m256 a, const int o)
+{
+  return (__m128)__builtin_ia32_vextractf128_ps256((__v8sf)a, o);
+}
+
+static __inline __m128i __attribute__((__always_inline__, __nodebug__))
+_mm256_extractf128_si256(__m256i a, const int o)
+{
+  return (__m128i)__builtin_ia32_vextractf128_si256((__v8si)a, o);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_extract_epi32(__m256i a, int const imm)
+{
+  __v8si b = (__v8si)a;
+  return b[imm];
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_extract_epi16(__m256i a, int const imm)
+{
+  __v16hi b = (__v16hi)a;
+  return b[imm];
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_extract_epi8(__m256i a, int const imm)
+{
+  __v32qi b = (__v32qi)a;
+  return b[imm];
+}
+
+#ifdef __x86_64__
+static __inline long long  __attribute__((__always_inline__, __nodebug__))
+_mm256_extract_epi64(__m256i a, const int imm)
+{
+  __v4di b = (__v4di)a;
+  return b[imm];
+}
+#endif
+
+/* Vector insert */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_insertf128_pd(__m256d a, __m128d b, const int o)
+{
+  return (__m256d)__builtin_ia32_vinsertf128_pd256((__v4df)a, (__v2df)b, o);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_insertf128_ps(__m256 a, __m128 b, const int o)
+{
+  return (__m256)__builtin_ia32_vinsertf128_ps256((__v8sf)a, (__v4sf)b, o);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_insertf128_si256(__m256i a, __m128i b, const int o)
+{
+  return (__m256i)__builtin_ia32_vinsertf128_si256((__v8si)a, (__v4si)b, o);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_insert_epi32(__m256i a, int b, int const imm)
+{
+  __v8si c = (__v8si)a;
+  c[imm & 7] = b;
+  return (__m256i)c;
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_insert_epi16(__m256i a, int b, int const imm)
+{
+  __v16hi c = (__v16hi)a;
+  c[imm & 15] = b;
+  return (__m256i)c;
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_insert_epi8(__m256i a, int b, int const imm)
+{
+  __v32qi c = (__v32qi)a;
+  c[imm & 31] = b;
+  return (__m256i)c;
+}
+
+#ifdef __x86_64__
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_insert_epi64(__m256i a, int b, int const imm)
+{
+  __v4di c = (__v4di)a;
+  c[imm & 3] = b;
+  return (__m256i)c;
+}
+#endif
+
+/* Conversion */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtepi32_pd(__m128i a)
+{
+  return (__m256d)__builtin_ia32_cvtdq2pd256((__v4si) a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtepi32_ps(__m256i a)
+{
+  return (__m256)__builtin_ia32_cvtdq2ps256((__v8si) a);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtpd_ps(__m256d a)
+{
+  return (__m128)__builtin_ia32_cvtpd2ps256((__v4df) a);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtps_epi32(__m256 a)
+{
+  return (__m256i)__builtin_ia32_cvtps2dq256((__v8sf) a);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtps_pd(__m128 a)
+{
+  return (__m256d)__builtin_ia32_cvtps2pd256((__v4sf) a);
+}
+
+static __inline __m128i __attribute__((__always_inline__, __nodebug__))
+_mm256_cvttpd_epi32(__m256d a)
+{
+  return (__m128i)__builtin_ia32_cvttpd2dq256((__v4df) a);
+}
+
+static __inline __m128i __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtpd_epi32(__m256d a)
+{
+  return (__m128i)__builtin_ia32_cvtpd2dq256((__v4df) a);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_cvttps_epi32(__m256 a)
+{
+  return (__m256i)__builtin_ia32_cvttps2dq256((__v8sf) a);
+}
+
+/* Vector replicate */
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_movehdup_ps(__m256 a)
+{
+  return __builtin_shufflevector(a, a, 1, 1, 3, 3, 5, 5, 7, 7);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_moveldup_ps(__m256 a)
+{
+  return __builtin_shufflevector(a, a, 0, 0, 2, 2, 4, 4, 6, 6);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_movedup_pd(__m256d a)
+{
+  return __builtin_shufflevector(a, a, 0, 0, 2, 2);
+}
+
+/* Unpack and Interleave */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_unpackhi_pd(__m256d a, __m256d b)
+{
+  return __builtin_shufflevector(a, b, 1, 5, 1+2, 5+2);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_unpacklo_pd(__m256d a, __m256d b)
+{
+  return __builtin_shufflevector(a, b, 0, 4, 0+2, 4+2);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_unpackhi_ps(__m256 a, __m256 b)
+{
+  return __builtin_shufflevector(a, b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_unpacklo_ps(__m256 a, __m256 b)
+{
+  return __builtin_shufflevector(a, b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1);
+}
+
+/* Bit Test */
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm_testz_pd(__m128d a, __m128d b)
+{
+  return __builtin_ia32_vtestzpd((__v2df)a, (__v2df)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm_testc_pd(__m128d a, __m128d b)
+{
+  return __builtin_ia32_vtestcpd((__v2df)a, (__v2df)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm_testnzc_pd(__m128d a, __m128d b)
+{
+  return __builtin_ia32_vtestnzcpd((__v2df)a, (__v2df)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm_testz_ps(__m128 a, __m128 b)
+{
+  return __builtin_ia32_vtestzps((__v4sf)a, (__v4sf)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm_testc_ps(__m128 a, __m128 b)
+{
+  return __builtin_ia32_vtestcps((__v4sf)a, (__v4sf)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm_testnzc_ps(__m128 a, __m128 b)
+{
+  return __builtin_ia32_vtestnzcps((__v4sf)a, (__v4sf)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testz_pd(__m256d a, __m256d b)
+{
+  return __builtin_ia32_vtestzpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testc_pd(__m256d a, __m256d b)
+{
+  return __builtin_ia32_vtestcpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testnzc_pd(__m256d a, __m256d b)
+{
+  return __builtin_ia32_vtestnzcpd256((__v4df)a, (__v4df)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testz_ps(__m256 a, __m256 b)
+{
+  return __builtin_ia32_vtestzps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testc_ps(__m256 a, __m256 b)
+{
+  return __builtin_ia32_vtestcps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testnzc_ps(__m256 a, __m256 b)
+{
+  return __builtin_ia32_vtestnzcps256((__v8sf)a, (__v8sf)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testz_si256(__m256i a, __m256i b)
+{
+  return __builtin_ia32_ptestz256((__v4di)a, (__v4di)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testc_si256(__m256i a, __m256i b)
+{
+  return __builtin_ia32_ptestc256((__v4di)a, (__v4di)b);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_testnzc_si256(__m256i a, __m256i b)
+{
+  return __builtin_ia32_ptestnzc256((__v4di)a, (__v4di)b);
+}
+
+/* Vector extract sign mask */
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_movemask_pd(__m256d a)
+{
+  return __builtin_ia32_movmskpd256((__v4df)a);
+}
+
+static __inline int __attribute__((__always_inline__, __nodebug__))
+_mm256_movemask_ps(__m256 a)
+{
+  return __builtin_ia32_movmskps256((__v8sf)a);
+}
+
+/* Vector zero */
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_zeroall(void)
+{
+  __builtin_ia32_vzeroall();
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_zeroupper(void)
+{
+  __builtin_ia32_vzeroupper();
+}
+
+/* Vector load with broadcast */
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_broadcast_ss(float const *a)
+{
+  return (__m128)__builtin_ia32_vbroadcastss(a);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_broadcast_sd(double const *a)
+{
+  return (__m256d)__builtin_ia32_vbroadcastsd256(a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_broadcast_ss(float const *a)
+{
+  return (__m256)__builtin_ia32_vbroadcastss256(a);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_broadcast_pd(__m128d const *a)
+{
+  return (__m256d)__builtin_ia32_vbroadcastf128_pd256(a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_broadcast_ps(__m128 const *a)
+{
+  return (__m256)__builtin_ia32_vbroadcastf128_ps256(a);
+}
+
+/* SIMD load ops */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_load_pd(double const *p)
+{
+  return *(__m256d *)p;
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_load_ps(float const *p)
+{
+  return *(__m256 *)p;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_loadu_pd(double const *p)
+{
+  return (__m256d)__builtin_ia32_loadupd256(p);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_loadu_ps(float const *p)
+{
+  return (__m256)__builtin_ia32_loadups256(p);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_load_si256(__m256i const *p)
+{
+  return *p;
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_loadu_si256(__m256i const *p)
+{
+  return (__m256i)__builtin_ia32_loaddqu256((char const *)p);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_lddqu_si256(__m256i const *p)
+{
+  return (__m256i)__builtin_ia32_lddqu256((char const *)p);
+}
+
+/* SIMD store ops */
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_store_pd(double *p, __m256d a)
+{
+  *(__m256d *)p = a;
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_store_ps(float *p, __m256 a)
+{
+  *(__m256 *)p = a;
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_storeu_pd(double *p, __m256d a)
+{
+  __builtin_ia32_storeupd256(p, (__v4df)a);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_storeu_ps(float *p, __m256 a)
+{
+  __builtin_ia32_storeups256(p, (__v8sf)a);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_store_si256(__m256i *p, __m256i a)
+{
+  *p = a;
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_storeu_si256(__m256i *p, __m256i a)
+{
+  __builtin_ia32_storedqu256((char *)p, (__v32qi)a);
+}
+
+/* Conditional load ops */
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm_maskload_pd(double const *p, __m128d m)
+{
+  return (__m128d)__builtin_ia32_maskloadpd((const __v2df *)p, (__v2df)m);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_maskload_pd(double const *p, __m256d m)
+{
+  return (__m256d)__builtin_ia32_maskloadpd256((const __v4df *)p, (__v4df)m);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_maskload_ps(float const *p, __m128 m)
+{
+  return (__m128)__builtin_ia32_maskloadps((const __v4sf *)p, (__v4sf)m);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_maskload_ps(float const *p, __m256 m)
+{
+  return (__m256)__builtin_ia32_maskloadps256((const __v8sf *)p, (__v8sf)m);
+}
+
+/* Conditional store ops */
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_maskstore_ps(float *p, __m256 m, __m256 a)
+{
+  __builtin_ia32_maskstoreps256((__v8sf *)p, (__v8sf)m, (__v8sf)a);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm_maskstore_pd(double *p, __m128d m, __m128d a)
+{
+  __builtin_ia32_maskstorepd((__v2df *)p, (__v2df)m, (__v2df)a);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_maskstore_pd(double *p, __m256d m, __m256d a)
+{
+  __builtin_ia32_maskstorepd256((__v4df *)p, (__v4df)m, (__v4df)a);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm_maskstore_ps(float *p, __m128 m, __m128 a)
+{
+  __builtin_ia32_maskstoreps((__v4sf *)p, (__v4sf)m, (__v4sf)a);
+}
+
+/* Cacheability support ops */
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_stream_si256(__m256i *a, __m256i b)
+{
+  __builtin_ia32_movntdq256((__v4di *)a, (__v4di)b);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_stream_pd(double *a, __m256d b)
+{
+  __builtin_ia32_movntpd256(a, (__v4df)b);
+}
+
+static __inline void __attribute__((__always_inline__, __nodebug__))
+_mm256_stream_ps(float *p, __m256 a)
+{
+  __builtin_ia32_movntps256(p, (__v8sf)a);
+}
+
+/* Create vectors */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_set_pd(double a, double b, double c, double d)
+{
+  return (__m256d){ d, c, b, a };
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_set_ps(float a, float b, float c, float d,
+	            float e, float f, float g, float h)
+{
+  return (__m256){ h, g, f, e, d, c, b, a };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set_epi32(int i0, int i1, int i2, int i3,
+		             int i4, int i5, int i6, int i7)
+{
+  return (__m256i)(__v8si){ i7, i6, i5, i4, i3, i2, i1, i0 };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set_epi16(short w15, short w14, short w13, short w12,
+		             short w11, short w10, short w09, short w08,
+		             short w07, short w06, short w05, short w04,
+		             short w03, short w02, short w01, short w00)
+{
+  return (__m256i)(__v16hi){ w00, w01, w02, w03, w04, w05, w06, w07,
+                             w08, w09, w10, w11, w12, w13, w14, w15 };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set_epi8(char b31, char b30, char b29, char b28,
+		            char b27, char b26, char b25, char b24,
+		            char b23, char b22, char b21, char b20,
+		            char b19, char b18, char b17, char b16,
+		            char b15, char b14, char b13, char b12,
+		            char b11, char b10, char b09, char b08,
+		            char b07, char b06, char b05, char b04,
+		            char b03, char b02, char b01, char b00)
+{
+  return (__m256i)(__v32qi){
+    b00, b01, b02, b03, b04, b05, b06, b07,
+    b08, b09, b10, b11, b12, b13, b14, b15,
+    b16, b17, b18, b19, b20, b21, b22, b23,
+    b24, b25, b26, b27, b28, b29, b30, b31
+  };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set_epi64x(long long a, long long b, long long c, long long d)
+{
+  return (__m256i)(__v4di){ d, c, b, a };
+}
+
+/* Create vectors with elements in reverse order */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_setr_pd(double a, double b, double c, double d)
+{
+  return (__m256d){ a, b, c, d };
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_setr_ps(float a, float b, float c, float d,
+		           float e, float f, float g, float h)
+{
+  return (__m256){ a, b, c, d, e, f, g, h };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_setr_epi32(int i0, int i1, int i2, int i3,
+		              int i4, int i5, int i6, int i7)
+{
+  return (__m256i)(__v8si){ i0, i1, i2, i3, i4, i5, i6, i7 };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_setr_epi16(short w15, short w14, short w13, short w12,
+		   short w11, short w10, short w09, short w08,
+		   short w07, short w06, short w05, short w04,
+		   short w03, short w02, short w01, short w00)
+{
+  return (__m256i)(__v16hi){ w15, w14, w13, w12, w11, w10, w09, w08,
+			                       w07, w06, w05, w04, w03, w02, w01, w00 };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_setr_epi8(char b31, char b30, char b29, char b28,
+		             char b27, char b26, char b25, char b24,
+		             char b23, char b22, char b21, char b20,
+		             char b19, char b18, char b17, char b16,
+		             char b15, char b14, char b13, char b12,
+		             char b11, char b10, char b09, char b08,
+		             char b07, char b06, char b05, char b04,
+		             char b03, char b02, char b01, char b00)
+{
+  return (__m256i)(__v32qi){
+    b31, b30, b29, b28, b27, b26, b25, b24,
+		b23, b22, b21, b20, b19, b18, b17, b16,
+		b15, b14, b13, b12, b11, b10, b09, b08,
+		b07, b06, b05, b04, b03, b02, b01, b00 };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_setr_epi64x(long long a, long long b, long long c, long long d)
+{
+  return (__m256i)(__v4di){ a, b, c, d };
+}
+
+/* Create vectors with repeated elements */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_set1_pd(double w)
+{
+  return (__m256d){ w, w, w, w };
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_set1_ps(float w)
+{
+  return (__m256){ w, w, w, w, w, w, w, w };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set1_epi32(int i)
+{
+  return (__m256i)(__v8si){ i, i, i, i, i, i, i, i };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set1_epi16(short w)
+{
+  return (__m256i)(__v16hi){ w, w, w, w, w, w, w, w, w, w, w, w, w, w, w, w };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set1_epi8(char b)
+{
+  return (__m256i)(__v32qi){ b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, b,
+                             b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, b };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_set1_epi64x(long long q)
+{
+  return (__m256i)(__v4di){ q, q, q, q };
+}
+
+/* Create zeroed vectors */
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_setzero_pd(void)
+{
+  return (__m256d){ 0, 0, 0, 0 };
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_setzero_ps(void)
+{
+  return (__m256){ 0, 0, 0, 0, 0, 0, 0, 0 };
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_setzero_si256(void)
+{
+  return (__m256i){ 0LL, 0LL, 0LL, 0LL };
+}
+
+/* Cast between vector types */
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_castpd_ps(__m256d in)
+{
+  return (__m256)in;
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_castpd_si256(__m256d in)
+{
+  return (__m256i)in;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_castps_pd(__m256 in)
+{
+  return (__m256d)in;
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_castps_si256(__m256 in)
+{
+  return (__m256i)in;
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_castsi256_ps(__m256i in)
+{
+  return (__m256)in;
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_castsi256_pd(__m256i in)
+{
+  return (__m256d)in;
+}
+
+static __inline __m128d __attribute__((__always_inline__, __nodebug__))
+_mm256_castpd256_pd128(__m256d in)
+{
+  return __builtin_shufflevector(in, in, 0, 1);
+}
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm256_castps256_ps128(__m256 in)
+{
+  return __builtin_shufflevector(in, in, 0, 1, 2, 3);
+}
+
+static __inline __m128i __attribute__((__always_inline__, __nodebug__))
+_mm256_castsi256_si128(__m256i in)
+{
+  return __builtin_shufflevector(in, in, 0, 1);
+}
+
+static __inline __m256d __attribute__((__always_inline__, __nodebug__))
+_mm256_castpd128_pd256(__m128d in)
+{
+  __m128d zero = _mm_setzero_pd();
+  return __builtin_shufflevector(in, zero, 0, 1, 2, 2);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_castps128_ps256(__m128 in)
+{
+  __m128 zero = _mm_setzero_ps();
+  return __builtin_shufflevector(in, zero, 0, 1, 2, 3, 4, 4, 4, 4);
+}
+
+static __inline __m256i __attribute__((__always_inline__, __nodebug__))
+_mm256_castsi128_si256(__m128i in)
+{
+  __m128i zero = _mm_setzero_si128();
+  return __builtin_shufflevector(in, zero, 0, 1, 2, 2);
+}
diff --git a/lib/Headers/emmintrin.h b/lib/Headers/emmintrin.h
index f297f36b549c..e5dfe26b49a6 100644
--- a/lib/Headers/emmintrin.h
+++ b/lib/Headers/emmintrin.h
@@ -1,4 +1,4 @@
-/*===---- xmmintrin.h - SSE intrinsics -------------------------------------===
+/*===---- emmintrin.h - SSE2 intrinsics ------------------------------------===
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -20,7 +20,7 @@
  *
  *===-----------------------------------------------------------------------===
  */
- 
+
 #ifndef __EMMINTRIN_H
 #define __EMMINTRIN_H
 
@@ -33,6 +33,9 @@
 typedef double __m128d __attribute__((__vector_size__(16)));
 typedef long long __m128i __attribute__((__vector_size__(16)));
 
+/* Type defines.  */
+typedef double __v2df __attribute__ ((__vector_size__ (16)));
+typedef long long __v2di __attribute__ ((__vector_size__ (16)));
 typedef short __v8hi __attribute__((__vector_size__(16)));
 typedef char __v16qi __attribute__((__vector_size__(16)));
 
@@ -1194,7 +1197,7 @@ static __inline__ int __attribute__((__always_inline__, __nodebug__))
 _mm_extract_epi16(__m128i a, int imm)
 {
   __v8hi b = (__v8hi)a;
-  return b[imm];
+  return (unsigned short)b[imm];
 }
 
 static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
diff --git a/lib/Headers/immintrin.h b/lib/Headers/immintrin.h
new file mode 100644
index 000000000000..a19deaac6db9
--- /dev/null
+++ b/lib/Headers/immintrin.h
@@ -0,0 +1,59 @@
+/*===---- immintrin.h - Intel intrinsics -----------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#define __IMMINTRIN_H
+
+#ifdef __MMX__
+#include <mmintrin.h>
+#endif
+
+#ifdef __SSE__
+#include <xmmintrin.h>
+#endif
+
+#ifdef __SSE2__
+#include <emmintrin.h>
+#endif
+
+#ifdef __SSE3__
+#include <pmmintrin.h>
+#endif
+
+#ifdef __SSSE3__
+#include <tmmintrin.h>
+#endif
+
+#if defined (__SSE4_2__) || defined (__SSE4_1__)
+#include <smmintrin.h>
+#endif
+
+#if defined (__AES__) || defined (__PCLMUL__)
+#include <wmmintrin.h>
+#endif
+
+#ifdef __AVX__
+#include <avxintrin.h>
+#endif
+
+#endif /* __IMMINTRIN_H */
diff --git a/lib/Headers/mmintrin.h b/lib/Headers/mmintrin.h
index 401d8a7aaede..bad9e1c059dc 100644
--- a/lib/Headers/mmintrin.h
+++ b/lib/Headers/mmintrin.h
@@ -443,6 +443,64 @@ _mm_setr_pi8(char __b7, char __b6, char __b5, char __b4, char __b3, char __b2,
     return (__m64)(__v8qi){ __b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0 };
 }
 
+
+/* Aliases for compatibility. */
+#define _m_empty _mm_empty
+#define _m_from_int _mm_cvtsi32_si64
+#define _m_to_int _mm_cvtsi64_si32
+#define _m_packsswb _mm_packs_pi16
+#define _m_packssdw _mm_packs_pi32
+#define _m_packuswb _mm_packs_pu16
+#define _m_punpckhbw _mm_unpackhi_pi8
+#define _m_punpckhwd _mm_unpackhi_pi16
+#define _m_punpckhdq _mm_unpackhi_pi32
+#define _m_punpcklbw _mm_unpacklo_pi8
+#define _m_punpcklwd _mm_unpacklo_pi16
+#define _m_punpckldq _mm_unpacklo_pi32
+#define _m_paddb _mm_add_pi8
+#define _m_paddw _mm_add_pi16
+#define _m_paddd _mm_add_pi32
+#define _m_paddsb _mm_adds_pi8
+#define _m_paddsw _mm_adds_pi16
+#define _m_paddusb _mm_adds_pu8
+#define _m_paddusw _mm_adds_pu16
+#define _m_psubb _mm_sub_pi8
+#define _m_psubw _mm_sub_pi16
+#define _m_psubd _mm_sub_pi32
+#define _m_psubsb _mm_subs_pi8
+#define _m_psubsw _mm_subs_pi16
+#define _m_psubusb _mm_subs_pu8
+#define _m_psubusw _mm_subs_pu16
+#define _m_pmaddwd _mm_madd_pi16
+#define _m_pmulhw _mm_mulhi_pi16
+#define _m_pmullw _mm_mullo_pi16
+#define _m_psllw _mm_sll_pi16
+#define _m_psllwi _mm_slli_pi16
+#define _m_pslld _mm_sll_pi32
+#define _m_pslldi _mm_slli_pi32
+#define _m_psllq _mm_sll_si64
+#define _m_psllqi _mm_slli_si64
+#define _m_psraw _mm_sra_pi16
+#define _m_psrawi _mm_srai_pi16
+#define _m_psrad _mm_sra_pi32
+#define _m_psradi _mm_srai_pi32
+#define _m_psrlw _mm_srl_pi16
+#define _m_psrlwi _mm_srli_pi16
+#define _m_psrld _mm_srl_pi32
+#define _m_psrldi _mm_srli_pi32
+#define _m_psrlq _mm_srl_si64
+#define _m_psrlqi _mm_srli_si64
+#define _m_pand _mm_and_si64
+#define _m_pandn _mm_andnot_si64
+#define _m_por _mm_or_si64
+#define _m_pxor _mm_xor_si64
+#define _m_pcmpeqb _mm_cmpeq_pi8
+#define _m_pcmpeqw _mm_cmpeq_pi16
+#define _m_pcmpeqd _mm_cmpeq_pi32
+#define _m_pcmpgtb _mm_cmpgt_pi8
+#define _m_pcmpgtw _mm_cmpgt_pi16
+#define _m_pcmpgtd _mm_cmpgt_pi32
+
 #endif /* __MMX__ */
 
 #endif /* __MMINTRIN_H */
diff --git a/lib/Headers/nmmintrin.h b/lib/Headers/nmmintrin.h
index cc213ce46b6e..f12622d7be68 100644
--- a/lib/Headers/nmmintrin.h
+++ b/lib/Headers/nmmintrin.h
@@ -1,25 +1,25 @@
-/*===---- nmmintrin.h - SSE intrinsics -------------------------------------===
-*
-* Permission is hereby granted, free of charge, to any person obtaining a copy
-* of this software and associated documentation files (the "Software"), to deal
-* in the Software without restriction, including without limitation the rights
-* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-* copies of the Software, and to permit persons to whom the Software is
-* furnished to do so, subject to the following conditions:
-*
-* The above copyright notice and this permission notice shall be included in
-* all copies or substantial portions of the Software.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-* THE SOFTWARE.
-*
-*===-----------------------------------------------------------------------===
-*/
+/*===---- nmmintrin.h - SSE4 intrinsics ------------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
 
 #ifndef _NMMINTRIN_H
 #define _NMMINTRIN_H
diff --git a/lib/Headers/smmintrin.h b/lib/Headers/smmintrin.h
index 4b0d9e754183..2b8b32190620 100644
--- a/lib/Headers/smmintrin.h
+++ b/lib/Headers/smmintrin.h
@@ -30,10 +30,6 @@
 
 #include <tmmintrin.h>
 
-/* Type defines.  */
-typedef double __v2df __attribute__ ((__vector_size__ (16)));
-typedef long long __v2di __attribute__ ((__vector_size__ (16)));
-
 /* SSE4 Rounding macros. */
 #define _MM_FROUND_TO_NEAREST_INT    0x00
 #define _MM_FROUND_TO_NEG_INF        0x01
@@ -213,11 +209,13 @@ _mm_max_epu32 (__m128i __V1, __m128i __V2)
                                                     __a;}))
 #endif /* __x86_64__ */
 
-/* Extract int from packed integer array at index.  */
+/* Extract int from packed integer array at index.  This returns the element
+ * as a zero extended value, so it is unsigned.
+ */
 #define _mm_extract_epi8(X, N) (__extension__ ({ __v16qi __a = (__v16qi)(X); \
-                                                 __a[N];}))
+                                                 (unsigned char)__a[N];}))
 #define _mm_extract_epi32(X, N) (__extension__ ({ __v4si __a = (__v4si)(X); \
-                                                  __a[N];}))
+                                                  (unsigned)__a[N];}))
 #ifdef __x86_64__
 #define _mm_extract_epi64(X, N) (__extension__ ({ __v2di __a = (__v2di)(X); \
                                                   __a[N];}))
diff --git a/lib/Headers/stddef.h b/lib/Headers/stddef.h
index b1d0d528746d..84ec1a7b4e24 100644
--- a/lib/Headers/stddef.h
+++ b/lib/Headers/stddef.h
@@ -34,12 +34,13 @@ typedef __typeof__(sizeof(int)) size_t;
 #ifndef __cplusplus
 #ifndef _WCHAR_T
 #define _WCHAR_T
-typedef __typeof__(*L"") wchar_t;
+typedef __WCHAR_TYPE__ wchar_t;
 #endif
 #endif
 
 #undef NULL
 #ifdef __cplusplus
+#undef __null  // VC++ hack.
 #define NULL __null
 #else
 #define NULL ((void*)0)
diff --git a/lib/Headers/x86intrin.h b/lib/Headers/x86intrin.h
new file mode 100644
index 000000000000..e5e7a6a2b316
--- /dev/null
+++ b/lib/Headers/x86intrin.h
@@ -0,0 +1,31 @@
+/*===---- x86intrin.h - X86 intrinsics -------------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __X86INTRIN_H
+#define __X86INTRIN_H
+
+#include <immintrin.h>
+
+// FIXME: SSE4A, 3dNOW, FMA4, XOP, LWP, ABM, POPCNT
+
+#endif /* __X86INTRIN_H */
diff --git a/lib/Headers/xmmintrin.h b/lib/Headers/xmmintrin.h
index 75e06b5f9754..8363b4594b1f 100644
--- a/lib/Headers/xmmintrin.h
+++ b/lib/Headers/xmmintrin.h
@@ -416,6 +416,12 @@ _mm_cvtps_pi32(__m128 a)
   return (__m64)__builtin_ia32_cvtps2pi(a);
 }
 
+static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
+_mm_cvt_ps2pi(__m128 a)
+{
+  return _mm_cvtps_pi32(a);
+}
+
 static __inline__ int __attribute__((__always_inline__, __nodebug__))
 _mm_cvttss_si32(__m128 a)
 {
@@ -440,6 +446,12 @@ _mm_cvttps_pi32(__m128 a)
   return (__m64)__builtin_ia32_cvttps2pi(a);
 }
 
+static __inline__ __m64 __attribute__((__always_inline__, __nodebug__))
+_mm_cvtt_ps2pi(__m128 a)
+{
+  return _mm_cvttps_pi32(a);
+}
+
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_cvtsi32_ss(__m128 a, int b)
 {
@@ -447,6 +459,12 @@ _mm_cvtsi32_ss(__m128 a, int b)
   return a;
 }
 
+static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_cvt_si2ss(__m128 a, int b)
+{
+  return _mm_cvtsi32_ss(a, b);
+}
+
 #ifdef __x86_64__
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
@@ -464,6 +482,12 @@ _mm_cvtpi32_ps(__m128 a, __m64 b)
   return __builtin_ia32_cvtpi2ps(a, (__v2si)b);
 }
 
+static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_cvt_pi2ps(__m128 a, __m64 b)
+{
+  return _mm_cvtpi32_ps(a, b);
+}
+
 static __inline__ float __attribute__((__always_inline__, __nodebug__))
 _mm_cvtss_f32(__m128 a)
 {
@@ -590,6 +614,12 @@ _mm_store1_ps(float *p, __m128 a)
 }
 
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
+_mm_store_ps1(float *p, __m128 a)
+{
+    return _mm_store1_ps(p, a);
+}
+
+static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_store_ps(float *p, __m128 a)
 {
   *(__m128 *)p = a;
@@ -602,9 +632,9 @@ _mm_storer_ps(float *p, __m128 a)
   _mm_store_ps(p, a);
 }
 
-#define _MM_HINT_T0 1
+#define _MM_HINT_T0 3
 #define _MM_HINT_T1 2
-#define _MM_HINT_T2 3
+#define _MM_HINT_T2 1
 #define _MM_HINT_NTA 0
 
 /* FIXME: We have to #define this because "sel" must be a constant integer, and
@@ -908,6 +938,23 @@ do { \
   (row3) = _mm_movehl_ps(tmp3, tmp1); \
 } while (0)
 
+/* Aliases for compatibility. */
+#define _m_pextrw _mm_extract_pi16
+#define _m_pinsrw _mm_insert_pi16
+#define _m_pmaxsw _mm_max_pi16
+#define _m_pmaxub _mm_max_pu8
+#define _m_pminsw _mm_min_pi16
+#define _m_pminub _mm_min_pu8
+#define _m_pmovmskb _mm_movemask_pi8
+#define _m_pmulhuw _mm_mulhi_pu16
+#define _m_pshufw _mm_shuffle_pi16
+#define _m_maskmovq _mm_maskmove_si64
+#define _m_pavgb _mm_avg_pu8
+#define _m_pavgw _mm_avg_pu16
+#define _m_psadbw _mm_sad_pu8
+#define _m_ _mm_
+#define _m_ _mm_
+
 /* Ugly hack for backwards-compatibility (compatible with gcc) */
 #ifdef __SSE2__
 #include <emmintrin.h>
diff --git a/lib/Index/CMakeLists.txt b/lib/Index/CMakeLists.txt
index 4d6703563613..61f69b218e26 100644
--- a/lib/Index/CMakeLists.txt
+++ b/lib/Index/CMakeLists.txt
@@ -11,6 +11,5 @@ add_clang_library(clangIndex
   IndexProvider.cpp
   Indexer.cpp
   Program.cpp
-  ResolveLocation.cpp
   SelectorMap.cpp
   )
diff --git a/lib/Index/Entity.cpp b/lib/Index/Entity.cpp
index 7a247191bee6..749dcc899322 100644
--- a/lib/Index/Entity.cpp
+++ b/lib/Index/Entity.cpp
@@ -134,7 +134,7 @@ Entity EntityGetter::VisitVarDecl(VarDecl *D) {
     return Entity();
 
   // If it's static it cannot be referred to by another translation unit.
-  if (D->getStorageClass() == VarDecl::Static)
+  if (D->getStorageClass() == SC_Static)
     return Entity(D);
 
   return VisitNamedDecl(D);
@@ -142,7 +142,7 @@ Entity EntityGetter::VisitVarDecl(VarDecl *D) {
 
 Entity EntityGetter::VisitFunctionDecl(FunctionDecl *D) {
   // If it's static it cannot be refered to by another translation unit.
-  if (D->getStorageClass() == FunctionDecl::Static)
+  if (D->getStorageClass() == SC_Static)
     return Entity(D);
 
   return VisitNamedDecl(D);
diff --git a/lib/Index/Makefile b/lib/Index/Makefile
index e87e638f69ec..8607d781e5d7 100644
--- a/lib/Index/Makefile
+++ b/lib/Index/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangIndex
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Index/ResolveLocation.cpp b/lib/Index/ResolveLocation.cpp
deleted file mode 100644
index ccd7a126b40c..000000000000
--- a/lib/Index/ResolveLocation.cpp
+++ /dev/null
@@ -1,602 +0,0 @@
-//===--- ResolveLocation.cpp - Source location resolver ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This defines the ResolveLocationInAST function, which resolves a
-//  source location into a ASTLocation.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Index/Utils.h"
-#include "clang/Index/ASTLocation.h"
-#include "clang/AST/TypeLocVisitor.h"
-#include "clang/AST/DeclVisitor.h"
-#include "clang/AST/StmtVisitor.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Basic/SourceManager.h"
-using namespace clang;
-using namespace idx;
-
-namespace {
-
-/// \brief Base for the LocResolver classes. Mostly does source range checking.
-class LocResolverBase {
-protected:
-  ASTContext &Ctx;
-  SourceLocation Loc;
-  
-  ASTLocation ResolveInDeclarator(Decl *D, Stmt *Stm, TypeSourceInfo *TInfo);
-
-  enum RangePos {
-    BeforeLoc,
-    ContainsLoc,
-    AfterLoc
-  };
-
-  RangePos CheckRange(SourceRange Range);
-  RangePos CheckRange(TypeSourceInfo *TInfo);
-  RangePos CheckRange(Decl *D) {
-    if (DeclaratorDecl *DD = dyn_cast<DeclaratorDecl>(D))
-      if (ContainsLocation(DD->getTypeSourceInfo()))
-        return ContainsLoc;
-    if (TypedefDecl *TD = dyn_cast<TypedefDecl>(D))
-      if (ContainsLocation(TD->getTypeSourceInfo()))
-        return ContainsLoc;
-
-    return CheckRange(D->getSourceRange());
-  }
-  RangePos CheckRange(Stmt *Node) { return CheckRange(Node->getSourceRange()); }
-  RangePos CheckRange(TypeLoc TL) { return CheckRange(TL.getLocalSourceRange()); }
-
-  template <typename T>
-  bool isBeforeLocation(T Node) {
-    return CheckRange(Node) == BeforeLoc;
-  }
-
-  template <typename T>
-  bool isAfterLocation(T Node) {
-    return CheckRange(Node) == AfterLoc;
-  }
-
-public:
-  LocResolverBase(ASTContext &ctx, SourceLocation loc)
-    : Ctx(ctx), Loc(loc) {}
-    
-  template <typename T>
-  bool ContainsLocation(T Node) {
-    return CheckRange(Node) == ContainsLoc;
-  }
-
-#ifndef NDEBUG
-  /// \brief Debugging output.
-  void print(Decl *D);
-  /// \brief Debugging output.
-  void print(Stmt *Node);
-#endif
-};
-
-/// \brief Searches a statement for the ASTLocation that corresponds to a source
-/// location.
-class StmtLocResolver : public LocResolverBase,
-                                          public StmtVisitor<StmtLocResolver,
-                                                             ASTLocation     > {
-  Decl * const Parent;
-
-public:
-  StmtLocResolver(ASTContext &ctx, SourceLocation loc, Decl *parent)
-    : LocResolverBase(ctx, loc), Parent(parent) {}
-
-  ASTLocation VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *Node);
-  ASTLocation VisitCXXOperatorCallExpr(CXXOperatorCallExpr *Node);
-  ASTLocation VisitDeclStmt(DeclStmt *Node);
-  ASTLocation VisitStmt(Stmt *Node);
-};
-
-/// \brief Searches a declaration for the ASTLocation that corresponds to a
-/// source location.
-class DeclLocResolver : public LocResolverBase,
-                                          public DeclVisitor<DeclLocResolver,
-                                                             ASTLocation     > {
-public:
-  DeclLocResolver(ASTContext &ctx, SourceLocation loc)
-    : LocResolverBase(ctx, loc) {}
-
-  ASTLocation VisitDeclContext(DeclContext *DC);
-  ASTLocation VisitTranslationUnitDecl(TranslationUnitDecl *TU);
-  ASTLocation VisitDeclaratorDecl(DeclaratorDecl *D);
-  ASTLocation VisitVarDecl(VarDecl *D);
-  ASTLocation VisitFunctionDecl(FunctionDecl *D);
-  ASTLocation VisitObjCClassDecl(ObjCClassDecl *D);                                                             
-  ASTLocation VisitObjCMethodDecl(ObjCMethodDecl *D);
-  ASTLocation VisitTypedefDecl(TypedefDecl *D);
-  ASTLocation VisitDecl(Decl *D);
-};
-
-class TypeLocResolver : public LocResolverBase,
-                        public TypeLocVisitor<TypeLocResolver, ASTLocation> {
-  Decl * const ParentDecl;
-
-public:
-  TypeLocResolver(ASTContext &ctx, SourceLocation loc, Decl *pd)
-    : LocResolverBase(ctx, loc), ParentDecl(pd) { }
-
-  ASTLocation VisitBuiltinTypeLoc(BuiltinTypeLoc TL);
-  ASTLocation VisitTypedefTypeLoc(TypedefTypeLoc TL);
-  ASTLocation VisitFunctionTypeLoc(FunctionTypeLoc TL);
-  ASTLocation VisitArrayTypeLoc(ArrayTypeLoc TL);
-  ASTLocation VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc TL);
-  ASTLocation VisitObjCObjectTypeLoc(ObjCObjectTypeLoc TL);
-  ASTLocation VisitTypeLoc(TypeLoc TL);
-};
-
-} // anonymous namespace
-
-ASTLocation
-StmtLocResolver::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *Node) {
-  assert(ContainsLocation(Node) &&
-         "Should visit only after verifying that loc is in range");
-
-  if (Node->isArgumentType()) {
-    TypeSourceInfo *TInfo = Node->getArgumentTypeInfo();
-    if (ContainsLocation(TInfo))
-      return ResolveInDeclarator(Parent, Node, TInfo);
-  } else {
-    Expr *SubNode = Node->getArgumentExpr();
-    if (ContainsLocation(SubNode))
-      return Visit(SubNode);
-  }
-
-  return ASTLocation(Parent, Node);
-}
-
-
-ASTLocation
-StmtLocResolver::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *Node) {
-  assert(ContainsLocation(Node) &&
-         "Should visit only after verifying that loc is in range");
-
-  if (Node->getNumArgs() == 1)
-    // Unary operator. Let normal child traversal handle it.
-    return VisitCallExpr(Node);
-
-  assert(Node->getNumArgs() == 2 &&
-         "Wrong args for the C++ operator call expr ?");
-
-  llvm::SmallVector<Expr *, 3> Nodes;
-  // Binary operator. Check in order of 1-left arg, 2-callee, 3-right arg.
-  Nodes.push_back(Node->getArg(0));
-  Nodes.push_back(Node->getCallee());
-  Nodes.push_back(Node->getArg(1));
-
-  for (unsigned i = 0, e = Nodes.size(); i != e; ++i) {
-    RangePos RP = CheckRange(Nodes[i]);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return Visit(Nodes[i]);
-  }
-
-  return ASTLocation(Parent, Node);
-}
-
-ASTLocation StmtLocResolver::VisitDeclStmt(DeclStmt *Node) {
-  assert(ContainsLocation(Node) &&
-         "Should visit only after verifying that loc is in range");
-
-  // Search all declarations of this DeclStmt.
-  for (DeclStmt::decl_iterator
-         I = Node->decl_begin(), E = Node->decl_end(); I != E; ++I) {
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return DeclLocResolver(Ctx, Loc).Visit(*I);
-  }
-
-  return ASTLocation(Parent, Node);
-}
-
-ASTLocation StmtLocResolver::VisitStmt(Stmt *Node) {
-  assert(ContainsLocation(Node) &&
-         "Should visit only after verifying that loc is in range");
-
-  // Search the child statements.
-  for (Stmt::child_iterator
-         I = Node->child_begin(), E = Node->child_end(); I != E; ++I) {
-    if (*I == NULL)
-      continue;
-
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return Visit(*I);
-  }
-
-  return ASTLocation(Parent, Node);
-}
-
-ASTLocation DeclLocResolver::VisitDeclContext(DeclContext *DC) {
-  for (DeclContext::decl_iterator
-         I = DC->decls_begin(), E = DC->decls_end(); I != E; ++I) {
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return Visit(*I);
-  }
-
-  return ASTLocation(cast<Decl>(DC));
-}
-
-ASTLocation DeclLocResolver::VisitTranslationUnitDecl(TranslationUnitDecl *TU) {
-  ASTLocation ASTLoc = VisitDeclContext(TU);
-  if (ASTLoc.getParentDecl() == TU)
-    return ASTLocation();
-  return ASTLoc;
-}
-
-ASTLocation DeclLocResolver::VisitFunctionDecl(FunctionDecl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-
-  if (ContainsLocation(D->getTypeSourceInfo()))
-    return ResolveInDeclarator(D, 0, D->getTypeSourceInfo());
-
-  // First, search through the parameters of the function.
-  for (FunctionDecl::param_iterator
-         I = D->param_begin(), E = D->param_end(); I != E; ++I) {
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      return ASTLocation(D);
-    if (RP == ContainsLoc)
-      return Visit(*I);
-  }
-
-  // We didn't find the location in the parameters and we didn't get passed it.
-
-  if (!D->isThisDeclarationADefinition())
-    return ASTLocation(D);
-
-  // Second, search through the declarations that are part of the function.
-  // If we find the location there, we won't have to search through its body.
-
-  for (DeclContext::decl_iterator
-         I = D->decls_begin(), E = D->decls_end(); I != E; ++I) {
-    if (isa<ParmVarDecl>(*I))
-      continue; // We already searched through the parameters.
-
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return Visit(*I);
-  }
-
-  // We didn't find a declaration that corresponds to the source location.
-
-  // Finally, search through the body of the function.
-  Stmt *Body = D->getBody();
-  assert(Body && "Expected definition");
-  assert(!isBeforeLocation(Body) &&
-         "This function is supposed to contain the loc");
-  if (isAfterLocation(Body))
-    return ASTLocation(D);
-
-  // The body contains the location.
-  assert(ContainsLocation(Body));
-  return StmtLocResolver(Ctx, Loc, D).Visit(Body);
-}
-
-ASTLocation DeclLocResolver::VisitDeclaratorDecl(DeclaratorDecl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-  if (ContainsLocation(D->getTypeSourceInfo()))
-    return ResolveInDeclarator(D, /*Stmt=*/0, D->getTypeSourceInfo());
-
-  return ASTLocation(D);
-}
-
-ASTLocation DeclLocResolver::VisitTypedefDecl(TypedefDecl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-
-  if (ContainsLocation(D->getTypeSourceInfo()))
-    return ResolveInDeclarator(D, /*Stmt=*/0, D->getTypeSourceInfo());
-
-  return ASTLocation(D);
-}
-
-ASTLocation DeclLocResolver::VisitVarDecl(VarDecl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-
-  // Check whether the location points to the init expression.
-  Expr *Init = D->getInit();
-  if (Init && ContainsLocation(Init))
-    return StmtLocResolver(Ctx, Loc, D).Visit(Init);
-  
-  if (ContainsLocation(D->getTypeSourceInfo()))
-    return ResolveInDeclarator(D, 0, D->getTypeSourceInfo());
-
-  return ASTLocation(D);
-}
-
-ASTLocation DeclLocResolver::VisitObjCClassDecl(ObjCClassDecl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-         
-  for (ObjCClassDecl::iterator I = D->begin(), E = D->end() ; I != E; ++I) {
-    if (CheckRange(I->getLocation()) == ContainsLoc)
-      return ASTLocation(D, I->getInterface(), I->getLocation());
-  }
-  return ASTLocation(D);
-}
-
-ASTLocation DeclLocResolver::VisitObjCMethodDecl(ObjCMethodDecl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-
-  // First, search through the parameters of the method.
-  for (ObjCMethodDecl::param_iterator
-         I = D->param_begin(), E = D->param_end(); I != E; ++I) {
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      return ASTLocation(D);
-    if (RP == ContainsLoc)
-      return Visit(*I);
-  }
-
-  // We didn't find the location in the parameters and we didn't get passed it.
-
-  if (!D->getBody())
-    return ASTLocation(D);
-
-  // Second, search through the declarations that are part of the method.
-  // If we find he location there, we won't have to search through its body.
-
-  for (DeclContext::decl_iterator
-         I = D->decls_begin(), E = D->decls_end(); I != E; ++I) {
-    if (isa<ParmVarDecl>(*I))
-      continue; // We already searched through the parameters.
-
-    RangePos RP = CheckRange(*I);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return Visit(*I);
-  }
-
-  // We didn't find a declaration that corresponds to the source location.
-
-  // Finally, search through the body of the method.
-  Stmt *Body = D->getBody();
-  assert(Body && "Expected definition");
-  assert(!isBeforeLocation(Body) &&
-         "This method is supposed to contain the loc");
-  if (isAfterLocation(Body))
-    return ASTLocation(D);
-
-  // The body contains the location.
-  assert(ContainsLocation(Body));
-  return StmtLocResolver(Ctx, Loc, D).Visit(Body);
-}
-
-ASTLocation DeclLocResolver::VisitDecl(Decl *D) {
-  assert(ContainsLocation(D) &&
-         "Should visit only after verifying that loc is in range");
-  if (DeclContext *DC = dyn_cast<DeclContext>(D))
-    return VisitDeclContext(DC);
-  return ASTLocation(D);
-}
-
-ASTLocation TypeLocResolver::VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
-  // Continue the 'id' magic by making the builtin type (which cannot
-  // actually be spelled) map to the typedef.
-  BuiltinType *T = TL.getTypePtr();
-  if (T->getKind() == BuiltinType::ObjCId) {
-    TypedefDecl *D = Ctx.getObjCIdType()->getAs<TypedefType>()->getDecl();
-    return ASTLocation(ParentDecl, D, TL.getNameLoc());
-  }
-
-  // Same thing with 'Class'.
-  if (T->getKind() == BuiltinType::ObjCClass) {
-    TypedefDecl *D = Ctx.getObjCClassType()->getAs<TypedefType>()->getDecl();
-    return ASTLocation(ParentDecl, D, TL.getNameLoc());
-  }
-
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation TypeLocResolver::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-  assert(ContainsLocation(TL) &&
-         "Should visit only after verifying that loc is in range");
-  if (ContainsLocation(TL.getNameLoc()))
-    return ASTLocation(ParentDecl, TL.getTypedefDecl(), TL.getNameLoc());
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation TypeLocResolver::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
-  assert(ContainsLocation(TL) &&
-         "Should visit only after verifying that loc is in range");
-
-  for (unsigned i = 0; i != TL.getNumArgs(); ++i) {
-    ParmVarDecl *Parm = TL.getArg(i);
-    RangePos RP = CheckRange(Parm);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return DeclLocResolver(Ctx, Loc).Visit(Parm);
-  }
-
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation TypeLocResolver::VisitArrayTypeLoc(ArrayTypeLoc TL) {
-  assert(ContainsLocation(TL) &&
-         "Should visit only after verifying that loc is in range");
-
-  Expr *E = TL.getSizeExpr();
-  if (E && ContainsLocation(E))
-    return StmtLocResolver(Ctx, Loc, ParentDecl).Visit(E);
-
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation TypeLocResolver::VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc TL) {
-  assert(ContainsLocation(TL) &&
-         "Should visit only after verifying that loc is in range");
-  if (ContainsLocation(TL.getNameLoc()))
-    return ASTLocation(ParentDecl, TL.getIFaceDecl(), TL.getNameLoc());
-
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation TypeLocResolver::VisitObjCObjectTypeLoc(ObjCObjectTypeLoc TL) {
-  assert(ContainsLocation(TL) &&
-         "Should visit only after verifying that loc is in range");
-
-  for (unsigned i = 0; i != TL.getNumProtocols(); ++i) {
-    SourceLocation L = TL.getProtocolLoc(i);
-    RangePos RP = CheckRange(L);
-    if (RP == AfterLoc)
-      break;
-    if (RP == ContainsLoc)
-      return ASTLocation(ParentDecl, TL.getProtocol(i), L);
-  }
-
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation TypeLocResolver::VisitTypeLoc(TypeLoc TL) {
-  assert(ContainsLocation(TL) &&
-         "Should visit only after verifying that loc is in range");
-  return ASTLocation(ParentDecl, TL);
-}
-
-ASTLocation LocResolverBase::ResolveInDeclarator(Decl *D, Stmt *Stm,
-                                                 TypeSourceInfo *TInfo) {
-  assert(ContainsLocation(TInfo) &&
-         "Should visit only after verifying that loc is in range");
-  
-  (void)TypeLocResolver(Ctx, Loc, D);
-  for (TypeLoc TL = TInfo->getTypeLoc(); TL; TL = TL.getNextTypeLoc())
-    if (ContainsLocation(TL))
-      return TypeLocResolver(Ctx, Loc, D).Visit(TL);
-  
-  assert(0 && "Should have found the loc in a typeloc");
-  return ASTLocation(D, Stm);
-}
-
-LocResolverBase::RangePos LocResolverBase::CheckRange(TypeSourceInfo *TInfo) {
-  if (!TInfo)
-    return BeforeLoc; // Keep looking.
-
-  for (TypeLoc TL = TInfo->getTypeLoc(); TL; TL = TL.getNextTypeLoc())
-    if (ContainsLocation(TL))
-      return ContainsLoc;
-
-  return BeforeLoc; // Keep looking.
-}
-
-LocResolverBase::RangePos LocResolverBase::CheckRange(SourceRange Range) {
-  if (!Range.isValid())
-    return BeforeLoc; // Keep looking.
-
-  // Update the end source range to cover the full length of the token
-  // positioned at the end of the source range.
-  //
-  // e.g.,
-  //   int foo
-  //   ^   ^
-  //
-  // will be updated to
-  //   int foo
-  //   ^     ^
-  unsigned TokSize = Lexer::MeasureTokenLength(Range.getEnd(),
-                                               Ctx.getSourceManager(),
-                                               Ctx.getLangOptions());
-  Range.setEnd(Range.getEnd().getFileLocWithOffset(TokSize-1));
-
-  SourceManager &SourceMgr = Ctx.getSourceManager();
-  if (SourceMgr.isBeforeInTranslationUnit(Range.getEnd(), Loc))
-    return BeforeLoc;
-
-  if (SourceMgr.isBeforeInTranslationUnit(Loc, Range.getBegin()))
-    return AfterLoc;
-
-  return ContainsLoc;
-}
-
-#ifndef NDEBUG
-void LocResolverBase::print(Decl *D) {
-  llvm::raw_ostream &OS = llvm::outs();
-  OS << "#### DECL " << D->getDeclKindName() << " ####\n";
-  D->print(OS);
-  OS << " <";
-  D->getLocStart().print(OS, Ctx.getSourceManager());
-  OS << " > - <";
-  D->getLocEnd().print(OS, Ctx.getSourceManager());
-  OS << ">\n\n";
-  OS.flush();
-}
-
-void LocResolverBase::print(Stmt *Node) {
-  llvm::raw_ostream &OS = llvm::outs();
-  OS << "#### STMT " << Node->getStmtClassName() << " ####\n";
-  Node->printPretty(OS, Ctx, 0, PrintingPolicy(Ctx.getLangOptions()));
-  OS << " <";
-  Node->getLocStart().print(OS, Ctx.getSourceManager());
-  OS << " > - <";
-  Node->getLocEnd().print(OS, Ctx.getSourceManager());
-  OS << ">\n\n";
-  OS.flush();
-}
-#endif
-
-
-/// \brief Returns the AST node that a source location points to.
-///
-ASTLocation idx::ResolveLocationInAST(ASTContext &Ctx, SourceLocation Loc,
-                                      ASTLocation *LastLoc) {
-  if (Loc.isInvalid())
-    return ASTLocation();
-
-  if (LastLoc && LastLoc->isValid()) {
-    DeclContext *DC = 0;
-  
-    if (Decl *Dcl = LastLoc->dyn_AsDecl()) {
-      DC = Dcl->getDeclContext();
-    } else if (LastLoc->isStmt()) {
-      Decl *Parent = LastLoc->getParentDecl();
-      if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Parent))
-        DC = FD;
-      else { 
-        // This is needed to handle statements within an initializer.
-        // Example:
-        //   void func() { long double fabsf = __builtin_fabsl(__x); }
-        // In this case, the 'parent' of __builtin_fabsl is fabsf.
-        DC = Parent->getDeclContext();
-      }
-    } else { // We have 'N_NamedRef' or 'N_Type'
-      DC = LastLoc->getParentDecl()->getDeclContext();
-    } 
-    assert(DC && "Missing DeclContext");
-    
-    FunctionDecl *FD = dyn_cast<FunctionDecl>(DC);
-    DeclLocResolver DLocResolver(Ctx, Loc);
-    
-    if (FD && FD->isThisDeclarationADefinition() &&
-        DLocResolver.ContainsLocation(FD)) {
-      return DLocResolver.VisitFunctionDecl(FD);
-    }
-    // Fall through and try the slow path...
-    // FIXME: Optimize more cases.
-  }
-  return DeclLocResolver(Ctx, Loc).Visit(Ctx.getTranslationUnitDecl());
-}
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 91b14f638ded..917829be47c6 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -27,7 +27,9 @@
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cctype>
@@ -247,6 +249,200 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
   return TheTok.getLength();
 }
 
+SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
+                                          const SourceManager &SM,
+                                          const LangOptions &LangOpts) {
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  bool Invalid = false;
+  llvm::StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
+  if (Invalid)
+    return Loc;
+
+  // Back up from the current location until we hit the beginning of a line
+  // (or the buffer). We'll relex from that point.
+  const char *BufStart = Buffer.data();
+  const char *StrData = BufStart+LocInfo.second;
+  if (StrData[0] == '\n' || StrData[0] == '\r')
+    return Loc;
+
+  const char *LexStart = StrData;
+  while (LexStart != BufStart) {
+    if (LexStart[0] == '\n' || LexStart[0] == '\r') {
+      ++LexStart;
+      break;
+    }
+
+    --LexStart;
+  }
+  
+  // Create a lexer starting at the beginning of this token.
+  SourceLocation LexerStartLoc = Loc.getFileLocWithOffset(-LocInfo.second);
+  Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end());
+  TheLexer.SetCommentRetentionState(true);
+  
+  // Lex tokens until we find the token that contains the source location.
+  Token TheTok;
+  do {
+    TheLexer.LexFromRawLexer(TheTok);
+    
+    if (TheLexer.getBufferLocation() > StrData) {
+      // Lexing this token has taken the lexer past the source location we're
+      // looking for. If the current token encompasses our source location,
+      // return the beginning of that token.
+      if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData)
+        return TheTok.getLocation();
+      
+      // We ended up skipping over the source location entirely, which means
+      // that it points into whitespace. We're done here.
+      break;
+    }
+  } while (TheTok.getKind() != tok::eof);
+  
+  // We've passed our source location; just return the original source location.
+  return Loc;
+}
+
+namespace {
+  enum PreambleDirectiveKind {
+    PDK_Skipped,
+    PDK_StartIf,
+    PDK_EndIf,
+    PDK_Unknown
+  };
+}
+
+std::pair<unsigned, bool>
+Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines) {
+  // Create a lexer starting at the beginning of the file. Note that we use a
+  // "fake" file source location at offset 1 so that the lexer will track our
+  // position within the file.
+  const unsigned StartOffset = 1;
+  SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset);
+  LangOptions LangOpts;
+  Lexer TheLexer(StartLoc, LangOpts, Buffer->getBufferStart(), 
+                 Buffer->getBufferStart(), Buffer->getBufferEnd());
+  
+  bool InPreprocessorDirective = false;
+  Token TheTok;
+  Token IfStartTok;
+  unsigned IfCount = 0;
+  unsigned Line = 0;
+
+  do {
+    TheLexer.LexFromRawLexer(TheTok);
+
+    if (InPreprocessorDirective) {
+      // If we've hit the end of the file, we're done.
+      if (TheTok.getKind() == tok::eof) {
+        InPreprocessorDirective = false;
+        break;
+      }
+      
+      // If we haven't hit the end of the preprocessor directive, skip this
+      // token.
+      if (!TheTok.isAtStartOfLine())
+        continue;
+        
+      // We've passed the end of the preprocessor directive, and will look
+      // at this token again below.
+      InPreprocessorDirective = false;
+    }
+    
+    // Keep track of the # of lines in the preamble.
+    if (TheTok.isAtStartOfLine()) {
+      ++Line;
+
+      // If we were asked to limit the number of lines in the preamble,
+      // and we're about to exceed that limit, we're done.
+      if (MaxLines && Line >= MaxLines)
+        break;
+    }
+
+    // Comments are okay; skip over them.
+    if (TheTok.getKind() == tok::comment)
+      continue;
+    
+    if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) {
+      // This is the start of a preprocessor directive. 
+      Token HashTok = TheTok;
+      InPreprocessorDirective = true;
+      
+      // Figure out which direective this is. Since we're lexing raw tokens,
+      // we don't have an identifier table available. Instead, just look at
+      // the raw identifier to recognize and categorize preprocessor directives.
+      TheLexer.LexFromRawLexer(TheTok);
+      if (TheTok.getKind() == tok::identifier && !TheTok.needsCleaning()) {
+        const char *IdStart = Buffer->getBufferStart() 
+                            + TheTok.getLocation().getRawEncoding() - 1;
+        llvm::StringRef Keyword(IdStart, TheTok.getLength());
+        PreambleDirectiveKind PDK
+          = llvm::StringSwitch<PreambleDirectiveKind>(Keyword)
+              .Case("include", PDK_Skipped)
+              .Case("__include_macros", PDK_Skipped)
+              .Case("define", PDK_Skipped)
+              .Case("undef", PDK_Skipped)
+              .Case("line", PDK_Skipped)
+              .Case("error", PDK_Skipped)
+              .Case("pragma", PDK_Skipped)
+              .Case("import", PDK_Skipped)
+              .Case("include_next", PDK_Skipped)
+              .Case("warning", PDK_Skipped)
+              .Case("ident", PDK_Skipped)
+              .Case("sccs", PDK_Skipped)
+              .Case("assert", PDK_Skipped)
+              .Case("unassert", PDK_Skipped)
+              .Case("if", PDK_StartIf)
+              .Case("ifdef", PDK_StartIf)
+              .Case("ifndef", PDK_StartIf)
+              .Case("elif", PDK_Skipped)
+              .Case("else", PDK_Skipped)
+              .Case("endif", PDK_EndIf)
+              .Default(PDK_Unknown);
+
+        switch (PDK) {
+        case PDK_Skipped:
+          continue;
+
+        case PDK_StartIf:
+          if (IfCount == 0)
+            IfStartTok = HashTok;
+            
+          ++IfCount;
+          continue;
+            
+        case PDK_EndIf:
+          // Mismatched #endif. The preamble ends here.
+          if (IfCount == 0)
+            break;
+
+          --IfCount;
+          continue;
+            
+        case PDK_Unknown:
+          // We don't know what this directive is; stop at the '#'.
+          break;
+        }
+      }
+      
+      // We only end up here if we didn't recognize the preprocessor
+      // directive or it was one that can't occur in the preamble at this
+      // point. Roll back the current token to the location of the '#'.
+      InPreprocessorDirective = false;
+      TheTok = HashTok;
+    }
+
+    // We hit a token that we don't recognize as being in the
+    // "preprocessing only" part of the file, so we're no longer in
+    // the preamble.
+    break;
+  } while (true);
+  
+  SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation();
+  return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(),
+                        IfCount? IfStartTok.isAtStartOfLine()
+                               : TheTok.isAtStartOfLine());
+}
+
 //===----------------------------------------------------------------------===//
 // Character information.
 //===----------------------------------------------------------------------===//
@@ -476,7 +672,7 @@ static char DecodeTrigraphChar(const char *CP, Lexer *L) {
   }
 
   if (!L->isLexingRawMode())
-    L->Diag(CP-2, diag::trigraph_converted) << std::string()+Res;
+    L->Diag(CP-2, diag::trigraph_converted) << llvm::StringRef(&Res, 1);
   return Res;
 }
 
@@ -647,6 +843,14 @@ Slash:
 // Helper methods for lexing.
 //===----------------------------------------------------------------------===//
 
+/// \brief Routine that indiscriminately skips bytes in the source file.
+void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) {
+  BufferPtr += Bytes;
+  if (BufferPtr > BufferEnd)
+    BufferPtr = BufferEnd;
+  IsAtStartOfLine = StartOfLine;
+}
+
 void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
   // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
   unsigned Size;
@@ -716,6 +920,16 @@ FinishIdentifier:
   }
 }
 
+/// isHexaLiteral - Return true if Start points to a hex constant.
+/// in microsoft mode (where this is supposed to be several different tokens).
+static bool isHexaLiteral(const char *Start, const LangOptions &Features) {
+  unsigned Size;
+  char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, Features);
+  if (C1 != '0')
+    return false;
+  char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, Features);
+  return (C2 == 'x' || C2 == 'X');
+}
 
 /// LexNumericConstant - Lex the remainder of a integer or floating point
 /// constant. From[-1] is the first character lexed.  Return the end of the
@@ -731,12 +945,16 @@ void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
   }
 
   // If we fell out, check for a sign, due to 1e+12.  If we have one, continue.
-  if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e'))
-    return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
+  if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) {
+    // If we are in Microsoft mode, don't continue if the constant is hex.
+    // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1
+    if (!Features.Microsoft || !isHexaLiteral(BufferPtr, Features))
+      return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
+  }
 
   // If we have a hex FP constant, continue.
   if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p') &&
-      (!PP || !PP->getLangOptions().CPlusPlus0x))
+      !Features.CPlusPlus0x)
     return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result));
 
   // Update the location of token as well as BufferPtr.
@@ -759,7 +977,9 @@ void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide) {
     
     if (C == '\n' || C == '\r' ||             // Newline.
         (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      if (!isLexingRawMode() && !Features.AsmPreprocessor)
+      if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
+        PP->CodeCompleteNaturalLanguage();
+      else if (!isLexingRawMode() && !Features.AsmPreprocessor)
         Diag(BufferPtr, diag::err_unterminated_string);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
@@ -836,7 +1056,9 @@ void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
       C = getAndAdvanceChar(CurPtr, Result);
     } else if (C == '\n' || C == '\r' ||             // Newline.
                (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
-      if (!isLexingRawMode() && !Features.AsmPreprocessor)
+      if (C == 0 && PP && PP->isCodeCompletionFile(FileLoc))
+        PP->CodeCompleteNaturalLanguage();
+      else if (!isLexingRawMode() && !Features.AsmPreprocessor)
         Diag(BufferPtr, diag::err_unterminated_char);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
@@ -980,7 +1202,13 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
         }
     }
 
-    if (CurPtr == BufferEnd+1) { --CurPtr; break; }
+    if (CurPtr == BufferEnd+1) { 
+      if (PP && PP->isCodeCompletionFile(FileLoc))
+        PP->CodeCompleteNaturalLanguage();
+
+      --CurPtr; 
+      break; 
+    }
   } while (C != '\n' && C != '\r');
 
   // Found but did not consume the newline.  Notify comment handlers about the
@@ -1219,7 +1447,9 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
           Diag(CurPtr-1, diag::warn_nested_block_comment);
       }
     } else if (C == 0 && CurPtr == BufferEnd+1) {
-      if (!isLexingRawMode() && !PP->isCodeCompletionFile(FileLoc))
+      if (PP && PP->isCodeCompletionFile(FileLoc))
+        PP->CodeCompleteNaturalLanguage();
+      else if (!isLexingRawMode())
         Diag(BufferPtr, diag::err_unterminated_block_comment);
       // Note: the user probably forgot a */.  We could continue immediately
       // after the /*, but this would involve lexing a lot of what really is the
@@ -1305,6 +1535,11 @@ std::string Lexer::ReadToEndOfLine() {
 
       // Next, lex the character, which should handle the EOM transition.
       Lex(Tmp);
+      if (Tmp.is(tok::code_completion)) {
+        if (PP && PP->getCodeCompletionHandler())
+          PP->getCodeCompletionHandler()->CodeCompleteNaturalLanguage();
+        Lex(Tmp);
+      }
       assert(Tmp.is(tok::eom) && "Unexpected token!");
 
       // Finally, we're done, return the string we found.
@@ -1318,6 +1553,22 @@ std::string Lexer::ReadToEndOfLine() {
 /// This returns true if Result contains a token, false if PP.Lex should be
 /// called again.
 bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
+  // Check if we are performing code completion.
+  if (PP && PP->isCodeCompletionFile(FileLoc)) {
+    // We're at the end of the file, but we've been asked to consider the
+    // end of the file to be a code-completion token. Return the
+    // code-completion token.
+    Result.startToken();
+    FormTokenWithChars(Result, CurPtr, tok::code_completion);
+    
+    // Only do the eof -> code_completion translation once.
+    PP->SetCodeCompletionPoint(0, 0, 0);
+    
+    // Silence any diagnostics that occur once we hit the code-completion point.
+    PP->getDiagnostics().setSuppressAllDiagnostics(true);
+    return true;
+  }
+
   // If we hit the end of the file while parsing a preprocessor directive,
   // end the preprocessor directive first.  The next token returned will
   // then be the end of file.
@@ -1340,29 +1591,14 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
     FormTokenWithChars(Result, BufferEnd, tok::eof);
     return true;
   }
-
-  // Otherwise, check if we are code-completing, then issue diagnostics for 
-  // unterminated #if and missing newline.
-
-  if (PP && PP->isCodeCompletionFile(FileLoc)) {
-    // We're at the end of the file, but we've been asked to consider the
-    // end of the file to be a code-completion token. Return the
-    // code-completion token.
-    Result.startToken();
-    FormTokenWithChars(Result, CurPtr, tok::code_completion);
-    
-    // Only do the eof -> code_completion translation once.
-    PP->SetCodeCompletionPoint(0, 0, 0);
-    
-    // Silence any diagnostics that occur once we hit the code-completion point.
-    PP->getDiagnostics().setSuppressAllDiagnostics(true);
-    return true;
-  }
   
+  // Issue diagnostics for unterminated #if and missing newline.
+
   // If we are in a #if directive, emit an error.
   while (!ConditionalStack.empty()) {
-    PP->Diag(ConditionalStack.back().IfLoc,
-             diag::err_pp_unterminated_conditional);
+    if (!PP->isCodeCompletionFile(FileLoc))
+      PP->Diag(ConditionalStack.back().IfLoc,
+               diag::err_pp_unterminated_conditional);
     ConditionalStack.pop_back();
   }
 
diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp
index b8fd3ce9e9ff..fb543d0f03b3 100644
--- a/lib/Lex/LiteralSupport.cpp
+++ b/lib/Lex/LiteralSupport.cpp
@@ -170,6 +170,7 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
 static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
                              char *&ResultBuf, bool &HadError,
                              SourceLocation Loc, Preprocessor &PP,
+                             bool wide,
                              bool Complain) {
   // FIXME: Add a warning - UCN's are only valid in C++ & C99.
   // FIXME: Handle wide strings.
@@ -190,6 +191,7 @@ static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
 
   UTF32 UcnVal = 0;
   unsigned short UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
+  unsigned short UcnLenSave = UcnLen;
   for (; ThisTokBuf != ThisTokEnd && UcnLen; ++ThisTokBuf, UcnLen--) {
     int CharVal = HexDigitValue(ThisTokBuf[0]);
     if (CharVal == -1) break;
@@ -214,6 +216,17 @@ static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
     HadError = 1;
     return;
   }
+  if (wide) {
+    (void)UcnLenSave;
+    assert(UcnLenSave == 4 && 
+           "ProcessUCNEscape - only ucn length of 4 supported");
+    // little endian assumed.
+    *ResultBuf++ = (UcnVal & 0x000000FF);
+    *ResultBuf++ = (UcnVal & 0x0000FF00) >> 8;
+    *ResultBuf++ = (UcnVal & 0x00FF0000) >> 16;
+    *ResultBuf++ = (UcnVal & 0xFF000000) >> 24;
+    return;
+  }
   // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
   // The conversion below was inspired by:
   //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
@@ -323,7 +336,7 @@ NumericLiteralParser(const char *begin, const char *end,
       // Done.
     } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) {
       PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
-              diag::err_invalid_decimal_digit) << std::string(s, s+1);
+              diag::err_invalid_decimal_digit) << llvm::StringRef(s, 1);
       hadError = true;
       return;
     } else if (*s == '.') {
@@ -439,7 +452,7 @@ NumericLiteralParser(const char *begin, const char *end,
     PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
             isFPConstant ? diag::err_invalid_suffix_float_constant :
                            diag::err_invalid_suffix_integer_constant)
-      << std::string(SuffixBegin, ThisTokEnd);
+      << llvm::StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
     hadError = true;
     return;
   }
@@ -510,7 +523,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
       // Done.
     } else if (isxdigit(*s)) {
       PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
-              diag::err_invalid_binary_digit) << std::string(s, s+1);
+              diag::err_invalid_binary_digit) << llvm::StringRef(s, 1);
       hadError = true;
     }
     // Other suffixes will be diagnosed by the caller.
@@ -540,7 +553,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
   // the code is using an incorrect base.
   if (isxdigit(*s) && *s != 'e' && *s != 'E') {
     PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
-            diag::err_invalid_octal_digit) << std::string(s, s+1);
+            diag::err_invalid_octal_digit) << llvm::StringRef(s, 1);
     hadError = true;
     return;
   }
@@ -830,12 +843,14 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
     }
 
     const char *ThisTokEnd = ThisTokBuf+ThisTokLen-1;  // Skip end quote.
-
+    bool wide = false;
     // TODO: Input character set mapping support.
 
     // Skip L marker for wide strings.
-    if (ThisTokBuf[0] == 'L')
+    if (ThisTokBuf[0] == 'L') {
+      wide = true;
       ++ThisTokBuf;
+    }
 
     assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?");
     ++ThisTokBuf;
@@ -880,7 +895,8 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
       // Is this a Universal Character Name escape?
       if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
         ProcessUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
-                         hadError, StringToks[i].getLocation(), PP, Complain);
+                         hadError, StringToks[i].getLocation(), PP, wide, 
+                         Complain);
         continue;
       }
       // Otherwise, this is a non-UCN escape character.  Process it.
@@ -911,6 +927,20 @@ StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
       hadError = 1;
       return;
     }
+  } else if (Complain) {
+    // Complain if this string literal has too many characters.
+    unsigned MaxChars = PP.getLangOptions().CPlusPlus? 65536
+                      : PP.getLangOptions().C99 ? 4095
+                      : 509;
+    
+    if (GetNumStringChars() > MaxChars)
+      PP.Diag(StringToks[0].getLocation(), diag::ext_string_too_long)
+        << GetNumStringChars() << MaxChars
+        << (PP.getLangOptions().CPlusPlus? 2
+            : PP.getLangOptions().C99 ? 1
+            : 0)
+        << SourceRange(StringToks[0].getLocation(),
+                       StringToks[NumStringToks-1].getLocation());
   }
 }
 
diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp
index fda884c4da4c..c6d09349b5ef 100644
--- a/lib/Lex/MacroInfo.cpp
+++ b/lib/Lex/MacroInfo.cpp
@@ -20,13 +20,32 @@ MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc) {
   IsC99Varargs = false;
   IsGNUVarargs = false;
   IsBuiltinMacro = false;
+  IsFromAST = false;
   IsDisabled = false;
   IsUsed = true;
+  IsAllowRedefinitionsWithoutWarning = false;
 
   ArgumentList = 0;
   NumArguments = 0;
 }
 
+MacroInfo::MacroInfo(const MacroInfo &MI, llvm::BumpPtrAllocator &PPAllocator) {
+  Location = MI.Location;
+  EndLocation = MI.EndLocation;
+  ReplacementTokens = MI.ReplacementTokens;
+  IsFunctionLike = MI.IsFunctionLike;
+  IsC99Varargs = MI.IsC99Varargs;
+  IsGNUVarargs = MI.IsGNUVarargs;
+  IsBuiltinMacro = MI.IsBuiltinMacro;
+  IsFromAST = MI.IsFromAST;
+  IsDisabled = MI.IsDisabled;
+  IsUsed = MI.IsUsed;
+  IsAllowRedefinitionsWithoutWarning = MI.IsAllowRedefinitionsWithoutWarning;
+  ArgumentList = 0;
+  NumArguments = 0;
+  setArgumentList(MI.ArgumentList, MI.NumArguments, PPAllocator);
+}
+
 /// isIdenticalTo - Return true if the specified macro definition is equal to
 /// this macro in spelling, arguments, and whitespace.  This is used to emit
 /// duplicate definition warnings.  This implements the rules in C99 6.10.3.
diff --git a/lib/Lex/Makefile b/lib/Lex/Makefile
index 938b8d5f0540..d80fb55c78a7 100644
--- a/lib/Lex/Makefile
+++ b/lib/Lex/Makefile
@@ -15,7 +15,6 @@ CLANG_LEVEL := ../..
 include $(CLANG_LEVEL)/../../Makefile.config
 
 LIBRARYNAME := clangLex
-BUILD_ARCHIVE = 1
 
 ifeq ($(ARCH),PowerPC)
 CXX.Flags += -maltivec
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 417724b77787..8da7def9ed4d 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -16,6 +16,7 @@
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/APInt.h"
@@ -25,7 +26,7 @@ using namespace clang;
 // Utility Methods for Preprocessor Directive Handling.
 //===----------------------------------------------------------------------===//
 
-MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
+MacroInfo *Preprocessor::AllocateMacroInfo() {
   MacroInfo *MI;
 
   if (!MICache.empty()) {
@@ -33,15 +34,26 @@ MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
     MICache.pop_back();
   } else
     MI = (MacroInfo*) BP.Allocate<MacroInfo>();
+  return MI;
+}
+
+MacroInfo *Preprocessor::AllocateMacroInfo(SourceLocation L) {
+  MacroInfo *MI = AllocateMacroInfo();
   new (MI) MacroInfo(L);
   return MI;
 }
 
+MacroInfo *Preprocessor::CloneMacroInfo(const MacroInfo &MacroToClone) {
+  MacroInfo *MI = AllocateMacroInfo();
+  new (MI) MacroInfo(MacroToClone, BP);
+  return MI;
+}
+
 /// ReleaseMacroInfo - Release the specified MacroInfo.  This memory will
 ///  be reused for allocating new MacroInfo objects.
-void Preprocessor::ReleaseMacroInfo(MacroInfo* MI) {
+void Preprocessor::ReleaseMacroInfo(MacroInfo *MI) {
   MICache.push_back(MI);
-  MI->FreeArgumentList(BP);
+  MI->FreeArgumentList();
 }
 
 
@@ -63,6 +75,13 @@ void Preprocessor::ReadMacroName(Token &MacroNameTok, char isDefineUndef) {
   // Read the token, don't allow macro expansion on it.
   LexUnexpandedToken(MacroNameTok);
 
+  if (MacroNameTok.is(tok::code_completion)) {
+    if (CodeComplete)
+      CodeComplete->CodeCompleteMacroName(isDefineUndef == 1);
+    LexUnexpandedToken(MacroNameTok);
+    return;
+  }
+  
   // Missing macro name?
   if (MacroNameTok.is(tok::eom)) {
     Diag(MacroNameTok, diag::err_pp_missing_macro_name);
@@ -166,13 +185,20 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
   while (1) {
     CurLexer->Lex(Tok);
 
+    if (Tok.is(tok::code_completion)) {
+      if (CodeComplete)
+        CodeComplete->CodeCompleteInConditionalExclusion();
+      continue;
+    }
+    
     // If this is the end of the buffer, we have an error.
     if (Tok.is(tok::eof)) {
       // Emit errors for each unterminated conditional on the stack, including
       // the current one.
       while (!CurPPLexer->ConditionalStack.empty()) {
-        Diag(CurPPLexer->ConditionalStack.back().IfLoc,
-             diag::err_pp_unterminated_conditional);
+        if (!isCodeCompletionFile(Tok.getLocation()))
+          Diag(CurPPLexer->ConditionalStack.back().IfLoc,
+               diag::err_pp_unterminated_conditional);
         CurPPLexer->ConditionalStack.pop_back();
       }
 
@@ -510,7 +536,11 @@ TryAgain:
     // Handle stuff like "# /*foo*/ define X" in -E -C mode.
     LexUnexpandedToken(Result);
     goto TryAgain;
-
+  case tok::code_completion:
+    if (CodeComplete)
+      CodeComplete->CodeCompleteDirective(
+                                    CurPPLexer->getConditionalStackDepth() > 0);
+    return;
   case tok::numeric_constant:  // # 7  GNU line marker directive.
     if (getLangOptions().AsmPreprocessor)
       break;  // # 4 is not a preprocessor directive in .S files.
@@ -1445,15 +1475,15 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
       if (!OtherMI->isUsed())
         Diag(OtherMI->getDefinitionLoc(), diag::pp_macro_not_used);
 
-      // Macros must be identical.  This means all tokes and whitespace
+      // Macros must be identical.  This means all tokens and whitespace
       // separation must be the same.  C99 6.10.3.2.
-      if (!MI->isIdenticalTo(*OtherMI, *this)) {
+      if (!OtherMI->isAllowRedefinitionsWithoutWarning() &&
+          !MI->isIdenticalTo(*OtherMI, *this)) {
         Diag(MI->getDefinitionLoc(), diag::ext_pp_macro_redef)
           << MacroNameTok.getIdentifierInfo();
         Diag(OtherMI->getDefinitionLoc(), diag::note_previous_definition);
       }
     }
-
     ReleaseMacroInfo(OtherMI);
   }
 
@@ -1490,7 +1520,8 @@ void Preprocessor::HandleUndefDirective(Token &UndefTok) {
 
   // If the callbacks want to know, tell them about the macro #undef.
   if (Callbacks)
-    Callbacks->MacroUndefined(MacroNameTok.getIdentifierInfo(), MI);
+    Callbacks->MacroUndefined(MacroNameTok.getLocation(),
+                              MacroNameTok.getIdentifierInfo(), MI);
 
   // Free macro definition.
   ReleaseMacroInfo(MI);
diff --git a/lib/Lex/PPExpressions.cpp b/lib/Lex/PPExpressions.cpp
index 756ce27a93dc..163e869400aa 100644
--- a/lib/Lex/PPExpressions.cpp
+++ b/lib/Lex/PPExpressions.cpp
@@ -19,11 +19,14 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "llvm/ADT/APSInt.h"
 using namespace clang;
 
+namespace {
+
 /// PPValue - Represents the value of a subexpression of a preprocessor
 /// conditional and the source range covered by it.
 class PPValue {
@@ -47,6 +50,8 @@ public:
   void setEnd(SourceLocation L) { Range.setEnd(L); }
 };
 
+}
+
 static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
                                      Token &PeekTok, bool ValueLive,
                                      Preprocessor &PP);
@@ -88,6 +93,12 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     PP.LexUnexpandedToken(PeekTok);
   }
 
+  if (PeekTok.is(tok::code_completion)) {
+    if (PP.getCodeCompletionHandler())
+      PP.getCodeCompletionHandler()->CodeCompleteMacroName(false);
+    PP.LexUnexpandedToken(PeekTok);
+  }
+  
   // If we don't have a pp-identifier now, this is an error.
   if ((II = PeekTok.getIdentifierInfo()) == 0) {
     PP.Diag(PeekTok, diag::err_pp_defined_requires_identifier);
@@ -138,6 +149,12 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
                           bool ValueLive, Preprocessor &PP) {
   DT.State = DefinedTracker::Unknown;
 
+  if (PeekTok.is(tok::code_completion)) {
+    if (PP.getCodeCompletionHandler())
+      PP.getCodeCompletionHandler()->CodeCompletePreprocessorExpression();
+    PP.LexUnexpandedToken(PeekTok);
+  }
+      
   // If this token's spelling is a pp-identifier, check to see if it is
   // 'defined' or if it is a macro.  Note that we check here because many
   // keywords are pp-identifiers, so we can't check the kind.
@@ -693,7 +710,7 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
   // Peek ahead one token.
   Token Tok;
   Lex(Tok);
-
+  
   // C99 6.10.1p3 - All expressions are evaluated as intmax_t or uintmax_t.
   unsigned BitWidth = getTargetInfo().getIntMaxTWidth();
 
diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index ebf606e9406f..9015c278fc92 100644
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -19,6 +19,7 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
@@ -71,6 +72,12 @@ void Preprocessor::RegisterBuiltinMacros() {
   Ident__has_builtin      = RegisterBuiltinMacro(*this, "__has_builtin");
   Ident__has_include      = RegisterBuiltinMacro(*this, "__has_include");
   Ident__has_include_next = RegisterBuiltinMacro(*this, "__has_include_next");
+
+  // Microsoft Extensions.
+  if (Features.Microsoft) 
+    Ident__pragma = RegisterBuiltinMacro(*this, "__pragma");
+  else
+    Ident__pragma = 0;
 }
 
 /// isTrivialSingleTokenExpansion - Return true if MI, which has a single token
@@ -323,6 +330,13 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
       // an argument value in a macro could expand to ',' or '(' or ')'.
       LexUnexpandedToken(Tok);
 
+      if (Tok.is(tok::code_completion)) {
+        if (CodeComplete)
+          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),
+                                                  MI, NumActuals);
+        LexUnexpandedToken(Tok);
+      }
+      
       if (Tok.is(tok::eof) || Tok.is(tok::eom)) { // "#if f(<eof>" & "#if f(\n"
         Diag(MacroName, diag::err_unterm_macro_invoc);
         // Do not lose the EOF/EOM.  Return it to the client.
@@ -506,6 +520,10 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
            .Case("cxx_static_assert", LangOpts.CPlusPlus0x)
            .Case("objc_nonfragile_abi", LangOpts.ObjCNonFragileABI)
            .Case("objc_weak_class", LangOpts.ObjCNonFragileABI)
+           .Case("ownership_holds", true)
+           .Case("ownership_returns", true)
+           .Case("ownership_takes", true)
+           .Case("cxx_inline_namespaces", true)
          //.Case("cxx_concepts", false)
          //.Case("cxx_lambdas", false)
          //.Case("cxx_nullptr", false)
@@ -630,10 +648,12 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
   IdentifierInfo *II = Tok.getIdentifierInfo();
   assert(II && "Can't be a macro without id info!");
 
-  // If this is an _Pragma directive, expand it, invoke the pragma handler, then
-  // lex the token after it.
+  // If this is an _Pragma or Microsoft __pragma directive, expand it,
+  // invoke the pragma handler, then lex the token after it.
   if (II == Ident_Pragma)
     return Handle_Pragma(Tok);
+  else if (II == Ident__pragma) // in non-MS mode this is null
+    return HandleMicrosoft__pragma(Tok);
 
   ++NumBuiltinMacroExpanded;
 
diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp
index 3b949d0ab40a..63b4823cf19e 100644
--- a/lib/Lex/PTHLexer.cpp
+++ b/lib/Lex/PTHLexer.cpp
@@ -101,16 +101,15 @@ LexNextToken:
     // Save the end-of-file token.
     EofToken = Tok;
 
+    // Save 'PP' to 'PPCache' as LexEndOfFile can delete 'this'.
     Preprocessor *PPCache = PP;
 
     assert(!ParsingPreprocessorDirective);
     assert(!LexingRawMode);
-
-    // FIXME: Issue diagnostics similar to Lexer.
-    if (PP->HandleEndOfFile(Tok, false))
+    
+    if (LexEndOfFile(Tok))
       return;
 
-    assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
     return PPCache->Lex(Tok);
   }
 
@@ -134,6 +133,29 @@ LexNextToken:
   MIOpt.ReadToken();
 }
 
+bool PTHLexer::LexEndOfFile(Token &Result) {
+  // If we hit the end of the file while parsing a preprocessor directive,
+  // end the preprocessor directive first.  The next token returned will
+  // then be the end of file.
+  if (ParsingPreprocessorDirective) {
+    ParsingPreprocessorDirective = false; // Done parsing the "line".
+    return true;  // Have a token.
+  }
+  
+  assert(!LexingRawMode);
+
+  // If we are in a #if directive, emit an error.
+  while (!ConditionalStack.empty()) {
+    if (!PP->isCodeCompletionFile(FileStartLoc))
+      PP->Diag(ConditionalStack.back().IfLoc,
+               diag::err_pp_unterminated_conditional);
+    ConditionalStack.pop_back();
+  }
+
+  // Finally, let the preprocessor handle this.
+  return PP->HandleEndOfFile(Result);
+}
+
 // FIXME: We can just grab the last token instead of storing a copy
 // into EofToken.
 void PTHLexer::getEOF(Token& Tok) {
diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index 7bf409405ab1..a7b289e137eb 100644
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -16,9 +16,12 @@
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
+#include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/ErrorHandling.h"
 #include <algorithm>
 using namespace clang;
 
@@ -166,6 +169,62 @@ void Preprocessor::Handle_Pragma(Token &Tok) {
       --e;
     }
   }
+  
+  Handle_Pragma(StrVal, PragmaLoc, RParenLoc);
+
+  // Finally, return whatever came after the pragma directive.
+  return Lex(Tok);
+}
+
+/// HandleMicrosoft__pragma - Like Handle_Pragma except the pragma text
+/// is not enclosed within a string literal.
+void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
+  // Remember the pragma token location.
+  SourceLocation PragmaLoc = Tok.getLocation();
+
+  // Read the '('.
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(PragmaLoc, diag::err__Pragma_malformed);
+    return;
+  }
+
+  // Get the tokens enclosed within the __pragma().
+  llvm::SmallVector<Token, 32> PragmaToks;
+  int NumParens = 0;
+  Lex(Tok);
+  while (Tok.isNot(tok::eof)) {
+    if (Tok.is(tok::l_paren))
+      NumParens++;
+    else if (Tok.is(tok::r_paren) && NumParens-- == 0)
+      break;
+    PragmaToks.push_back(Tok);
+    Lex(Tok);
+  }
+
+  if (Tok.is(tok::eof)) {
+    Diag(PragmaLoc, diag::err_unterminated___pragma);
+    return;
+  }
+
+  // Build the pragma string.
+  std::string StrVal = " ";
+  for (llvm::SmallVector<Token, 32>::iterator I =
+       PragmaToks.begin(), E = PragmaToks.end(); I != E; ++I) {
+    StrVal += getSpelling(*I);
+  }
+  
+  SourceLocation RParenLoc = Tok.getLocation();
+
+  Handle_Pragma(StrVal, PragmaLoc, RParenLoc);
+
+  // Finally, return whatever came after the pragma directive.
+  return Lex(Tok);
+}
+
+void Preprocessor::Handle_Pragma(const std::string &StrVal,
+                                 SourceLocation PragmaLoc,
+                                 SourceLocation RParenLoc) {
 
   // Plop the string (including the newline and trailing null) into a buffer
   // where we can lex it.
@@ -183,9 +242,6 @@ void Preprocessor::Handle_Pragma(Token &Tok) {
 
   // With everything set up, lex this as a #pragma directive.
   HandlePragmaDirective();
-
-  // Finally, return whatever came after the pragma directive.
-  return Lex(Tok);
 }
 
 
@@ -328,7 +384,9 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
       Lex(DependencyTok);
     }
 
-    Message.erase(Message.end()-1);
+    // Remove the trailing ' ' if present.
+    if (!Message.empty())
+      Message.erase(Message.end()-1);
     Diag(FilenameTok, diag::pp_out_of_date_dependency) << Message;
   }
 }
@@ -483,6 +541,109 @@ void Preprocessor::HandlePragmaMessage(Token &Tok) {
     Callbacks->PragmaMessage(MessageLoc, MessageString);
 }
 
+/// ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.  
+/// Return the IdentifierInfo* associated with the macro to push or pop.
+IdentifierInfo *Preprocessor::ParsePragmaPushOrPopMacro(Token &Tok) {
+  // Remember the pragma token location.
+  Token PragmaTok = Tok;
+
+  // Read the '('.
+  Lex(Tok);
+  if (Tok.isNot(tok::l_paren)) {
+    Diag(PragmaTok.getLocation(), diag::err_pragma_push_pop_macro_malformed)
+      << getSpelling(PragmaTok);
+    return 0;
+  }
+
+  // Read the macro name string.
+  Lex(Tok);
+  if (Tok.isNot(tok::string_literal)) {
+    Diag(PragmaTok.getLocation(), diag::err_pragma_push_pop_macro_malformed)
+      << getSpelling(PragmaTok);
+    return 0;
+  }
+
+  // Remember the macro string.
+  std::string StrVal = getSpelling(Tok);
+
+  // Read the ')'.
+  Lex(Tok);
+  if (Tok.isNot(tok::r_paren)) {
+    Diag(PragmaTok.getLocation(), diag::err_pragma_push_pop_macro_malformed)
+      << getSpelling(PragmaTok);
+    return 0;
+  }
+
+  assert(StrVal[0] == '"' && StrVal[StrVal.size()-1] == '"' &&
+         "Invalid string token!");
+
+  // Create a Token from the string.
+  Token MacroTok;
+  MacroTok.startToken();
+  MacroTok.setKind(tok::identifier);
+  CreateString(&StrVal[1], StrVal.size() - 2, MacroTok);
+
+  // Get the IdentifierInfo of MacroToPushTok.
+  return LookUpIdentifierInfo(MacroTok);
+}
+
+/// HandlePragmaPushMacro - Handle #pragma push_macro.  
+/// The syntax is:
+///   #pragma push_macro("macro")
+void Preprocessor::HandlePragmaPushMacro(Token &PushMacroTok) {
+  // Parse the pragma directive and get the macro IdentifierInfo*.
+  IdentifierInfo *IdentInfo = ParsePragmaPushOrPopMacro(PushMacroTok);
+  if (!IdentInfo) return;
+
+  // Get the MacroInfo associated with IdentInfo.
+  MacroInfo *MI = getMacroInfo(IdentInfo);
+ 
+  MacroInfo *MacroCopyToPush = 0;
+  if (MI) {
+    // Make a clone of MI.
+    MacroCopyToPush = CloneMacroInfo(*MI);
+    
+    // Allow the original MacroInfo to be redefined later.
+    MI->setIsAllowRedefinitionsWithoutWarning(true);
+  }
+
+  // Push the cloned MacroInfo so we can retrieve it later.
+  PragmaPushMacroInfo[IdentInfo].push_back(MacroCopyToPush);
+}
+
+/// HandlePragmaPopMacro - Handle #pragma push_macro.  
+/// The syntax is:
+///   #pragma pop_macro("macro")
+void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
+  SourceLocation MessageLoc = PopMacroTok.getLocation();
+
+  // Parse the pragma directive and get the macro IdentifierInfo*.
+  IdentifierInfo *IdentInfo = ParsePragmaPushOrPopMacro(PopMacroTok);
+  if (!IdentInfo) return;
+
+  // Find the vector<MacroInfo*> associated with the macro.
+  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> >::iterator iter =
+    PragmaPushMacroInfo.find(IdentInfo);
+  if (iter != PragmaPushMacroInfo.end()) {
+    // Release the MacroInfo currently associated with IdentInfo.
+    MacroInfo *CurrentMI = getMacroInfo(IdentInfo);
+    if (CurrentMI) ReleaseMacroInfo(CurrentMI);
+
+    // Get the MacroInfo we want to reinstall.
+    MacroInfo *MacroToReInstall = iter->second.back();
+
+    // Reinstall the previously pushed macro.
+    setMacroInfo(IdentInfo, MacroToReInstall);
+
+    // Pop PragmaPushMacroInfo stack.
+    iter->second.pop_back();
+    if (iter->second.size() == 0)
+      PragmaPushMacroInfo.erase(iter);
+  } else {
+    Diag(MessageLoc, diag::warn_pragma_pop_macro_no_push)
+      << IdentInfo->getName();
+  }
+}
 
 /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
 /// If 'Namespace' is non-null, then it is a token required to exist on the
@@ -582,24 +743,51 @@ struct PragmaDependencyHandler : public PragmaHandler {
   }
 };
 
+struct PragmaDebugHandler : public PragmaHandler {
+  PragmaDebugHandler() : PragmaHandler("__debug") {}
+  virtual void HandlePragma(Preprocessor &PP, Token &DepToken) {
+    Token Tok;
+    PP.LexUnexpandedToken(Tok);
+    if (Tok.isNot(tok::identifier)) {
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
+      return;
+    }
+    IdentifierInfo *II = Tok.getIdentifierInfo();
+
+    if (II->isStr("assert")) {
+      assert(0 && "This is an assertion!");
+    } else if (II->isStr("crash")) {
+      *(volatile int*) 0x11 = 0;
+    } else if (II->isStr("llvm_fatal_error")) {
+      llvm::report_fatal_error("#pragma clang __debug llvm_fatal_error");
+    } else if (II->isStr("llvm_unreachable")) {
+      llvm_unreachable("#pragma clang __debug llvm_unreachable");
+    } else if (II->isStr("overflow_stack")) {
+      DebugOverflowStack();
+    } else if (II->isStr("handle_crash")) {
+      llvm::CrashRecoveryContext *CRC =llvm::CrashRecoveryContext::GetCurrent();
+      if (CRC)
+        CRC->HandleCrash();
+    } else {
+      PP.Diag(Tok, diag::warn_pragma_debug_unexpected_command)
+        << II->getName();
+    }
+  }
+
+  void DebugOverflowStack() {
+    DebugOverflowStack();
+  }
+};
+
 /// PragmaDiagnosticHandler - e.g. '#pragma GCC diagnostic ignored "-Wformat"'
-/// Since clang's diagnostic supports extended functionality beyond GCC's
-/// the constructor takes a clangMode flag to tell it whether or not to allow
-/// clang's extended functionality, or whether to reject it.
 struct PragmaDiagnosticHandler : public PragmaHandler {
-private:
-  const bool ClangMode;
 public:
-  explicit PragmaDiagnosticHandler(const bool clangMode)
-    : PragmaHandler("diagnostic"), ClangMode(clangMode) {}
-
+  explicit PragmaDiagnosticHandler() : PragmaHandler("diagnostic") {}
   virtual void HandlePragma(Preprocessor &PP, Token &DiagToken) {
     Token Tok;
     PP.LexUnexpandedToken(Tok);
     if (Tok.isNot(tok::identifier)) {
-      unsigned Diag = ClangMode ? diag::warn_pragma_diagnostic_clang_invalid
-                                 : diag::warn_pragma_diagnostic_gcc_invalid;
-      PP.Diag(Tok, Diag);
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
       return;
     }
     IdentifierInfo *II = Tok.getIdentifierInfo();
@@ -613,22 +801,16 @@ public:
       Map = diag::MAP_IGNORE;
     else if (II->isStr("fatal"))
       Map = diag::MAP_FATAL;
-    else if (ClangMode) {
-      if (II->isStr("pop")) {
-        if (!PP.getDiagnostics().popMappings())
-          PP.Diag(Tok, diag::warn_pragma_diagnostic_clang_cannot_ppp);
-        return;
-      }
-
-      if (II->isStr("push")) {
-        PP.getDiagnostics().pushMappings();
-        return;
-      }
-
-      PP.Diag(Tok, diag::warn_pragma_diagnostic_clang_invalid);
+    else if (II->isStr("pop")) {
+      if (!PP.getDiagnostics().popMappings())
+        PP.Diag(Tok, diag::warn_pragma_diagnostic_cannot_pop);
+
+      return;
+    } else if (II->isStr("push")) {
+      PP.getDiagnostics().pushMappings();
       return;
     } else {
-      PP.Diag(Tok, diag::warn_pragma_diagnostic_gcc_invalid);
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
       return;
     }
 
@@ -660,9 +842,7 @@ public:
     if (Literal.hadError)
       return;
     if (Literal.Pascal) {
-      unsigned Diag = ClangMode ? diag::warn_pragma_diagnostic_clang_invalid
-                                 : diag::warn_pragma_diagnostic_gcc_invalid;
-      PP.Diag(Tok, Diag);
+      PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
       return;
     }
 
@@ -699,6 +879,25 @@ struct PragmaMessageHandler : public PragmaHandler {
   }
 };
 
+/// PragmaPushMacroHandler - "#pragma push_macro" saves the value of the
+/// macro on the top of the stack.
+struct PragmaPushMacroHandler : public PragmaHandler {
+  PragmaPushMacroHandler() : PragmaHandler("push_macro") {}
+  virtual void HandlePragma(Preprocessor &PP, Token &PushMacroTok) {
+    PP.HandlePragmaPushMacro(PushMacroTok);
+  }
+};
+
+
+/// PragmaPopMacroHandler - "#pragma pop_macro" sets the value of the
+/// macro to the value on the top of the stack.
+struct PragmaPopMacroHandler : public PragmaHandler {
+  PragmaPopMacroHandler() : PragmaHandler("pop_macro") {}
+  virtual void HandlePragma(Preprocessor &PP, Token &PopMacroTok) {
+    PP.HandlePragmaPopMacro(PopMacroTok);
+  }
+};
+
 // Pragma STDC implementations.
 
 enum STDCSetting {
@@ -780,17 +979,20 @@ struct PragmaSTDC_UnknownHandler : public PragmaHandler {
 void Preprocessor::RegisterBuiltinPragmas() {
   AddPragmaHandler(new PragmaOnceHandler());
   AddPragmaHandler(new PragmaMarkHandler());
+  AddPragmaHandler(new PragmaPushMacroHandler());
+  AddPragmaHandler(new PragmaPopMacroHandler());
 
   // #pragma GCC ...
   AddPragmaHandler("GCC", new PragmaPoisonHandler());
   AddPragmaHandler("GCC", new PragmaSystemHeaderHandler());
   AddPragmaHandler("GCC", new PragmaDependencyHandler());
-  AddPragmaHandler("GCC", new PragmaDiagnosticHandler(false));
+  AddPragmaHandler("GCC", new PragmaDiagnosticHandler());
   // #pragma clang ...
   AddPragmaHandler("clang", new PragmaPoisonHandler());
   AddPragmaHandler("clang", new PragmaSystemHeaderHandler());
+  AddPragmaHandler("clang", new PragmaDebugHandler());
   AddPragmaHandler("clang", new PragmaDependencyHandler());
-  AddPragmaHandler("clang", new PragmaDiagnosticHandler(true));
+  AddPragmaHandler("clang", new PragmaDiagnosticHandler());
 
   AddPragmaHandler("STDC", new PragmaSTDC_FP_CONTRACTHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler());
diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp
index 6966c38b23d8..c446d96b4527 100644
--- a/lib/Lex/PreprocessingRecord.cpp
+++ b/lib/Lex/PreprocessingRecord.cpp
@@ -118,7 +118,8 @@ void PreprocessingRecord::MacroDefined(const IdentifierInfo *II,
   PreprocessedEntities.push_back(Def);
 }
 
-void PreprocessingRecord::MacroUndefined(const IdentifierInfo *II, 
+void PreprocessingRecord::MacroUndefined(SourceLocation Loc,
+                                         const IdentifierInfo *II,
                                          const MacroInfo *MI) {
   llvm::DenseMap<const MacroInfo *, MacroDefinition *>::iterator Pos
     = MacroDefinitions.find(MI);
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 51f729334cb9..5160acf19e1f 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -34,6 +34,7 @@
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/ScratchBuffer.h"
 #include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -53,8 +54,9 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            bool OwnsHeaders)
   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
     SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0),
-    Identifiers(opts, IILookup), BuiltinInfo(Target), CodeCompletionFile(0),
-    CurPPLexer(0), CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0) {
+    Identifiers(opts, IILookup), BuiltinInfo(Target), CodeComplete(0),
+    CodeCompletionFile(0), SkipMainFilePreamble(0, true), CurPPLexer(0), 
+    CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0) {
   ScratchBuf = new ScratchBuffer(SourceMgr);
   CounterValue = 0; // __COUNTER__ starts at 0.
   OwnsHeaderSearch = OwnsHeaders;
@@ -110,7 +112,7 @@ Preprocessor::~Preprocessor() {
     // will be released when the BumpPtrAllocator 'BP' object gets
     // destroyed.  We still need to run the dtor, however, to free
     // memory alocated by MacroInfo.
-    I->second->Destroy(BP);
+    I->second->Destroy();
     I->first->setHasMacroDefinition(false);
   }
   for (std::vector<MacroInfo*>::iterator I = MICache.begin(),
@@ -119,7 +121,7 @@ Preprocessor::~Preprocessor() {
     // will be released when the BumpPtrAllocator 'BP' object gets
     // destroyed.  We still need to run the dtor, however, to free
     // memory alocated by MacroInfo.
-    (*I)->Destroy(BP);
+    (*I)->Destroy();
   }
 
   // Free any cached macro expanders.
@@ -163,7 +165,7 @@ void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
     llvm::errs() << " [ExpandDisabled]";
   if (Tok.needsCleaning()) {
     const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
-    llvm::errs() << " [UnClean='" << std::string(Start, Start+Tok.getLength())
+    llvm::errs() << " [UnClean='" << llvm::StringRef(Start, Tok.getLength())
                  << "']";
   }
 
@@ -282,6 +284,13 @@ bool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const {
       == CodeCompletionFile;
 }
 
+void Preprocessor::CodeCompleteNaturalLanguage() {
+  SetCodeCompletionPoint(0, 0, 0);
+  getDiagnostics().setSuppressAllDiagnostics(true);
+  if (CodeComplete)
+    CodeComplete->CodeCompleteNaturalLanguage();
+}
+
 //===----------------------------------------------------------------------===//
 // Token Spelling
 //===----------------------------------------------------------------------===//
@@ -508,6 +517,12 @@ void Preprocessor::EnterMainSourceFile() {
   // Enter the main file source buffer.
   EnterSourceFile(MainFileID, 0, SourceLocation());
 
+  // If we've been asked to skip bytes in the main file (e.g., as part of a
+  // precompiled preamble), do so now.
+  if (SkipMainFilePreamble.first > 0)
+    CurLexer->SkipBytes(SkipMainFilePreamble.first, 
+                        SkipMainFilePreamble.second);
+  
   // Tell the header info that the main file was entered.  If the file is later
   // #imported, it won't be re-entered.
   if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
@@ -516,7 +531,7 @@ void Preprocessor::EnterMainSourceFile() {
   // Preprocess Predefines to populate the initial preprocessor state.
   llvm::MemoryBuffer *SB =
     llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
-  assert(SB && "Cannot fail to create predefined source buffer");
+  assert(SB && "Cannot create predefined source buffer");
   FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
   assert(!FID.isInvalid() && "Could not create FileID for predefines?");
 
@@ -639,6 +654,8 @@ bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
 
 CommentHandler::~CommentHandler() { }
 
+CodeCompletionHandler::~CodeCompletionHandler() { }
+
 void Preprocessor::createPreprocessingRecord() {
   if (Record)
     return;
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index 56bb073e5919..94719b0baa3c 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -268,6 +268,13 @@ void TokenLexer::ExpandFunctionArguments() {
       // Remove the paste operator, report use of the extension.
       PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
       ResultToks.pop_back();
+      
+      // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
+      // then removal of the comma should produce a placemarker token (in C99
+      // terms) which we model by popping off the previous ##, giving us a plain
+      // "X" when __VA_ARGS__ is empty.
+      if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash))
+        ResultToks.pop_back();
     }
     continue;
   }
@@ -478,7 +485,7 @@ bool TokenLexer::PasteTokens(Token &Tok) {
           return true;
         }
 
-        // Do not emit the warning when preprocessing assembler code.
+        // Do not emit the error when preprocessing assembler code.
         if (!PP.getLangOptions().AsmPreprocessor) {
           // Explicitly convert the token location to have proper instantiation
           // information so that the user knows where it came from.
@@ -486,8 +493,13 @@ bool TokenLexer::PasteTokens(Token &Tok) {
           SourceLocation Loc =
             SM.createInstantiationLoc(PasteOpLoc, InstantiateLocStart,
                                       InstantiateLocEnd, 2);
-          PP.Diag(Loc, diag::err_pp_bad_paste)
-            << std::string(Buffer.begin(), Buffer.end());
+          // If we're in microsoft extensions mode, downgrade this from a hard
+          // error to a warning that defaults to an error.  This allows
+          // disabling it.
+          PP.Diag(Loc,
+                  PP.getLangOptions().Microsoft ? diag::err_pp_bad_paste_ms 
+                                                : diag::err_pp_bad_paste)
+            << Buffer.str();
         }
 
         // Do not consume the RHS.
diff --git a/lib/Makefile b/lib/Makefile
index 4fca6249ba3d..dbd0eb699ee1 100755
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -9,7 +9,7 @@
 CLANG_LEVEL := ..
 
 PARALLEL_DIRS = Headers Basic Lex Parse AST Sema CodeGen Analysis \
-                Checker Rewrite Frontend Index Driver
+                Checker Rewrite Serialization Frontend FrontendTool Index Driver
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Parse/CMakeLists.txt b/lib/Parse/CMakeLists.txt
index fafcf77b006c..189af3dc882f 100644
--- a/lib/Parse/CMakeLists.txt
+++ b/lib/Parse/CMakeLists.txt
@@ -1,9 +1,7 @@
 set(LLVM_NO_RTTI 1)
 
 add_clang_library(clangParse
-  AttributeList.cpp
-  DeclSpec.cpp
-  MinimalAction.cpp
+  ParseAST.cpp
   ParseCXXInlineMethods.cpp
   ParseDecl.cpp
   ParseDeclCXX.cpp
@@ -18,4 +16,4 @@ add_clang_library(clangParse
   Parser.cpp
   )
 
-add_dependencies(clangParse ClangAttrList ClangDiagnosticParse)
+add_dependencies(clangParse ClangAttrClasses ClangAttrList ClangDeclNodes ClangDiagnosticParse ClangStmtNodes)
diff --git a/lib/Parse/Makefile b/lib/Parse/Makefile
index 238e02d9996c..5ec7c333c504 100644
--- a/lib/Parse/Makefile
+++ b/lib/Parse/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangParse
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Parse/MinimalAction.cpp b/lib/Parse/MinimalAction.cpp
deleted file mode 100644
index b7205160c177..000000000000
--- a/lib/Parse/MinimalAction.cpp
+++ /dev/null
@@ -1,281 +0,0 @@
-//===--- MinimalAction.cpp - Implement the MinimalAction class ------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file implements the MinimalAction interface.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Parse/Parser.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Basic/TargetInfo.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/RecyclingAllocator.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace clang;
-
-///  Out-of-line virtual destructor to provide home for ActionBase class.
-ActionBase::~ActionBase() {}
-
-///  Out-of-line virtual destructor to provide home for Action class.
-Action::~Action() {}
-
-Action::ObjCMessageKind Action::getObjCMessageKind(Scope *S,
-                                                   IdentifierInfo *Name,
-                                                   SourceLocation NameLoc,
-                                                   bool IsSuper,
-                                                   bool HasTrailingDot,
-                                                   TypeTy *&ReceiverType) {
-  ReceiverType = 0;
-
-  if (IsSuper && !HasTrailingDot && S->isInObjcMethodScope())
-    return ObjCSuperMessage;
-      
-  if (TypeTy *TyName = getTypeName(*Name, NameLoc, S)) {
-    DeclSpec DS;
-    const char *PrevSpec = 0;
-    unsigned DiagID = 0;
-    if (!DS.SetTypeSpecType(DeclSpec::TST_typename, NameLoc, PrevSpec,
-                            DiagID, TyName)) {
-      DS.SetRangeEnd(NameLoc);
-      Declarator DeclaratorInfo(DS, Declarator::TypeNameContext);
-      TypeResult Ty = ActOnTypeName(S, DeclaratorInfo);
-      if (!Ty.isInvalid())
-        ReceiverType = Ty.get();
-    }
-    return ObjCClassMessage;
-  }
-      
-  return ObjCInstanceMessage;
-}
-
-// Defined out-of-line here because of dependecy on AttributeList
-Action::DeclPtrTy Action::ActOnUsingDirective(Scope *CurScope,
-                                              SourceLocation UsingLoc,
-                                              SourceLocation NamespcLoc,
-                                              CXXScopeSpec &SS,
-                                              SourceLocation IdentLoc,
-                                              IdentifierInfo *NamespcName,
-                                              AttributeList *AttrList) {
-
-  // FIXME: Parser seems to assume that Action::ActOn* takes ownership over
-  // passed AttributeList, however other actions don't free it, is it
-  // temporary state or bug?
-  delete AttrList;
-  return DeclPtrTy();
-}
-
-// Defined out-of-line here because of dependency on AttributeList
-Action::DeclPtrTy Action::ActOnUsingDeclaration(Scope *CurScope,
-                                                AccessSpecifier AS,
-                                                bool HasUsingKeyword,
-                                                SourceLocation UsingLoc,
-                                                CXXScopeSpec &SS,
-                                                UnqualifiedId &Name,
-                                                AttributeList *AttrList,
-                                                bool IsTypeName,
-                                                SourceLocation TypenameLoc) {
-
-  // FIXME: Parser seems to assume that Action::ActOn* takes ownership over
-  // passed AttributeList, however other actions don't free it, is it
-  // temporary state or bug?
-  delete AttrList;
-  return DeclPtrTy();
-}
-
-
-void PrettyStackTraceActionsDecl::print(llvm::raw_ostream &OS) const {
-  if (Loc.isValid()) {
-    Loc.print(OS, SM);
-    OS << ": ";
-  }
-  OS << Message;
-
-  std::string Name = Actions.getDeclName(TheDecl);
-  if (!Name.empty())
-    OS << " '" << Name << '\'';
-
-  OS << '\n';
-}
-
-/// TypeNameInfo - A link exists here for each scope that an identifier is
-/// defined.
-namespace {
-  struct TypeNameInfo {
-    TypeNameInfo *Prev;
-    bool isTypeName;
-
-    TypeNameInfo(bool istypename, TypeNameInfo *prev) {
-      isTypeName = istypename;
-      Prev = prev;
-    }
-  };
-
-  struct TypeNameInfoTable {
-    llvm::RecyclingAllocator<llvm::BumpPtrAllocator, TypeNameInfo> Allocator;
-
-    void AddEntry(bool isTypename, IdentifierInfo *II) {
-      TypeNameInfo *TI = Allocator.Allocate<TypeNameInfo>();
-      new (TI) TypeNameInfo(isTypename, II->getFETokenInfo<TypeNameInfo>());
-      II->setFETokenInfo(TI);
-    }
-
-    void DeleteEntry(TypeNameInfo *Entry) {
-      Entry->~TypeNameInfo();
-      Allocator.Deallocate(Entry);
-    }
-  };
-}
-
-static TypeNameInfoTable *getTable(void *TP) {
-  return static_cast<TypeNameInfoTable*>(TP);
-}
-
-MinimalAction::MinimalAction(Preprocessor &pp)
-  : Idents(pp.getIdentifierTable()), PP(pp) {
-  TypeNameInfoTablePtr = new TypeNameInfoTable();
-}
-
-MinimalAction::~MinimalAction() {
-  delete getTable(TypeNameInfoTablePtr);
-}
-
-void MinimalAction::ActOnTranslationUnitScope(SourceLocation Loc, Scope *S) {
-  TUScope = S;
-
-  TypeNameInfoTable &TNIT = *getTable(TypeNameInfoTablePtr);
-
-  if (PP.getTargetInfo().getPointerWidth(0) >= 64) {
-    // Install [u]int128_t for 64-bit targets.
-    TNIT.AddEntry(true, &Idents.get("__int128_t"));
-    TNIT.AddEntry(true, &Idents.get("__uint128_t"));
-  }
-
-  if (PP.getLangOptions().ObjC1) {
-    // Recognize the ObjC built-in type identifiers as types.
-    TNIT.AddEntry(true, &Idents.get("id"));
-    TNIT.AddEntry(true, &Idents.get("SEL"));
-    TNIT.AddEntry(true, &Idents.get("Class"));
-    TNIT.AddEntry(true, &Idents.get("Protocol"));
-  }
-}
-
-/// isTypeName - This looks at the IdentifierInfo::FETokenInfo field to
-/// determine whether the name is a type name (objc class name or typedef) or
-/// not in this scope.
-///
-/// FIXME: Use the passed CXXScopeSpec for accurate C++ type checking.
-Action::TypeTy *
-MinimalAction::getTypeName(IdentifierInfo &II, SourceLocation Loc,
-                           Scope *S, CXXScopeSpec *SS,
-                           bool isClassName, TypeTy *ObjectType) {
-  if (TypeNameInfo *TI = II.getFETokenInfo<TypeNameInfo>())
-    if (TI->isTypeName)
-      return TI;
-  return 0;
-}
-
-/// isCurrentClassName - Always returns false, because MinimalAction
-/// does not support C++ classes with constructors.
-bool MinimalAction::isCurrentClassName(const IdentifierInfo &, Scope *,
-                                       const CXXScopeSpec *) {
-  return false;
-}
-
-TemplateNameKind
-MinimalAction::isTemplateName(Scope *S,
-                              CXXScopeSpec &SS,
-                              UnqualifiedId &Name,
-                              TypeTy *ObjectType,
-                              bool EnteringScope,
-                              TemplateTy &TemplateDecl,
-                              bool &MemberOfUnknownSpecialization) {
-  MemberOfUnknownSpecialization = false;
-  return TNK_Non_template;
-}
-
-/// ActOnDeclarator - If this is a typedef declarator, we modify the
-/// IdentifierInfo::FETokenInfo field to keep track of this fact, until S is
-/// popped.
-Action::DeclPtrTy
-MinimalAction::ActOnDeclarator(Scope *S, Declarator &D) {
-  IdentifierInfo *II = D.getIdentifier();
-
-  // If there is no identifier associated with this declarator, bail out.
-  if (II == 0) return DeclPtrTy();
-
-  TypeNameInfo *weCurrentlyHaveTypeInfo = II->getFETokenInfo<TypeNameInfo>();
-  bool isTypeName =
-    D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_typedef;
-
-  // this check avoids creating TypeNameInfo objects for the common case.
-  // It does need to handle the uncommon case of shadowing a typedef name with a
-  // non-typedef name. e.g. { typedef int a; a xx; { int a; } }
-  if (weCurrentlyHaveTypeInfo || isTypeName) {
-    // Allocate and add the 'TypeNameInfo' "decl".
-    getTable(TypeNameInfoTablePtr)->AddEntry(isTypeName, II);
-
-    // Remember that this needs to be removed when the scope is popped.
-    S->AddDecl(DeclPtrTy::make(II));
-  }
-  return DeclPtrTy();
-}
-
-Action::DeclPtrTy
-MinimalAction::ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
-                                        IdentifierInfo *ClassName,
-                                        SourceLocation ClassLoc,
-                                        IdentifierInfo *SuperName,
-                                        SourceLocation SuperLoc,
-                                        const DeclPtrTy *ProtoRefs,
-                                        unsigned NumProtocols,
-                                        const SourceLocation *ProtoLocs,
-                                        SourceLocation EndProtoLoc,
-                                        AttributeList *AttrList) {
-  // Allocate and add the 'TypeNameInfo' "decl".
-  getTable(TypeNameInfoTablePtr)->AddEntry(true, ClassName);
-  return DeclPtrTy();
-}
-
-/// ActOnForwardClassDeclaration -
-/// Scope will always be top level file scope.
-Action::DeclPtrTy
-MinimalAction::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
-                                            IdentifierInfo **IdentList,
-                                            SourceLocation *IdentLocs,
-                                            unsigned NumElts) {
-  for (unsigned i = 0; i != NumElts; ++i) {
-    // Allocate and add the 'TypeNameInfo' "decl".
-    getTable(TypeNameInfoTablePtr)->AddEntry(true, IdentList[i]);
-
-    // Remember that this needs to be removed when the scope is popped.
-    TUScope->AddDecl(DeclPtrTy::make(IdentList[i]));
-  }
-  return DeclPtrTy();
-}
-
-/// ActOnPopScope - When a scope is popped, if any typedefs are now
-/// out-of-scope, they are removed from the IdentifierInfo::FETokenInfo field.
-void MinimalAction::ActOnPopScope(SourceLocation Loc, Scope *S) {
-  TypeNameInfoTable &Table = *getTable(TypeNameInfoTablePtr);
-
-  for (Scope::decl_iterator I = S->decl_begin(), E = S->decl_end();
-       I != E; ++I) {
-    IdentifierInfo &II = *(*I).getAs<IdentifierInfo>();
-    TypeNameInfo *TI = II.getFETokenInfo<TypeNameInfo>();
-    assert(TI && "This decl didn't get pushed??");
-
-    if (TI) {
-      TypeNameInfo *Next = TI->Prev;
-      Table.DeleteEntry(TI);
-
-      II.setFETokenInfo(Next);
-    }
-  }
-}
diff --git a/lib/Sema/ParseAST.cpp b/lib/Parse/ParseAST.cpp
index bb0bd9e1cb5e..d02787941b0d 100644
--- a/lib/Sema/ParseAST.cpp
+++ b/lib/Parse/ParseAST.cpp
@@ -11,12 +11,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Sema/ParseAST.h"
-#include "Sema.h"
+#include "clang/Parse/ParseAST.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/Sema/ExternalSemaSource.h"
 #include "clang/AST/ASTConsumer.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/AST/Stmt.h"
 #include "clang/Parse/Parser.h"
@@ -56,68 +57,56 @@ void clang::ParseAST(Preprocessor &PP, ASTConsumer *Consumer,
                      ASTContext &Ctx, bool PrintStats,
                      bool CompleteTranslationUnit,
                      CodeCompleteConsumer *CompletionConsumer) {
+  Sema S(PP, Ctx, *Consumer, CompleteTranslationUnit, CompletionConsumer);
+  ParseAST(S, PrintStats);
+}
+
+void clang::ParseAST(Sema &S, bool PrintStats) {
   // Collect global stats on Decls/Stmts (until we have a module streamer).
   if (PrintStats) {
     Decl::CollectingStats(true);
     Stmt::CollectingStats(true);
   }
 
-  Sema S(PP, Ctx, *Consumer, CompleteTranslationUnit, CompletionConsumer);
-  Parser P(PP, S);
-  PP.EnterMainSourceFile();
+  ASTConsumer *Consumer = &S.getASTConsumer();
 
-  // Initialize the parser.
+  Parser P(S.getPreprocessor(), S);
+  S.getPreprocessor().EnterMainSourceFile();
   P.Initialize();
-
-  Consumer->Initialize(Ctx);
-
-  if (SemaConsumer *SC = dyn_cast<SemaConsumer>(Consumer))
-    SC->InitializeSema(S);
-
-  if (ExternalASTSource *External = Ctx.getExternalSource()) {
-    if (ExternalSemaSource *ExternalSema =
-          dyn_cast<ExternalSemaSource>(External))
-      ExternalSema->InitializeSema(S);
-
+  S.Initialize();
+  
+  if (ExternalASTSource *External = S.getASTContext().getExternalSource())
     External->StartTranslationUnit(Consumer);
-  }
-
+  
   Parser::DeclGroupPtrTy ADecl;
-
+  
   while (!P.ParseTopLevelDecl(ADecl)) {  // Not end of file.
     // If we got a null return and something *was* parsed, ignore it.  This
     // is due to a top-level semicolon, an action override, or a parse error
     // skipping something.
     if (ADecl)
-      Consumer->HandleTopLevelDecl(ADecl.getAsVal<DeclGroupRef>());
+      Consumer->HandleTopLevelDecl(ADecl.get());
   };
   // Check for any pending objective-c implementation decl.
-  while ((ADecl = P.RetrievePendingObjCImpDecl()))
-    Consumer->HandleTopLevelDecl(ADecl.getAsVal<DeclGroupRef>());
-
+  while ((ADecl = P.FinishPendingObjCActions()))
+    Consumer->HandleTopLevelDecl(ADecl.get());
+  
   // Process any TopLevelDecls generated by #pragma weak.
   for (llvm::SmallVector<Decl*,2>::iterator
-        I = S.WeakTopLevelDecls().begin(),
-        E = S.WeakTopLevelDecls().end(); I != E; ++I)
+       I = S.WeakTopLevelDecls().begin(),
+       E = S.WeakTopLevelDecls().end(); I != E; ++I)
     Consumer->HandleTopLevelDecl(DeclGroupRef(*I));
-
+  
   // Dump record layouts, if requested.
-  if (PP.getLangOptions().DumpRecordLayouts)
-    DumpRecordLayouts(Ctx);
-
-  Consumer->HandleTranslationUnit(Ctx);
-
-  if (ExternalSemaSource *ESS =
-        dyn_cast_or_null<ExternalSemaSource>(Ctx.getExternalSource()))
-    ESS->ForgetSema();
-
-  if (SemaConsumer *SC = dyn_cast<SemaConsumer>(Consumer))
-    SC->ForgetSema();
-
+  if (S.getLangOptions().DumpRecordLayouts)
+    DumpRecordLayouts(S.getASTContext());
+  
+  Consumer->HandleTranslationUnit(S.getASTContext());
+  
   if (PrintStats) {
     fprintf(stderr, "\nSTATISTICS:\n");
     P.getActions().PrintStats();
-    Ctx.PrintStats();
+    S.getASTContext().PrintStats();
     Decl::PrintStats();
     Stmt::PrintStats();
     Consumer->PrintStats();
diff --git a/lib/Parse/ParseCXXInlineMethods.cpp b/lib/Parse/ParseCXXInlineMethods.cpp
index 62a7ecd5d490..d327db485c09 100644
--- a/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/lib/Parse/ParseCXXInlineMethods.cpp
@@ -13,26 +13,25 @@
 
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Scope.h"
 using namespace clang;
 
 /// ParseCXXInlineMethodDef - We parsed and verified that the specified
 /// Declarator is a well formed C++ inline method definition. Now lex its body
 /// and store its tokens for parsing after the C++ class is complete.
-Parser::DeclPtrTy
-Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, Declarator &D,
+Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS, Declarator &D,
                                 const ParsedTemplateInfo &TemplateInfo) {
   assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
          "This isn't a function declarator!");
   assert((Tok.is(tok::l_brace) || Tok.is(tok::colon) || Tok.is(tok::kw_try)) &&
          "Current token not a '{', ':' or 'try'!");
 
-  Action::MultiTemplateParamsArg TemplateParams(Actions,
+  MultiTemplateParamsArg TemplateParams(Actions,
           TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->data() : 0,
           TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->size() : 0);
 
-  DeclPtrTy FnD;
+  Decl *FnD;
   if (D.getDeclSpec().isFriendSpecified())
     // FIXME: Friend templates
     FnD = Actions.ActOnFriendFunctionDecl(getCurScope(), D, true,
@@ -139,12 +138,17 @@ void Parser::ParseLexedMethodDeclarations(ParsingClass &Class) {
         assert(Tok.is(tok::equal) && "Default argument not starting with '='");
         SourceLocation EqualLoc = ConsumeToken();
 
-        OwningExprResult DefArgResult(ParseAssignmentExpression());
+        ExprResult DefArgResult(ParseAssignmentExpression());
         if (DefArgResult.isInvalid())
           Actions.ActOnParamDefaultArgumentError(LM.DefaultArgs[I].Param);
-        else
+        else {
+          if (Tok.is(tok::cxx_defaultarg_end))
+            ConsumeToken();
+          else
+            Diag(Tok.getLocation(), diag::err_default_arg_unparsed);
           Actions.ActOnParamDefaultArgument(LM.DefaultArgs[I].Param, EqualLoc,
-                                            move(DefArgResult));
+                                            DefArgResult.take());
+        }
 
         assert(!PP.getSourceManager().isBeforeInTranslationUnit(origLoc,
                                                            Tok.getLocation()) &&
@@ -227,7 +231,7 @@ void Parser::ParseLexedMethodDefs(ParsingClass &Class) {
 
       // Error recovery.
       if (!Tok.is(tok::l_brace)) {
-        Actions.ActOnFinishFunctionBody(LM.D, Action::StmtArg(Actions));
+        Actions.ActOnFinishFunctionBody(LM.D, 0);
         continue;
       }
     } else
diff --git a/lib/Parse/ParseDecl.cpp b/lib/Parse/ParseDecl.cpp
index 62ef3ec0179b..555fcf0dec55 100644
--- a/lib/Parse/ParseDecl.cpp
+++ b/lib/Parse/ParseDecl.cpp
@@ -13,8 +13,9 @@
 
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/PrettyDeclStackTrace.h"
 #include "RAIIObjectsForParser.h"
 #include "llvm/ADT/SmallSet.h"
 using namespace clang;
@@ -28,7 +29,7 @@ using namespace clang;
 ///         specifier-qualifier-list abstract-declarator[opt]
 ///
 /// Called type-id in C++.
-Action::TypeResult Parser::ParseTypeName(SourceRange *Range) {
+TypeResult Parser::ParseTypeName(SourceRange *Range) {
   // Parse the common declaration-specifiers piece.
   DeclSpec DS;
   ParseSpecifierQualifierList(DS);
@@ -131,7 +132,7 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
 
             // now parse the non-empty comma separated list of expressions
             while (1) {
-              OwningExprResult ArgExpr(ParseAssignmentExpression());
+              ExprResult ArgExpr(ParseAssignmentExpression());
               if (ArgExpr.isInvalid()) {
                 ArgExprsOk = false;
                 SkipUntil(tok::r_paren);
@@ -174,11 +175,13 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
           case tok::kw_double:
           case tok::kw_void:
           case tok::kw_typeof:
+            CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                         0, SourceLocation(), 0, 0, CurrAttr);
+            if (CurrAttr->getKind() == AttributeList::AT_IBOutletCollection)
+              Diag(Tok, diag::err_iboutletcollection_builtintype);
             // If it's a builtin type name, eat it and expect a rparen
             // __attribute__(( vec_type_hint(char) ))
             ConsumeToken();
-            CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc,
-                                         0, SourceLocation(), 0, 0, CurrAttr);
             if (Tok.is(tok::r_paren))
               ConsumeParen();
             break;
@@ -189,7 +192,7 @@ AttributeList *Parser::ParseGNUAttributes(SourceLocation *EndLoc) {
 
             // now parse the list of expressions
             while (1) {
-              OwningExprResult ArgExpr(ParseAssignmentExpression());
+              ExprResult ArgExpr(ParseAssignmentExpression());
               if (ArgExpr.isInvalid()) {
                 ArgExprsOk = false;
                 SkipUntil(tok::r_paren);
@@ -254,9 +257,9 @@ AttributeList* Parser::ParseMicrosoftDeclSpec(AttributeList *CurrAttr) {
       ConsumeParen();
       // FIXME: This doesn't parse __declspec(property(get=get_func_name))
       // correctly.
-      OwningExprResult ArgExpr(ParseAssignmentExpression());
+      ExprResult ArgExpr(ParseAssignmentExpression());
       if (!ArgExpr.isInvalid()) {
-        ExprTy* ExprList = ArgExpr.take();
+        Expr *ExprList = ArgExpr.take();
         CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
                                      SourceLocation(), &ExprList, 1,
                                      CurrAttr, true);
@@ -290,6 +293,17 @@ AttributeList* Parser::ParseMicrosoftTypeAttributes(AttributeList *CurrAttr) {
   return CurrAttr;
 }
 
+AttributeList* Parser::ParseBorlandTypeAttributes(AttributeList *CurrAttr) {
+  // Treat these like attributes
+  while (Tok.is(tok::kw___pascal)) {
+    IdentifierInfo *AttrName = Tok.getIdentifierInfo();
+    SourceLocation AttrNameLoc = ConsumeToken();
+    CurrAttr = new AttributeList(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
+                                 SourceLocation(), 0, 0, CurrAttr, true);
+  }
+  return CurrAttr;
+}
+
 /// ParseDeclaration - Parse a full 'declaration', which consists of
 /// declaration-specifiers, some number of declarators, and a semicolon.
 /// 'Context' should be a Declarator::TheContext value.  This returns the
@@ -311,7 +325,7 @@ Parser::DeclGroupPtrTy Parser::ParseDeclaration(unsigned Context,
                                                 CXX0XAttributeList Attr) {
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
   
-  DeclPtrTy SingleDecl;
+  Decl *SingleDecl = 0;
   switch (Tok.getKind()) {
   case tok::kw_template:
   case tok::kw_export:
@@ -320,6 +334,17 @@ Parser::DeclGroupPtrTy Parser::ParseDeclaration(unsigned Context,
         << Attr.Range;
     SingleDecl = ParseDeclarationStartingWithTemplate(Context, DeclEnd);
     break;
+  case tok::kw_inline:
+    // Could be the start of an inline namespace. Allowed as an ext in C++03.
+    if (getLang().CPlusPlus && NextToken().is(tok::kw_namespace)) {
+      if (Attr.HasAttr)
+        Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
+          << Attr.Range;
+      SourceLocation InlineLoc = ConsumeToken();
+      SingleDecl = ParseNamespace(Context, DeclEnd, InlineLoc);
+      break;
+    }
+    return ParseSimpleDeclaration(Context, DeclEnd, Attr.AttrList, true);
   case tok::kw_namespace:
     if (Attr.HasAttr)
       Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
@@ -366,7 +391,7 @@ Parser::DeclGroupPtrTy Parser::ParseSimpleDeclaration(unsigned Context,
   // declaration-specifiers init-declarator-list[opt] ';'
   if (Tok.is(tok::semi)) {
     if (RequireSemi) ConsumeToken();
-    DeclPtrTy TheDecl = Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS_none,
+    Decl *TheDecl = Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS_none,
                                                            DS);
     DS.complete(TheDecl);
     return Actions.ConvertDeclToDeclGroup(TheDecl);
@@ -410,7 +435,7 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
         DS.ClearStorageClassSpecs();
       }
 
-      DeclPtrTy TheDecl = ParseFunctionDefinition(D);
+      Decl *TheDecl = ParseFunctionDefinition(D);
       return Actions.ConvertDeclToDeclGroup(TheDecl);
     }
     
@@ -427,10 +452,10 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
     }
   }
 
-  llvm::SmallVector<DeclPtrTy, 8> DeclsInGroup;
-  DeclPtrTy FirstDecl = ParseDeclarationAfterDeclarator(D);
+  llvm::SmallVector<Decl *, 8> DeclsInGroup;
+  Decl *FirstDecl = ParseDeclarationAfterDeclarator(D);
   D.complete(FirstDecl);
-  if (FirstDecl.get())
+  if (FirstDecl)
     DeclsInGroup.push_back(FirstDecl);
 
   // If we don't have a comma, it is either the end of the list (a ';') or an
@@ -457,9 +482,9 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
 
     ParseDeclarator(D);
 
-    DeclPtrTy ThisDecl = ParseDeclarationAfterDeclarator(D);
+    Decl *ThisDecl = ParseDeclarationAfterDeclarator(D);
     D.complete(ThisDecl);
-    if (ThisDecl.get())
+    if (ThisDecl)
       DeclsInGroup.push_back(ThisDecl);    
   }
 
@@ -507,15 +532,15 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
 /// According to the standard grammar, =default and =delete are function
 /// definitions, but that definitely doesn't fit with the parser here.
 ///
-Parser::DeclPtrTy Parser::ParseDeclarationAfterDeclarator(Declarator &D,
+Decl *Parser::ParseDeclarationAfterDeclarator(Declarator &D,
                                      const ParsedTemplateInfo &TemplateInfo) {
   // If a simple-asm-expr is present, parse it.
   if (Tok.is(tok::kw_asm)) {
     SourceLocation Loc;
-    OwningExprResult AsmLabel(ParseSimpleAsm(&Loc));
+    ExprResult AsmLabel(ParseSimpleAsm(&Loc));
     if (AsmLabel.isInvalid()) {
       SkipUntil(tok::semi, true, true);
-      return DeclPtrTy();
+      return 0;
     }
 
     D.setAsmLabel(AsmLabel.release());
@@ -530,7 +555,7 @@ Parser::DeclPtrTy Parser::ParseDeclarationAfterDeclarator(Declarator &D,
   }
 
   // Inform the current actions module that we just parsed this declarator.
-  DeclPtrTy ThisDecl;
+  Decl *ThisDecl = 0;
   switch (TemplateInfo.Kind) {
   case ParsedTemplateInfo::NonTemplate:
     ThisDecl = Actions.ActOnDeclarator(getCurScope(), D);
@@ -539,21 +564,21 @@ Parser::DeclPtrTy Parser::ParseDeclarationAfterDeclarator(Declarator &D,
   case ParsedTemplateInfo::Template:
   case ParsedTemplateInfo::ExplicitSpecialization:
     ThisDecl = Actions.ActOnTemplateDeclarator(getCurScope(),
-                             Action::MultiTemplateParamsArg(Actions,
+                             MultiTemplateParamsArg(Actions,
                                           TemplateInfo.TemplateParams->data(),
                                           TemplateInfo.TemplateParams->size()),
                                                D);
     break;
       
   case ParsedTemplateInfo::ExplicitInstantiation: {
-    Action::DeclResult ThisRes 
+    DeclResult ThisRes 
       = Actions.ActOnExplicitInstantiation(getCurScope(),
                                            TemplateInfo.ExternLoc,
                                            TemplateInfo.TemplateLoc,
                                            D);
     if (ThisRes.isInvalid()) {
       SkipUntil(tok::semi, true, true);
-      return DeclPtrTy();
+      return 0;
     }
     
     ThisDecl = ThisRes.get();
@@ -580,7 +605,7 @@ Parser::DeclPtrTy Parser::ParseDeclarationAfterDeclarator(Declarator &D,
         return ThisDecl;
       }
       
-      OwningExprResult Init(ParseInitializer());
+      ExprResult Init(ParseInitializer());
 
       if (getLang().CPlusPlus && D.getCXXScopeSpec().isSet()) {
         Actions.ActOnCXXExitDeclInitializer(getCurScope(), ThisDecl);
@@ -591,7 +616,7 @@ Parser::DeclPtrTy Parser::ParseDeclarationAfterDeclarator(Declarator &D,
         SkipUntil(tok::comma, true, true);
         Actions.ActOnInitializerError(ThisDecl);
       } else
-        Actions.AddInitializerToDecl(ThisDecl, move(Init));
+        Actions.AddInitializerToDecl(ThisDecl, Init.take());
     }
   } else if (Tok.is(tok::l_paren)) {
     // Parse C++ direct initializer: '(' expression-list ')'
@@ -771,7 +796,7 @@ bool Parser::ParseImplicitInt(DeclSpec &DS, CXXScopeSpec *SS,
 
   // This is almost certainly an invalid type name. Let the action emit a 
   // diagnostic and attempt to recover.
-  Action::TypeTy *T = 0;
+  ParsedType T;
   if (Actions.DiagnoseUnknownTypeName(*Tok.getIdentifierInfo(), Loc,
                                       getCurScope(), SS, T)) {
     // The action emitted a diagnostic, so we don't have to.
@@ -781,8 +806,7 @@ bool Parser::ParseImplicitInt(DeclSpec &DS, CXXScopeSpec *SS,
       // name token, and we're done.
       const char *PrevSpec;
       unsigned DiagID;
-      DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID, T, 
-                         false);
+      DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID, T);
       DS.SetRangeEnd(Tok.getLocation());
       ConsumeToken();
       
@@ -851,21 +875,7 @@ Parser::getDeclSpecContextFromDeclaratorContext(unsigned Context) {
 void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
                                         const ParsedTemplateInfo &TemplateInfo,
                                         AccessSpecifier AS,
-                                        DeclSpecContext DSContext) {
-  if (Tok.is(tok::code_completion)) {
-    Action::CodeCompletionContext CCC = Action::CCC_Namespace;
-    if (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate)
-      CCC = DSContext == DSC_class? Action::CCC_MemberTemplate 
-                                  : Action::CCC_Template;
-    else if (DSContext == DSC_class)
-      CCC = Action::CCC_Class;
-    else if (ObjCImpDecl)
-      CCC = Action::CCC_ObjCImplementation;
-    
-    Actions.CodeCompleteOrdinaryName(getCurScope(), CCC);
-    ConsumeCodeCompletionToken();
-  }
-  
+                                        DeclSpecContext DSContext) {  
   DS.SetRangeStart(Tok.getLocation());
   while (1) {
     bool isInvalid = false;
@@ -882,6 +892,38 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       DS.Finish(Diags, PP);
       return;
 
+    case tok::code_completion: {
+      Sema::ParserCompletionContext CCC = Sema::PCC_Namespace;
+      if (DS.hasTypeSpecifier()) {
+        bool AllowNonIdentifiers
+          = (getCurScope()->getFlags() & (Scope::ControlScope |
+                                          Scope::BlockScope |
+                                          Scope::TemplateParamScope |
+                                          Scope::FunctionPrototypeScope |
+                                          Scope::AtCatchScope)) == 0;
+        bool AllowNestedNameSpecifiers
+          = DSContext == DSC_top_level || 
+            (DSContext == DSC_class && DS.isFriendSpecified());
+
+        Actions.CodeCompleteDeclarator(getCurScope(), AllowNonIdentifiers, 
+                                       AllowNestedNameSpecifiers);
+        ConsumeCodeCompletionToken();
+        return;
+      } 
+      
+      if (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate)
+        CCC = DSContext == DSC_class? Sema::PCC_MemberTemplate 
+                                    : Sema::PCC_Template;
+      else if (DSContext == DSC_class)
+        CCC = Sema::PCC_Class;
+      else if (ObjCImpDecl)
+        CCC = Sema::PCC_ObjCImplementation;
+      
+      Actions.CodeCompleteOrdinaryName(getCurScope(), CCC);
+      ConsumeCodeCompletionToken();
+      return;
+    }
+
     case tok::coloncolon: // ::foo::bar
       // C++ scope specifier.  Annotate and loop, or bail out on error.
       if (TryAnnotateCXXScopeToken(true)) {
@@ -898,7 +940,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         goto DoneWithDeclSpec;
 
       CXXScopeSpec SS;
-      SS.setScopeRep(Tok.getAnnotationValue());
+      SS.setScopeRep((NestedNameSpecifier*) Tok.getAnnotationValue());
       SS.setRange(Tok.getAnnotationRange());
 
       // We are looking for a qualified typename.
@@ -961,10 +1003,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       if (Next.is(tok::annot_typename)) {
         DS.getTypeSpecScope() = SS;
         ConsumeToken(); // The C++ scope.
-        if (Tok.getAnnotationValue())
+        if (Tok.getAnnotationValue()) {
+          ParsedType T = getTypeAnnotation(Tok);
           isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, 
-                                         PrevSpec, DiagID, 
-                                         Tok.getAnnotationValue());
+                                         PrevSpec, DiagID, T);
+        }
         else
           DS.SetTypeSpecError();
         DS.SetRangeEnd(Tok.getAnnotationEndLoc());
@@ -993,8 +1036,9 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
           << Next.getIdentifierInfo();
       }
 
-      TypeTy *TypeRep = Actions.getTypeName(*Next.getIdentifierInfo(),
-                                            Next.getLocation(), getCurScope(), &SS);
+      ParsedType TypeRep = Actions.getTypeName(*Next.getIdentifierInfo(),
+                                               Next.getLocation(),
+                                               getCurScope(), &SS);
 
       // If the referenced identifier is not a type, then this declspec is
       // erroneous: We already checked about that it has no type specifier, and
@@ -1021,10 +1065,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     }
 
     case tok::annot_typename: {
-      if (Tok.getAnnotationValue())
+      if (Tok.getAnnotationValue()) {
+        ParsedType T = getTypeAnnotation(Tok);
         isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec,
-                                       DiagID, Tok.getAnnotationValue());
-      else
+                                       DiagID, T);
+      } else
         DS.SetTypeSpecError();
       
       if (isInvalid)
@@ -1041,7 +1086,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         continue;
 
       SourceLocation LAngleLoc, EndProtoLoc;
-      llvm::SmallVector<DeclPtrTy, 8> ProtocolDecl;
+      llvm::SmallVector<Decl *, 8> ProtocolDecl;
       llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
       ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
                                   LAngleLoc, EndProtoLoc);
@@ -1077,12 +1122,13 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         break;
 
       // It has to be available as a typedef too!
-      TypeTy *TypeRep = Actions.getTypeName(*Tok.getIdentifierInfo(),
-                                            Tok.getLocation(), getCurScope());
+      ParsedType TypeRep =
+        Actions.getTypeName(*Tok.getIdentifierInfo(),
+                            Tok.getLocation(), getCurScope());
 
       // If this is not a typedef name, don't parse it as part of the declspec,
       // it must be an implicit int or an error.
-      if (TypeRep == 0) {
+      if (!TypeRep) {
         if (ParseImplicitInt(DS, 0, TemplateInfo, AS)) continue;
         goto DoneWithDeclSpec;
       }
@@ -1110,7 +1156,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
         continue;
 
       SourceLocation LAngleLoc, EndProtoLoc;
-      llvm::SmallVector<DeclPtrTy, 8> ProtocolDecl;
+      llvm::SmallVector<Decl *, 8> ProtocolDecl;
       llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
       ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
                                   LAngleLoc, EndProtoLoc);
@@ -1172,6 +1218,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
       DS.AddAttributes(ParseMicrosoftTypeAttributes());
       continue;
 
+    // Borland single token adornments.
+    case tok::kw___pascal:
+      DS.AddAttributes(ParseBorlandTypeAttributes());
+      continue;
+
     // storage-class-specifier
     case tok::kw_typedef:
       isInvalid = DS.SetStorageClassSpec(DeclSpec::SCS_typedef, Loc, PrevSpec,
@@ -1383,7 +1434,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
 
       {
         SourceLocation LAngleLoc, EndProtoLoc;
-        llvm::SmallVector<DeclPtrTy, 8> ProtocolDecl;
+        llvm::SmallVector<Decl *, 8> ProtocolDecl;
         llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
         ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
                                     LAngleLoc, EndProtoLoc);
@@ -1403,7 +1454,12 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     if (isInvalid) {
       assert(PrevSpec && "Method did not return previous specifier!");
       assert(DiagID);
-      Diag(Tok, DiagID) << PrevSpec;
+      
+      if (DiagID == diag::ext_duplicate_declspec)
+        Diag(Tok, DiagID)
+          << PrevSpec << FixItHint::CreateRemoval(Tok.getLocation());
+      else
+        Diag(Tok, DiagID) << PrevSpec;
     }
     DS.SetRangeEnd(Tok.getLocation());
     ConsumeToken();
@@ -1495,10 +1551,10 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
 
   // simple-type-specifier:
   case tok::annot_typename: {
-    if (Tok.getAnnotationValue())
+    if (ParsedType T = getTypeAnnotation(Tok)) {
       isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec,
-                                     DiagID, Tok.getAnnotationValue());
-    else
+                                     DiagID, T);
+    } else
       DS.SetTypeSpecError();
     DS.SetRangeEnd(Tok.getAnnotationEndLoc());
     ConsumeToken(); // The typename
@@ -1511,7 +1567,7 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
       return true;
 
     SourceLocation LAngleLoc, EndProtoLoc;
-    llvm::SmallVector<DeclPtrTy, 8> ProtocolDecl;
+    llvm::SmallVector<Decl *, 8> ProtocolDecl;
     llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
     ParseObjCProtocolReferences(ProtocolDecl, ProtocolLocs, false,
                                 LAngleLoc, EndProtoLoc);
@@ -1643,6 +1699,7 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
 
     isInvalid = DS.SetTypeSpecType(DeclSpec::TST_auto, Loc, PrevSpec, DiagID);
     break;
+
   case tok::kw___ptr64:
   case tok::kw___w64:
   case tok::kw___cdecl:
@@ -1652,6 +1709,10 @@ bool Parser::ParseOptionalTypeSpecifier(DeclSpec &DS, bool& isInvalid,
     DS.AddAttributes(ParseMicrosoftTypeAttributes());
     return true;
 
+  case tok::kw___pascal:
+    DS.AddAttributes(ParseBorlandTypeAttributes());
+    return true;
+
   default:
     // Not a type-specifier; do nothing.
     return false;
@@ -1728,7 +1789,7 @@ ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
 
     if (Tok.is(tok::colon)) {
       ConsumeToken();
-      OwningExprResult Res(ParseConstantExpression());
+      ExprResult Res(ParseConstantExpression());
       if (Res.isInvalid())
         SkipUntil(tok::semi, true, true);
       else
@@ -1743,7 +1804,7 @@ ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
     }
 
     // We're done with this declarator;  invoke the callback.
-    DeclPtrTy D = Fields.invoke(DeclaratorInfo);
+    Decl *D = Fields.invoke(DeclaratorInfo);
     PD.complete(D);
 
     // If we don't have a comma, it is either the end of the list (a ';')
@@ -1769,10 +1830,9 @@ ParseStructDeclaration(DeclSpec &DS, FieldCallback &Fields) {
 /// [OBC]   '@' 'defs' '(' class-name ')'
 ///
 void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
-                                  unsigned TagType, DeclPtrTy TagDecl) {
-  PrettyStackTraceActionsDecl CrashInfo(TagDecl, RecordLoc, Actions,
-                                        PP.getSourceManager(),
-                                        "parsing struct/union body");
+                                  unsigned TagType, Decl *TagDecl) {
+  PrettyDeclStackTraceEntry CrashInfo(Actions, TagDecl, RecordLoc,
+                                      "parsing struct/union body");
 
   SourceLocation LBraceLoc = ConsumeBrace();
 
@@ -1782,10 +1842,10 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
   // Empty structs are an extension in C (C99 6.7.2.1p7), but are allowed in
   // C++.
   if (Tok.is(tok::r_brace) && !getLang().CPlusPlus)
-    Diag(Tok, diag::ext_empty_struct_union_enum)
-      << DeclSpec::getSpecifierName((DeclSpec::TST)TagType);
+    Diag(Tok, diag::ext_empty_struct_union)
+      << (TagType == TST_union);
 
-  llvm::SmallVector<DeclPtrTy, 32> FieldDecls;
+  llvm::SmallVector<Decl *, 32> FieldDecls;
 
   // While we still have something to read, read the declarations in the struct.
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
@@ -1806,16 +1866,16 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
     if (!Tok.is(tok::at)) {
       struct CFieldCallback : FieldCallback {
         Parser &P;
-        DeclPtrTy TagDecl;
-        llvm::SmallVectorImpl<DeclPtrTy> &FieldDecls;
+        Decl *TagDecl;
+        llvm::SmallVectorImpl<Decl *> &FieldDecls;
 
-        CFieldCallback(Parser &P, DeclPtrTy TagDecl,
-                       llvm::SmallVectorImpl<DeclPtrTy> &FieldDecls) :
+        CFieldCallback(Parser &P, Decl *TagDecl,
+                       llvm::SmallVectorImpl<Decl *> &FieldDecls) :
           P(P), TagDecl(TagDecl), FieldDecls(FieldDecls) {}
 
-        virtual DeclPtrTy invoke(FieldDeclarator &FD) {
+        virtual Decl *invoke(FieldDeclarator &FD) {
           // Install the declarator into the current TagDecl.
-          DeclPtrTy Field = P.Actions.ActOnField(P.getCurScope(), TagDecl,
+          Decl *Field = P.Actions.ActOnField(P.getCurScope(), TagDecl,
                               FD.D.getDeclSpec().getSourceRange().getBegin(),
                                                  FD.D, FD.BitfieldSize);
           FieldDecls.push_back(Field);
@@ -1838,7 +1898,7 @@ void Parser::ParseStructUnionBody(SourceLocation RecordLoc,
         SkipUntil(tok::semi, true);
         continue;
       }
-      llvm::SmallVector<DeclPtrTy, 16> Fields;
+      llvm::SmallVector<Decl *, 16> Fields;
       Actions.ActOnDefs(getCurScope(), TagDecl, Tok.getLocation(),
                         Tok.getIdentifierInfo(), Fields);
       FieldDecls.insert(FieldDecls.end(), Fields.begin(), Fields.end());
@@ -1905,7 +1965,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
 
   CXXScopeSpec &SS = DS.getTypeSpecScope();
   if (getLang().CPlusPlus) {
-    if (ParseOptionalCXXScopeSpecifier(SS, 0, false))
+    if (ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false))
       return;
 
     if (SS.isSet() && Tok.isNot(tok::identifier)) {
@@ -1944,18 +2004,18 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   // enum foo {..};  void bar() { enum foo; }    <- new foo in bar.
   // enum foo {..};  void bar() { enum foo x; }  <- use of old foo.
   //
-  Action::TagUseKind TUK;
+  Sema::TagUseKind TUK;
   if (Tok.is(tok::l_brace))
-    TUK = Action::TUK_Definition;
+    TUK = Sema::TUK_Definition;
   else if (Tok.is(tok::semi))
-    TUK = Action::TUK_Declaration;
+    TUK = Sema::TUK_Declaration;
   else
-    TUK = Action::TUK_Reference;
+    TUK = Sema::TUK_Reference;
   
   // enums cannot be templates, although they can be referenced from a 
   // template.
   if (TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate &&
-      TUK != Action::TUK_Reference) {
+      TUK != Sema::TUK_Reference) {
     Diag(Tok, diag::err_enum_template);
     
     // Skip the rest of this declarator, up until the comma or semicolon.
@@ -1968,11 +2028,11 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   SourceLocation TSTLoc = NameLoc.isValid()? NameLoc : StartLoc;
   const char *PrevSpec = 0;
   unsigned DiagID;
-  DeclPtrTy TagDecl = Actions.ActOnTag(getCurScope(), DeclSpec::TST_enum, TUK,
-                                       StartLoc, SS, Name, NameLoc, Attr.get(),
-                                       AS,
-                                       Action::MultiTemplateParamsArg(Actions),
-                                       Owned, IsDependent);
+  Decl *TagDecl = Actions.ActOnTag(getCurScope(), DeclSpec::TST_enum, TUK,
+                                   StartLoc, SS, Name, NameLoc, Attr.get(),
+                                   AS,
+                                   MultiTemplateParamsArg(Actions),
+                                   Owned, IsDependent);
   if (IsDependent) {
     // This enum has a dependent nested-name-specifier. Handle it as a 
     // dependent tag.
@@ -1991,13 +2051,13 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
     }
     
     if (DS.SetTypeSpecType(DeclSpec::TST_typename, TSTLoc, PrevSpec, DiagID,
-                           Type.get(), false))
+                           Type.get()))
       Diag(StartLoc, DiagID) << PrevSpec;
     
     return;
   }
 
-  if (!TagDecl.get()) {
+  if (!TagDecl) {
     // The action failed to produce an enumeration tag. If this is a 
     // definition, consume the entire definition.
     if (Tok.is(tok::l_brace)) {
@@ -2012,10 +2072,10 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
   if (Tok.is(tok::l_brace))
     ParseEnumBody(StartLoc, TagDecl);
 
-  // FIXME: The DeclSpec should keep the locations of both the keyword and the
-  // name (if there is one).
+  // FIXME: The DeclSpec should keep the locations of both the keyword
+  // and the name (if there is one).
   if (DS.SetTypeSpecType(DeclSpec::TST_enum, TSTLoc, PrevSpec, DiagID,
-                         TagDecl.getAs<void>(), Owned))
+                         TagDecl, Owned))
     Diag(StartLoc, DiagID) << PrevSpec;
 }
 
@@ -2029,7 +2089,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
 ///       enumeration-constant:
 ///         identifier
 ///
-void Parser::ParseEnumBody(SourceLocation StartLoc, DeclPtrTy EnumDecl) {
+void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
   // Enter the scope of the enum body and start the definition.
   ParseScope EnumScope(this, Scope::DeclScope);
   Actions.ActOnTagStartDefinition(getCurScope(), EnumDecl);
@@ -2040,9 +2100,9 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, DeclPtrTy EnumDecl) {
   if (Tok.is(tok::r_brace) && !getLang().CPlusPlus)
     Diag(Tok, diag::error_empty_enum);
 
-  llvm::SmallVector<DeclPtrTy, 32> EnumConstantDecls;
+  llvm::SmallVector<Decl *, 32> EnumConstantDecls;
 
-  DeclPtrTy LastEnumConstDecl;
+  Decl *LastEnumConstDecl = 0;
 
   // Parse the enumerator-list.
   while (Tok.is(tok::identifier)) {
@@ -2050,7 +2110,7 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, DeclPtrTy EnumDecl) {
     SourceLocation IdentLoc = ConsumeToken();
 
     SourceLocation EqualLoc;
-    OwningExprResult AssignedVal(Actions);
+    ExprResult AssignedVal;
     if (Tok.is(tok::equal)) {
       EqualLoc = ConsumeToken();
       AssignedVal = ParseConstantExpression();
@@ -2059,11 +2119,11 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, DeclPtrTy EnumDecl) {
     }
 
     // Install the enumerator constant into EnumDecl.
-    DeclPtrTy EnumConstDecl = Actions.ActOnEnumConstant(getCurScope(), EnumDecl,
-                                                        LastEnumConstDecl,
-                                                        IdentLoc, Ident,
-                                                        EqualLoc,
-                                                        AssignedVal.release());
+    Decl *EnumConstDecl = Actions.ActOnEnumConstant(getCurScope(), EnumDecl,
+                                                    LastEnumConstDecl,
+                                                    IdentLoc, Ident,
+                                                    EqualLoc,
+                                                    AssignedVal.release());
     EnumConstantDecls.push_back(EnumConstDecl);
     LastEnumConstDecl = EnumConstDecl;
 
@@ -2229,6 +2289,7 @@ bool Parser::isTypeSpecifierQualifier() {
   case tok::kw___thiscall:
   case tok::kw___w64:
   case tok::kw___ptr64:
+  case tok::kw___pascal:
     return true;
   }
 }
@@ -2337,6 +2398,7 @@ bool Parser::isDeclarationSpecifier() {
   case tok::kw___w64:
   case tok::kw___ptr64:
   case tok::kw___forceinline:
+  case tok::kw___pascal:
     return true;
   }
 }
@@ -2346,7 +2408,7 @@ bool Parser::isConstructorDeclarator() {
 
   // Parse the C++ scope specifier.
   CXXScopeSpec SS;
-  if (ParseOptionalCXXScopeSpecifier(SS, 0, true)) {
+  if (ParseOptionalCXXScopeSpecifier(SS, ParsedType(), true)) {
     TPA.Revert();
     return false;
   }
@@ -2388,15 +2450,19 @@ bool Parser::isConstructorDeclarator() {
 }
 
 /// ParseTypeQualifierListOpt
-///       type-qualifier-list: [C99 6.7.5]
-///         type-qualifier
-/// [GNU]   attributes                        [ only if AttributesAllowed=true ]
-///         type-qualifier-list type-qualifier
-/// [GNU]   type-qualifier-list attributes    [ only if AttributesAllowed=true ]
-/// [C++0x] attribute-specifier[opt] is allowed before cv-qualifier-seq
-///           if CXX0XAttributesAllowed = true
+///          type-qualifier-list: [C99 6.7.5]
+///            type-qualifier
+/// [vendor]   attributes                        
+///              [ only if VendorAttributesAllowed=true ]
+///            type-qualifier-list type-qualifier
+/// [vendor]   type-qualifier-list attributes    
+///              [ only if VendorAttributesAllowed=true ]
+/// [C++0x]    attribute-specifier[opt] is allowed before cv-qualifier-seq
+///              [ only if CXX0XAttributesAllowed=true ]
+/// Note: vendor can be GNU, MS, etc.
 ///
-void Parser::ParseTypeQualifierListOpt(DeclSpec &DS, bool GNUAttributesAllowed,
+void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
+                                       bool VendorAttributesAllowed,
                                        bool CXX0XAttributesAllowed) {
   if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier()) {
     SourceLocation Loc = Tok.getLocation();
@@ -2414,6 +2480,11 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS, bool GNUAttributesAllowed,
     SourceLocation Loc = Tok.getLocation();
 
     switch (Tok.getKind()) {
+    case tok::code_completion:
+      Actions.CodeCompleteTypeQualifiers(DS);
+      ConsumeCodeCompletionToken();
+      break;
+        
     case tok::kw_const:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_const   , Loc, PrevSpec, DiagID,
                                  getLang());
@@ -2432,13 +2503,19 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS, bool GNUAttributesAllowed,
     case tok::kw___stdcall:
     case tok::kw___fastcall:
     case tok::kw___thiscall:
-      if (GNUAttributesAllowed) {
+      if (VendorAttributesAllowed) {
         DS.AddAttributes(ParseMicrosoftTypeAttributes());
         continue;
       }
       goto DoneWithTypeQuals;
+    case tok::kw___pascal:
+      if (VendorAttributesAllowed) {
+        DS.AddAttributes(ParseBorlandTypeAttributes());
+        continue;
+      }
+      goto DoneWithTypeQuals;
     case tok::kw___attribute:
-      if (GNUAttributesAllowed) {
+      if (VendorAttributesAllowed) {
         DS.AddAttributes(ParseGNUAttributes());
         continue; // do *not* consume the next token!
       }
@@ -2494,6 +2571,7 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
                                      DirectDeclParseFunction DirectDeclParser) {
   if (Diags.hasAllExtensionsSilenced())
     D.setExtension();
+  
   // C++ member pointers start with a '::' or a nested-name.
   // Member pointers get special handling, since there's no place for the
   // scope spec in the generic path below.
@@ -2501,7 +2579,7 @@ void Parser::ParseDeclaratorInternal(Declarator &D,
       (Tok.is(tok::coloncolon) || Tok.is(tok::identifier) ||
        Tok.is(tok::annot_cxxscope))) {
     CXXScopeSpec SS;
-    ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, true); // ignore fail
+    ParseOptionalCXXScopeSpecifier(SS, ParsedType(), true); // ignore fail
 
     if (SS.isNotEmpty()) {
       if (Tok.isNot(tok::star)) {
@@ -2660,8 +2738,7 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
   if (getLang().CPlusPlus && D.mayHaveIdentifier()) {
     // ParseDeclaratorInternal might already have parsed the scope.
     if (D.getCXXScopeSpec().isEmpty()) {
-      ParseOptionalCXXScopeSpecifier(D.getCXXScopeSpec(), /*ObjectType=*/0,
-                                     true);
+      ParseOptionalCXXScopeSpecifier(D.getCXXScopeSpec(), ParsedType(), true);
     }
 
     if (D.getCXXScopeSpec().isValid()) {
@@ -2690,7 +2767,7 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
                              /*EnteringContext=*/true, 
                              /*AllowDestructorName=*/true, 
                              AllowConstructorName,
-                             /*ObjectType=*/0,
+                             ParsedType(),
                              D.getName()) ||
           // Once we're past the identifier, if the scope was bad, mark the
           // whole declarator bad.
@@ -2724,7 +2801,10 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
     // scope when parsing the parenthesized declarator, then exited
     // the scope already. Re-enter the scope, if we need to.
     if (D.getCXXScopeSpec().isSet()) {
-      if (Actions.ShouldEnterDeclaratorScope(getCurScope(), D.getCXXScopeSpec()))
+      // If there was an error parsing parenthesized declarator, declarator
+      // scope may have been enterred before. Don't do it again.
+      if (!D.isInvalidType() &&
+          Actions.ShouldEnterDeclaratorScope(getCurScope(), D.getCXXScopeSpec()))
         // Change the declaration context for name lookup, until this function
         // is exited (and the declarator has been parsed).
         DeclScopeObj.EnterDeclaratorScope();
@@ -2820,6 +2900,10 @@ void Parser::ParseParenDeclarator(Declarator &D) {
        Tok.is(tok::kw___w64) || Tok.is(tok::kw___ptr64)) {
     AttrList.reset(ParseMicrosoftTypeAttributes(AttrList.take()));
   }
+  // Eat any Borland extensions.
+  if  (Tok.is(tok::kw___pascal)) {
+    AttrList.reset(ParseBorlandTypeAttributes(AttrList.take()));
+  }
 
   // If we haven't past the identifier yet (or where the identifier would be
   // stored, if this is an abstract declarator), then this is probably just
@@ -2922,7 +3006,7 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
     bool hasExceptionSpec = false;
     SourceLocation ThrowLoc;
     bool hasAnyExceptionSpec = false;
-    llvm::SmallVector<TypeTy*, 2> Exceptions;
+    llvm::SmallVector<ParsedType, 2> Exceptions;
     llvm::SmallVector<SourceRange, 2> ExceptionRanges;
     if (getLang().CPlusPlus) {
       ParseTypeQualifierListOpt(DS, false /*no attributes*/);
@@ -3061,7 +3145,7 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
 
       // Inform the actions module about the parameter declarator, so it gets
       // added to the current scope.
-      DeclPtrTy Param = Actions.ActOnParamDeclarator(getCurScope(), ParmDecl);
+      Decl *Param = Actions.ActOnParamDeclarator(getCurScope(), ParmDecl);
 
       // Parse the default argument, if any. We parse the default
       // arguments in all dialects; the semantic analysis in
@@ -3085,21 +3169,29 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
             delete DefArgToks;
             DefArgToks = 0;
             Actions.ActOnParamDefaultArgumentError(Param);
-          } else
+          } else {
+            // Mark the end of the default argument so that we know when to
+            // stop when we parse it later on.
+            Token DefArgEnd;
+            DefArgEnd.startToken();
+            DefArgEnd.setKind(tok::cxx_defaultarg_end);
+            DefArgEnd.setLocation(Tok.getLocation());
+            DefArgToks->push_back(DefArgEnd);
             Actions.ActOnParamUnparsedDefaultArgument(Param, EqualLoc,
                                                 (*DefArgToks)[1].getLocation());
+          }
         } else {
           // Consume the '='.
           ConsumeToken();
 
-          OwningExprResult DefArgResult(ParseAssignmentExpression());
+          ExprResult DefArgResult(ParseAssignmentExpression());
           if (DefArgResult.isInvalid()) {
             Actions.ActOnParamDefaultArgumentError(Param);
             SkipUntil(tok::comma, tok::r_paren, true, true);
           } else {
             // Inform the actions module about the default argument
             Actions.ActOnParamDefaultArgument(Param, EqualLoc,
-                                              move(DefArgResult));
+                                              DefArgResult.take());
           }
         }
       }
@@ -3141,7 +3233,7 @@ void Parser::ParseFunctionDeclarator(SourceLocation LParenLoc, Declarator &D,
   bool hasExceptionSpec = false;
   SourceLocation ThrowLoc;
   bool hasAnyExceptionSpec = false;
-  llvm::SmallVector<TypeTy*, 2> Exceptions;
+  llvm::SmallVector<ParsedType, 2> Exceptions;
   llvm::SmallVector<SourceRange, 2> ExceptionRanges;
   
   if (getLang().CPlusPlus) {
@@ -3202,8 +3294,7 @@ void Parser::ParseFunctionDeclaratorIdentifierList(SourceLocation LParenLoc,
   // The first identifier was already read, and is known to be the first
   // identifier in the list.  Remember this identifier in ParamInfo.
   ParamsSoFar.insert(FirstIdent);
-  ParamInfo.push_back(DeclaratorChunk::ParamInfo(FirstIdent, FirstIdentLoc,
-                                                 DeclPtrTy()));
+  ParamInfo.push_back(DeclaratorChunk::ParamInfo(FirstIdent, FirstIdentLoc, 0));
 
   while (Tok.is(tok::comma)) {
     // Eat the comma.
@@ -3229,7 +3320,7 @@ void Parser::ParseFunctionDeclaratorIdentifierList(SourceLocation LParenLoc,
       // Remember this identifier in ParamInfo.
       ParamInfo.push_back(DeclaratorChunk::ParamInfo(ParmII,
                                                      Tok.getLocation(),
-                                                     DeclPtrTy()));
+                                                     0));
     }
 
     // Eat the identifier.
@@ -3271,7 +3362,7 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
     }
     
     // Remember that we parsed the empty array type.
-    OwningExprResult NumElements(Actions);
+    ExprResult NumElements;
     D.AddTypeInfo(DeclaratorChunk::getArray(0, false, false, 0,
                                             StartLoc, EndLoc),
                   EndLoc);
@@ -3279,7 +3370,7 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
   } else if (Tok.getKind() == tok::numeric_constant &&
              GetLookAheadToken(1).is(tok::r_square)) {
     // [4] is very common.  Parse the numeric constant expression.
-    OwningExprResult ExprRes(Actions.ActOnNumericConstant(Tok));
+    ExprResult ExprRes(Actions.ActOnNumericConstant(Tok));
     ConsumeToken();
 
     SourceLocation EndLoc = MatchRHSPunctuation(tok::r_square, StartLoc);
@@ -3317,7 +3408,7 @@ void Parser::ParseBracketDeclarator(Declarator &D) {
 
   // Handle "direct-declarator [ type-qual-list[opt] * ]".
   bool isStar = false;
-  OwningExprResult NumElements(Actions);
+  ExprResult NumElements;
 
   // Handle the case where we have '[*]' as the array size.  However, a leading
   // star could be the start of an expression, for example 'X[*p + 4]'.  Verify
@@ -3382,12 +3473,12 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) {
   const bool hasParens = Tok.is(tok::l_paren);
 
   bool isCastExpr;
-  TypeTy *CastTy;
+  ParsedType CastTy;
   SourceRange CastRange;
-  OwningExprResult Operand = ParseExprAfterTypeofSizeofAlignof(OpTok,
-                                                               isCastExpr,
-                                                               CastTy,
-                                                               CastRange);
+  ExprResult Operand = ParseExprAfterTypeofSizeofAlignof(OpTok,
+                                                         isCastExpr,
+                                                         CastTy,
+                                                         CastRange);
   if (hasParens)
     DS.setTypeofParensRange(CastRange);
 
@@ -3422,7 +3513,7 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) {
   unsigned DiagID;
   // Check for duplicate type specifiers (e.g. "int typeof(int)").
   if (DS.SetTypeSpecType(DeclSpec::TST_typeofExpr, StartLoc, PrevSpec,
-                         DiagID, Operand.release()))
+                         DiagID, Operand.get()))
     Diag(StartLoc, DiagID) << PrevSpec;
 }
 
diff --git a/lib/Parse/ParseDeclCXX.cpp b/lib/Parse/ParseDeclCXX.cpp
index 590ba6c6f8bc..b277156a0d06 100644
--- a/lib/Parse/ParseDeclCXX.cpp
+++ b/lib/Parse/ParseDeclCXX.cpp
@@ -14,37 +14,42 @@
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/PrettyDeclStackTrace.h"
 #include "RAIIObjectsForParser.h"
 using namespace clang;
 
 /// ParseNamespace - We know that the current token is a namespace keyword. This
-/// may either be a top level namespace or a block-level namespace alias.
+/// may either be a top level namespace or a block-level namespace alias. If
+/// there was an inline keyword, it has already been parsed.
 ///
 ///       namespace-definition: [C++ 7.3: basic.namespace]
 ///         named-namespace-definition
 ///         unnamed-namespace-definition
 ///
 ///       unnamed-namespace-definition:
-///         'namespace' attributes[opt] '{' namespace-body '}'
+///         'inline'[opt] 'namespace' attributes[opt] '{' namespace-body '}'
 ///
 ///       named-namespace-definition:
 ///         original-namespace-definition
 ///         extension-namespace-definition
 ///
 ///       original-namespace-definition:
-///         'namespace' identifier attributes[opt] '{' namespace-body '}'
+///         'inline'[opt] 'namespace' identifier attributes[opt]
+///             '{' namespace-body '}'
 ///
 ///       extension-namespace-definition:
-///         'namespace' original-namespace-name '{' namespace-body '}'
+///         'inline'[opt] 'namespace' original-namespace-name
+///             '{' namespace-body '}'
 ///
 ///       namespace-alias-definition:  [C++ 7.3.2: namespace.alias]
 ///         'namespace' identifier '=' qualified-namespace-specifier ';'
 ///
-Parser::DeclPtrTy Parser::ParseNamespace(unsigned Context,
-                                         SourceLocation &DeclEnd) {
+Decl *Parser::ParseNamespace(unsigned Context,
+                             SourceLocation &DeclEnd,
+                             SourceLocation InlineLoc) {
   assert(Tok.is(tok::kw_namespace) && "Not a namespace!");
   SourceLocation NamespaceLoc = ConsumeToken();  // eat the 'namespace'.
 
@@ -75,6 +80,9 @@ Parser::DeclPtrTy Parser::ParseNamespace(unsigned Context,
   if (Tok.is(tok::equal)) {
     if (AttrList)
       Diag(attrTok, diag::err_unexpected_namespace_attributes_alias);
+    if (InlineLoc.isValid())
+      Diag(InlineLoc, diag::err_inline_namespace_alias)
+          << FixItHint::CreateRemoval(InlineLoc);
 
     return ParseNamespaceAlias(NamespaceLoc, IdentLoc, Ident, DeclEnd);
   }
@@ -82,7 +90,7 @@ Parser::DeclPtrTy Parser::ParseNamespace(unsigned Context,
   if (Tok.isNot(tok::l_brace)) {
     Diag(Tok, Ident ? diag::err_expected_lbrace :
          diag::err_expected_ident_lbrace);
-    return DeclPtrTy();
+    return 0;
   }
 
   SourceLocation LBrace = ConsumeBrace();
@@ -92,19 +100,22 @@ Parser::DeclPtrTy Parser::ParseNamespace(unsigned Context,
       getCurScope()->getFnParent()) {
     Diag(LBrace, diag::err_namespace_nonnamespace_scope);
     SkipUntil(tok::r_brace, false);
-    return DeclPtrTy();
+    return 0;
   }
 
+  // If we're still good, complain about inline namespaces in non-C++0x now.
+  if (!getLang().CPlusPlus0x && InlineLoc.isValid())
+    Diag(InlineLoc, diag::ext_inline_namespace);
+
   // Enter a scope for the namespace.
   ParseScope NamespaceScope(this, Scope::DeclScope);
 
-  DeclPtrTy NamespcDecl =
-    Actions.ActOnStartNamespaceDef(getCurScope(), IdentLoc, Ident, LBrace,
-                                   AttrList.get());
+  Decl *NamespcDecl =
+    Actions.ActOnStartNamespaceDef(getCurScope(), InlineLoc, IdentLoc, Ident,
+                                   LBrace, AttrList.get());
 
-  PrettyStackTraceActionsDecl CrashInfo(NamespcDecl, NamespaceLoc, Actions,
-                                        PP.getSourceManager(),
-                                        "parsing namespace");
+  PrettyDeclStackTraceEntry CrashInfo(Actions, NamespcDecl, NamespaceLoc,
+                                      "parsing namespace");
 
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
     CXX0XAttributeList Attr;
@@ -126,7 +137,7 @@ Parser::DeclPtrTy Parser::ParseNamespace(unsigned Context,
 /// ParseNamespaceAlias - Parse the part after the '=' in a namespace
 /// alias definition.
 ///
-Parser::DeclPtrTy Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
+Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
                                               SourceLocation AliasLoc,
                                               IdentifierInfo *Alias,
                                               SourceLocation &DeclEnd) {
@@ -141,13 +152,13 @@ Parser::DeclPtrTy Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
 
   CXXScopeSpec SS;
   // Parse (optional) nested-name-specifier.
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false);
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
 
   if (SS.isInvalid() || Tok.isNot(tok::identifier)) {
     Diag(Tok, diag::err_expected_namespace_name);
     // Skip to end of the definition and eat the ';'.
     SkipUntil(tok::semi);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Parse identifier.
@@ -170,7 +181,7 @@ Parser::DeclPtrTy Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc,
 ///         'extern' string-literal '{' declaration-seq[opt] '}'
 ///         'extern' string-literal declaration
 ///
-Parser::DeclPtrTy Parser::ParseLinkage(ParsingDeclSpec &DS,
+Decl *Parser::ParseLinkage(ParsingDeclSpec &DS,
                                        unsigned Context) {
   assert(Tok.is(tok::string_literal) && "Not a string literal!");
   llvm::SmallString<8> LangBuffer;
@@ -178,12 +189,12 @@ Parser::DeclPtrTy Parser::ParseLinkage(ParsingDeclSpec &DS,
   bool Invalid = false;
   llvm::StringRef Lang = PP.getSpelling(Tok, LangBuffer, &Invalid);
   if (Invalid)
-    return DeclPtrTy();
+    return 0;
 
   SourceLocation Loc = ConsumeStringToken();
 
   ParseScope LinkageScope(this, Scope::DeclScope);
-  DeclPtrTy LinkageSpec
+  Decl *LinkageSpec
     = Actions.ActOnStartLinkageSpecification(getCurScope(),
                                              /*FIXME: */SourceLocation(),
                                              Loc, Lang,
@@ -196,7 +207,8 @@ Parser::DeclPtrTy Parser::ParseLinkage(ParsingDeclSpec &DS,
   }
 
   if (Tok.isNot(tok::l_brace)) {
-    ParseDeclarationOrFunctionDefinition(DS, Attr.AttrList);
+    DS.setExternInLinkageSpec(true);
+    ParseExternalDeclaration(Attr, &DS);
     return Actions.ActOnFinishLinkageSpecification(getCurScope(), LinkageSpec,
                                                    SourceLocation());
   }
@@ -221,7 +233,7 @@ Parser::DeclPtrTy Parser::ParseLinkage(ParsingDeclSpec &DS,
 
 /// ParseUsingDirectiveOrDeclaration - Parse C++ using using-declaration or
 /// using-directive. Assumes that current token is 'using'.
-Parser::DeclPtrTy Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
+Decl *Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
                                                      SourceLocation &DeclEnd,
                                                      CXX0XAttributeList Attr) {
   assert(Tok.is(tok::kw_using) && "Not using token");
@@ -257,7 +269,7 @@ Parser::DeclPtrTy Parser::ParseUsingDirectiveOrDeclaration(unsigned Context,
 ///        'using' 'namespace' ::[opt] nested-name-specifier[opt]
 ///                 namespace-name attributes[opt] ;
 ///
-Parser::DeclPtrTy Parser::ParseUsingDirective(unsigned Context,
+Decl *Parser::ParseUsingDirective(unsigned Context,
                                               SourceLocation UsingLoc,
                                               SourceLocation &DeclEnd,
                                               AttributeList *Attr) {
@@ -273,7 +285,7 @@ Parser::DeclPtrTy Parser::ParseUsingDirective(unsigned Context,
 
   CXXScopeSpec SS;
   // Parse (optional) nested-name-specifier.
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false);
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
 
   IdentifierInfo *NamespcName = 0;
   SourceLocation IdentLoc = SourceLocation();
@@ -284,7 +296,7 @@ Parser::DeclPtrTy Parser::ParseUsingDirective(unsigned Context,
     // If there was invalid namespace name, skip to end of decl, and eat ';'.
     SkipUntil(tok::semi);
     // FIXME: Are there cases, when we would like to call ActOnUsingDirective?
-    return DeclPtrTy();
+    return 0;
   }
 
   // Parse identifier.
@@ -316,7 +328,7 @@ Parser::DeclPtrTy Parser::ParseUsingDirective(unsigned Context,
 ///               unqualified-id
 ///       'using' :: unqualified-id
 ///
-Parser::DeclPtrTy Parser::ParseUsingDeclaration(unsigned Context,
+Decl *Parser::ParseUsingDeclaration(unsigned Context,
                                                 SourceLocation UsingLoc,
                                                 SourceLocation &DeclEnd,
                                                 AccessSpecifier AS) {
@@ -335,12 +347,12 @@ Parser::DeclPtrTy Parser::ParseUsingDeclaration(unsigned Context,
     IsTypeName = false;
 
   // Parse nested-name-specifier.
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false);
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
 
   // Check nested-name specifier.
   if (SS.isInvalid()) {
     SkipUntil(tok::semi);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Parse the unqualified-id. We allow parsing of both constructor and
@@ -351,10 +363,10 @@ Parser::DeclPtrTy Parser::ParseUsingDeclaration(unsigned Context,
                          /*EnteringContext=*/false,
                          /*AllowDestructorName=*/true,
                          /*AllowConstructorName=*/true,
-                         /*ObjectType=*/0,
+                         ParsedType(),
                          Name)) {
     SkipUntil(tok::semi);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Parse (optional) attributes (most likely GNU strong-using extension).
@@ -377,43 +389,44 @@ Parser::DeclPtrTy Parser::ParseUsingDeclaration(unsigned Context,
 ///      static_assert-declaration:
 ///        static_assert ( constant-expression  ,  string-literal  ) ;
 ///
-Parser::DeclPtrTy Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
+Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
   assert(Tok.is(tok::kw_static_assert) && "Not a static_assert declaration");
   SourceLocation StaticAssertLoc = ConsumeToken();
 
   if (Tok.isNot(tok::l_paren)) {
     Diag(Tok, diag::err_expected_lparen);
-    return DeclPtrTy();
+    return 0;
   }
 
   SourceLocation LParenLoc = ConsumeParen();
 
-  OwningExprResult AssertExpr(ParseConstantExpression());
+  ExprResult AssertExpr(ParseConstantExpression());
   if (AssertExpr.isInvalid()) {
     SkipUntil(tok::semi);
-    return DeclPtrTy();
+    return 0;
   }
 
   if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "", tok::semi))
-    return DeclPtrTy();
+    return 0;
 
   if (Tok.isNot(tok::string_literal)) {
     Diag(Tok, diag::err_expected_string_literal);
     SkipUntil(tok::semi);
-    return DeclPtrTy();
+    return 0;
   }
 
-  OwningExprResult AssertMessage(ParseStringLiteralExpression());
+  ExprResult AssertMessage(ParseStringLiteralExpression());
   if (AssertMessage.isInvalid())
-    return DeclPtrTy();
+    return 0;
 
   MatchRHSPunctuation(tok::r_paren, LParenLoc);
 
   DeclEnd = Tok.getLocation();
   ExpectAndConsume(tok::semi, diag::err_expected_semi_after_static_assert);
 
-  return Actions.ActOnStaticAssertDeclaration(StaticAssertLoc, move(AssertExpr),
-                                              move(AssertMessage));
+  return Actions.ActOnStaticAssertDeclaration(StaticAssertLoc,
+                                              AssertExpr.take(),
+                                              AssertMessage.take());
 }
 
 /// ParseDecltypeSpecifier - Parse a C++0x decltype specifier.
@@ -437,8 +450,8 @@ void Parser::ParseDecltypeSpecifier(DeclSpec &DS) {
   // C++0x [dcl.type.simple]p4:
   //   The operand of the decltype specifier is an unevaluated operand.
   EnterExpressionEvaluationContext Unevaluated(Actions,
-                                               Action::Unevaluated);
-  OwningExprResult Result = ParseExpression();
+                                               Sema::Unevaluated);
+  ExprResult Result = ParseExpression();
   if (Result.isInvalid()) {
     SkipUntil(tok::r_paren);
     return;
@@ -483,7 +496,7 @@ Parser::TypeResult Parser::ParseClassName(SourceLocation &EndLocation,
       AnnotateTemplateIdTokenAsType(SS);
 
       assert(Tok.is(tok::annot_typename) && "template-id -> type failed");
-      TypeTy *Type = Tok.getAnnotationValue();
+      ParsedType Type = getTypeAnnotation(Tok);
       EndLocation = Tok.getAnnotationEndLoc();
       ConsumeToken();
 
@@ -536,13 +549,13 @@ Parser::TypeResult Parser::ParseClassName(SourceLocation &EndLocation,
     // Retrieve the type from the annotation token, consume that token, and
     // return.
     EndLocation = Tok.getAnnotationEndLoc();
-    TypeTy *Type = Tok.getAnnotationValue();
+    ParsedType Type = getTypeAnnotation(Tok);
     ConsumeToken();
     return Type;
   }
 
   // We have an identifier; check whether it is actually a type.
-  TypeTy *Type = Actions.getTypeName(*Id, IdLoc, getCurScope(), SS, true);
+  ParsedType Type = Actions.getTypeName(*Id, IdLoc, getCurScope(), SS, true);
   if (!Type) {
     Diag(IdLoc, diag::err_expected_class_name);
     return true;
@@ -550,7 +563,19 @@ Parser::TypeResult Parser::ParseClassName(SourceLocation &EndLocation,
 
   // Consume the identifier.
   EndLocation = IdLoc;
-  return Type;
+
+  // Fake up a Declarator to use with ActOnTypeName.
+  DeclSpec DS;
+  DS.SetRangeStart(IdLoc);
+  DS.SetRangeEnd(EndLocation);
+  DS.getTypeSpecScope() = *SS;
+
+  const char *PrevSpec = 0;
+  unsigned DiagID;
+  DS.SetTypeSpecType(TST_typename, IdLoc, PrevSpec, DiagID, Type);
+
+  Declarator DeclaratorInfo(DS, Declarator::TypeNameContext);
+  return Actions.ActOnTypeName(getCurScope(), DeclaratorInfo);
 }
 
 /// ParseClassSpecifier - Parse a C++ class-specifier [C++ class] or
@@ -633,7 +658,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     AttrList = ParseGNUAttributes();
 
   // If declspecs exist after tag, parse them.
-  if (Tok.is(tok::kw___declspec))
+  while (Tok.is(tok::kw___declspec))
     AttrList = ParseMicrosoftDeclSpec(AttrList);
 
   // If C++0x attributes exist here, parse them.
@@ -648,7 +673,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // token sequence "struct __is_pod", make __is_pod into a normal
     // identifier rather than a keyword, to allow libstdc++ 4.2 to work
     // properly.
-    Tok.getIdentifierInfo()->setTokenID(tok::identifier);
+    Tok.getIdentifierInfo()->RevertTokenIDToIdentifier();
     Tok.setKind(tok::identifier);
   }
 
@@ -658,7 +683,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // token sequence "struct __is_empty", make __is_empty into a normal
     // identifier rather than a keyword, to allow libstdc++ 4.2 to work
     // properly.
-    Tok.getIdentifierInfo()->setTokenID(tok::identifier);
+    Tok.getIdentifierInfo()->RevertTokenIDToIdentifier();
     Tok.setKind(tok::identifier);
   }
 
@@ -668,7 +693,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // "FOO : BAR" is not a potential typo for "FOO::BAR".
     ColonProtectionRAIIObject X(*this);
 
-    ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, true);
+    if (ParseOptionalCXXScopeSpecifier(SS, ParsedType(), true))
+      DS.SetTypeSpecError();
     if (SS.isSet())
       if (Tok.isNot(tok::identifier) && Tok.isNot(tok::annot_template_id))
         Diag(Tok, diag::err_expected_ident);
@@ -769,9 +795,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   // or
   // &T::operator struct s;
   // For these, SuppressDeclarations is true.
-  Action::TagUseKind TUK;
+  Sema::TagUseKind TUK;
   if (SuppressDeclarations)
-    TUK = Action::TUK_Reference;
+    TUK = Sema::TUK_Reference;
   else if (Tok.is(tok::l_brace) || (getLang().CPlusPlus && Tok.is(tok::colon))){
     if (DS.isFriendSpecified()) {
       // C++ [class.friend]p2:
@@ -782,20 +808,23 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       // Skip everything up to the semicolon, so that this looks like a proper
       // friend class (or template thereof) declaration.
       SkipUntil(tok::semi, true, true);
-      TUK = Action::TUK_Friend;
+      TUK = Sema::TUK_Friend;
     } else {
       // Okay, this is a class definition.
-      TUK = Action::TUK_Definition;
+      TUK = Sema::TUK_Definition;
     }
   } else if (Tok.is(tok::semi))
-    TUK = DS.isFriendSpecified() ? Action::TUK_Friend : Action::TUK_Declaration;
+    TUK = DS.isFriendSpecified() ? Sema::TUK_Friend : Sema::TUK_Declaration;
   else
-    TUK = Action::TUK_Reference;
-
-  if (!Name && !TemplateId && TUK != Action::TUK_Definition) {
-    // We have a declaration or reference to an anonymous class.
-    Diag(StartLoc, diag::err_anon_type_definition)
-      << DeclSpec::getSpecifierName(TagType);
+    TUK = Sema::TUK_Reference;
+
+  if (!Name && !TemplateId && (DS.getTypeSpecType() == DeclSpec::TST_error ||
+                               TUK != Sema::TUK_Definition)) {
+    if (DS.getTypeSpecType() != DeclSpec::TST_error) {
+      // We have a declaration or reference to an anonymous class.
+      Diag(StartLoc, diag::err_anon_type_definition)
+        << DeclSpec::getSpecifierName(TagType);
+    }
 
     SkipUntil(tok::comma, true);
 
@@ -805,8 +834,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   }
 
   // Create the tag portion of the class or class template.
-  Action::DeclResult TagOrTempResult = true; // invalid
-  Action::TypeResult TypeResult = true; // invalid
+  DeclResult TagOrTempResult = true; // invalid
+  TypeResult TypeResult = true; // invalid
 
   bool Owned = false;
   if (TemplateId) {
@@ -816,7 +845,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                        TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
     if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
-        TUK == Action::TUK_Declaration) {
+        TUK == Sema::TUK_Declaration) {
       // This is an explicit instantiation of a class template.
       TagOrTempResult
         = Actions.ActOnExplicitInstantiation(getCurScope(),
@@ -825,7 +854,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                              TagType,
                                              StartLoc,
                                              SS,
-                                     TemplateTy::make(TemplateId->Template),
+                                             TemplateId->Template,
                                              TemplateId->TemplateNameLoc,
                                              TemplateId->LAngleLoc,
                                              TemplateArgsPtr,
@@ -836,11 +865,11 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // they have template headers, in which case they're ill-formed
     // (FIXME: "template <class T> friend class A<T>::B<int>;").
     // We diagnose this error in ActOnClassTemplateSpecialization.
-    } else if (TUK == Action::TUK_Reference ||
-               (TUK == Action::TUK_Friend &&
+    } else if (TUK == Sema::TUK_Reference ||
+               (TUK == Sema::TUK_Friend &&
                 TemplateInfo.Kind == ParsedTemplateInfo::NonTemplate)) {
       TypeResult
-        = Actions.ActOnTemplateIdType(TemplateTy::make(TemplateId->Template),
+        = Actions.ActOnTemplateIdType(TemplateId->Template,
                                       TemplateId->TemplateNameLoc,
                                       TemplateId->LAngleLoc,
                                       TemplateArgsPtr,
@@ -862,7 +891,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
         // but it actually has a definition. Most likely, this was
         // meant to be an explicit specialization, but the user forgot
         // the '<>' after 'template'.
-        assert(TUK == Action::TUK_Definition && "Expected a definition here");
+        assert(TUK == Sema::TUK_Definition && "Expected a definition here");
 
         SourceLocation LAngleLoc
           = PP.getLocForEndOfToken(TemplateInfo.TemplateLoc);
@@ -887,19 +916,19 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       TagOrTempResult
         = Actions.ActOnClassTemplateSpecialization(getCurScope(), TagType, TUK,
                        StartLoc, SS,
-                       TemplateTy::make(TemplateId->Template),
+                       TemplateId->Template,
                        TemplateId->TemplateNameLoc,
                        TemplateId->LAngleLoc,
                        TemplateArgsPtr,
                        TemplateId->RAngleLoc,
                        AttrList,
-                       Action::MultiTemplateParamsArg(Actions,
+                       MultiTemplateParamsArg(Actions,
                                     TemplateParams? &(*TemplateParams)[0] : 0,
                                  TemplateParams? TemplateParams->size() : 0));
     }
     TemplateId->Destroy();
   } else if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
-             TUK == Action::TUK_Declaration) {
+             TUK == Sema::TUK_Declaration) {
     // Explicit instantiation of a member of a class template
     // specialization, e.g.,
     //
@@ -913,7 +942,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                                            NameLoc, AttrList);
   } else {
     if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
-        TUK == Action::TUK_Definition) {
+        TUK == Sema::TUK_Definition) {
       // FIXME: Diagnose this particular error.
     }
 
@@ -922,7 +951,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     // Declaration or definition of a class type
     TagOrTempResult = Actions.ActOnTag(getCurScope(), TagType, TUK, StartLoc, SS,
                                        Name, NameLoc, AttrList, AS,
-                                  Action::MultiTemplateParamsArg(Actions,
+                                       MultiTemplateParamsArg(Actions,
                                     TemplateParams? &(*TemplateParams)[0] : 0,
                                     TemplateParams? TemplateParams->size() : 0),
                                        Owned, IsDependent);
@@ -935,7 +964,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   }
 
   // If there is a body, parse it and inform the actions module.
-  if (TUK == Action::TUK_Definition) {
+  if (TUK == Sema::TUK_Definition) {
     assert(Tok.is(tok::l_brace) ||
            (getLang().CPlusPlus && Tok.is(tok::colon)));
     if (getLang().CPlusPlus)
@@ -944,27 +973,25 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       ParseStructUnionBody(StartLoc, TagType, TagOrTempResult.get());
   }
 
-  void *Result;
+  // FIXME: The DeclSpec should keep the locations of both the keyword and the
+  // name (if there is one).
+  SourceLocation TSTLoc = NameLoc.isValid()? NameLoc : StartLoc;
+
+  const char *PrevSpec = 0;
+  unsigned DiagID;
+  bool Result;
   if (!TypeResult.isInvalid()) {
-    TagType = DeclSpec::TST_typename;
-    Result = TypeResult.get();
-    Owned = false;
+    Result = DS.SetTypeSpecType(DeclSpec::TST_typename, TSTLoc,
+                                PrevSpec, DiagID, TypeResult.get());
   } else if (!TagOrTempResult.isInvalid()) {
-    Result = TagOrTempResult.get().getAs<void>();
+    Result = DS.SetTypeSpecType(TagType, TSTLoc, PrevSpec, DiagID,
+                                TagOrTempResult.get(), Owned);
   } else {
     DS.SetTypeSpecError();
     return;
   }
 
-  const char *PrevSpec = 0;
-  unsigned DiagID;
-
-  // FIXME: The DeclSpec should keep the locations of both the keyword and the
-  // name (if there is one).
-  SourceLocation TSTLoc = NameLoc.isValid()? NameLoc : StartLoc;
-
-  if (DS.SetTypeSpecType(TagType, TSTLoc, PrevSpec, DiagID,
-                         Result, Owned))
+  if (Result)
     Diag(StartLoc, DiagID) << PrevSpec;
 
   // At this point, we've successfully parsed a class-specifier in 'definition'
@@ -974,7 +1001,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   // the end of the declaration and recover that way.
   //
   // This switch enumerates the valid "follow" set for definition.
-  if (TUK == Action::TUK_Definition) {
+  if (TUK == Sema::TUK_Definition) {
     bool ExpectedSemi = true;
     switch (Tok.getKind()) {
     default: break;
@@ -1048,12 +1075,12 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
 ///       base-specifier-list:
 ///         base-specifier '...'[opt]
 ///         base-specifier-list ',' base-specifier '...'[opt]
-void Parser::ParseBaseClause(DeclPtrTy ClassDecl) {
+void Parser::ParseBaseClause(Decl *ClassDecl) {
   assert(Tok.is(tok::colon) && "Not a base clause");
   ConsumeToken();
 
   // Build up an array of parsed base specifiers.
-  llvm::SmallVector<BaseTy *, 8> BaseInfo;
+  llvm::SmallVector<CXXBaseSpecifier *, 8> BaseInfo;
 
   while (true) {
     // Parse a base-specifier.
@@ -1090,7 +1117,7 @@ void Parser::ParseBaseClause(DeclPtrTy ClassDecl) {
 ///                        class-name
 ///         access-specifier 'virtual'[opt] ::[opt] nested-name-specifier[opt]
 ///                        class-name
-Parser::BaseResult Parser::ParseBaseSpecifier(DeclPtrTy ClassDecl) {
+Parser::BaseResult Parser::ParseBaseSpecifier(Decl *ClassDecl) {
   bool IsVirtual = false;
   SourceLocation StartLoc = Tok.getLocation();
 
@@ -1120,8 +1147,7 @@ Parser::BaseResult Parser::ParseBaseSpecifier(DeclPtrTy ClassDecl) {
 
   // Parse optional '::' and optional nested-name-specifier.
   CXXScopeSpec SS;
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0,
-                                 /*EnteringContext=*/false);
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(), /*EnteringContext=*/false);
 
   // The location of the base class itself.
   SourceLocation BaseLoc = Tok.getLocation();
@@ -1158,7 +1184,7 @@ AccessSpecifier Parser::getAccessSpecifierIfPresent() const {
 }
 
 void Parser::HandleMemberFunctionDefaultArgs(Declarator& DeclaratorInfo,
-                                             DeclPtrTy ThisDecl) {
+                                             Decl *ThisDecl) {
   // We just declared a member function. If this member function
   // has any default arguments, we'll need to parse them later.
   LateParsedMethodDeclaration *LateMethod = 0;
@@ -1218,7 +1244,8 @@ void Parser::HandleMemberFunctionDefaultArgs(Declarator& DeclaratorInfo,
 ///         '=' constant-expression
 ///
 void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
-                                       const ParsedTemplateInfo &TemplateInfo) {
+                                       const ParsedTemplateInfo &TemplateInfo,
+                                       ParsingDeclRAIIObject *TemplateDiags) {
   // Access declarations.
   if (!TemplateInfo.Kind &&
       (Tok.is(tok::identifier) || Tok.is(tok::coloncolon)) &&
@@ -1233,11 +1260,11 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     if (isAccessDecl) {
       // Collect the scope specifier token we annotated earlier.
       CXXScopeSpec SS;
-      ParseOptionalCXXScopeSpecifier(SS, /*ObjectType*/ 0, false);
+      ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
 
       // Try to parse an unqualified-id.
       UnqualifiedId Name;
-      if (ParseUnqualifiedId(SS, false, true, true, /*ObjectType*/ 0, Name)) {
+      if (ParseUnqualifiedId(SS, false, true, true, ParsedType(), Name)) {
         SkipUntil(tok::semi);
         return;
       }
@@ -1281,7 +1308,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     // __extension__ silences extension warnings in the subexpression.
     ExtensionRAIIObject O(Diags);  // Use RAII to do this.
     ConsumeToken();
-    return ParseCXXClassMemberDeclaration(AS, TemplateInfo);
+    return ParseCXXClassMemberDeclaration(AS, TemplateInfo, TemplateDiags);
   }
 
   // Don't parse FOO:BAR as if it were a typo for FOO::BAR, in this context it
@@ -1317,17 +1344,19 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
   SourceLocation DSStart = Tok.getLocation();
   // decl-specifier-seq:
   // Parse the common declaration-specifiers piece.
-  ParsingDeclSpec DS(*this);
+  ParsingDeclSpec DS(*this, TemplateDiags);
   DS.AddAttributes(AttrList.AttrList);
   ParseDeclarationSpecifiers(DS, TemplateInfo, AS, DSC_class);
 
-  Action::MultiTemplateParamsArg TemplateParams(Actions,
+  MultiTemplateParamsArg TemplateParams(Actions,
       TemplateInfo.TemplateParams? TemplateInfo.TemplateParams->data() : 0,
       TemplateInfo.TemplateParams? TemplateInfo.TemplateParams->size() : 0);
 
   if (Tok.is(tok::semi)) {
     ConsumeToken();
-    Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS, DS);
+    Decl *TheDecl =
+      Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS, DS);
+    DS.complete(TheDecl);
     return;
   }
 
@@ -1385,9 +1414,9 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
   //   member-declarator
   //   member-declarator-list ',' member-declarator
 
-  llvm::SmallVector<DeclPtrTy, 8> DeclsInGroup;
-  OwningExprResult BitfieldSize(Actions);
-  OwningExprResult Init(Actions);
+  llvm::SmallVector<Decl *, 8> DeclsInGroup;
+  ExprResult BitfieldSize;
+  ExprResult Init;
   bool Deleted = false;
 
   while (1) {
@@ -1426,7 +1455,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     // If a simple-asm-expr is present, parse it.
     if (Tok.is(tok::kw_asm)) {
       SourceLocation Loc;
-      OwningExprResult AsmLabel(ParseSimpleAsm(&Loc));
+      ExprResult AsmLabel(ParseSimpleAsm(&Loc));
       if (AsmLabel.isInvalid())
         SkipUntil(tok::comma, true, true);
  
@@ -1445,7 +1474,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     // this call will *not* return the created decl; It will return null.
     // See Sema::ActOnCXXMemberDeclarator for details.
 
-    DeclPtrTy ThisDecl;
+    Decl *ThisDecl = 0;
     if (DS.isFriendSpecified()) {
       // TODO: handle initializers, bitfields, 'delete'
       ThisDecl = Actions.ActOnFriendFunctionDecl(getCurScope(), DeclaratorInfo,
@@ -1515,14 +1544,13 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
 ///         access-specifier ':' member-specification[opt]
 ///
 void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
-                                         unsigned TagType, DeclPtrTy TagDecl) {
+                                         unsigned TagType, Decl *TagDecl) {
   assert((TagType == DeclSpec::TST_struct ||
          TagType == DeclSpec::TST_union  ||
          TagType == DeclSpec::TST_class) && "Invalid TagType!");
 
-  PrettyStackTraceActionsDecl CrashInfo(TagDecl, RecordLoc, Actions,
-                                        PP.getSourceManager(),
-                                        "parsing struct/union/class body");
+  PrettyDeclStackTraceEntry CrashInfo(Actions, TagDecl, RecordLoc,
+                                      "parsing struct/union/class body");
 
   // Determine whether this is a non-nested class. Note that local
   // classes are *not* considered to be nested classes.
@@ -1681,21 +1709,28 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
 /// [C++]  mem-initializer-list:
 ///          mem-initializer
 ///          mem-initializer , mem-initializer-list
-void Parser::ParseConstructorInitializer(DeclPtrTy ConstructorDecl) {
+void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) {
   assert(Tok.is(tok::colon) && "Constructor initializer always starts with ':'");
 
   SourceLocation ColonLoc = ConsumeToken();
 
-  llvm::SmallVector<MemInitTy*, 4> MemInitializers;
+  llvm::SmallVector<CXXBaseOrMemberInitializer*, 4> MemInitializers;
   bool AnyErrors = false;
 
   do {
-    MemInitResult MemInit = ParseMemInitializer(ConstructorDecl);
-    if (!MemInit.isInvalid())
-      MemInitializers.push_back(MemInit.get());
-    else
-      AnyErrors = true;
-
+    if (Tok.is(tok::code_completion)) {
+      Actions.CodeCompleteConstructorInitializer(ConstructorDecl, 
+                                                 MemInitializers.data(), 
+                                                 MemInitializers.size());
+      ConsumeCodeCompletionToken();
+    } else {
+      MemInitResult MemInit = ParseMemInitializer(ConstructorDecl);
+      if (!MemInit.isInvalid())
+        MemInitializers.push_back(MemInit.get());
+      else
+        AnyErrors = true;
+    }
+    
     if (Tok.is(tok::comma))
       ConsumeToken();
     else if (Tok.is(tok::l_brace))
@@ -1724,11 +1759,11 @@ void Parser::ParseConstructorInitializer(DeclPtrTy ConstructorDecl) {
 /// [C++] mem-initializer-id:
 ///         '::'[opt] nested-name-specifier[opt] class-name
 ///         identifier
-Parser::MemInitResult Parser::ParseMemInitializer(DeclPtrTy ConstructorDecl) {
+Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
   // parse '::'[opt] nested-name-specifier[opt]
   CXXScopeSpec SS;
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false);
-  TypeTy *TemplateTypeTy = 0;
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
+  ParsedType TemplateTypeTy;
   if (Tok.is(tok::annot_template_id)) {
     TemplateIdAnnotation *TemplateId
       = static_cast<TemplateIdAnnotation *>(Tok.getAnnotationValue());
@@ -1736,7 +1771,7 @@ Parser::MemInitResult Parser::ParseMemInitializer(DeclPtrTy ConstructorDecl) {
         TemplateId->Kind == TNK_Dependent_template_name) {
       AnnotateTemplateIdTokenAsType(&SS);
       assert(Tok.is(tok::annot_typename) && "template-id -> type failed");
-      TemplateTypeTy = Tok.getAnnotationValue();
+      TemplateTypeTy = getTypeAnnotation(Tok);
     }
   }
   if (!TemplateTypeTy && Tok.isNot(tok::identifier)) {
@@ -1785,9 +1820,9 @@ Parser::MemInitResult Parser::ParseMemInitializer(DeclPtrTy ConstructorDecl) {
 ///         type-id-list ',' type-id
 ///
 bool Parser::ParseExceptionSpecification(SourceLocation &EndLoc,
-                                         llvm::SmallVector<TypeTy*, 2>
+                                         llvm::SmallVectorImpl<ParsedType>
                                              &Exceptions,
-                                         llvm::SmallVector<SourceRange, 2>
+                                         llvm::SmallVectorImpl<SourceRange>
                                              &Ranges,
                                          bool &hasAnyExceptionSpec) {
   assert(Tok.is(tok::kw_throw) && "expected throw");
@@ -1831,7 +1866,7 @@ bool Parser::ParseExceptionSpecification(SourceLocation &EndLoc,
 /// \brief We have just started parsing the definition of a new class,
 /// so push that class onto our stack of classes that is currently
 /// being parsed.
-void Parser::PushParsingClass(DeclPtrTy ClassDecl, bool NonNestedClass) {
+void Parser::PushParsingClass(Decl *ClassDecl, bool NonNestedClass) {
   assert((NonNestedClass || !ClassStack.empty()) &&
          "Nested class without outer class");
   ClassStack.push(new ParsingClass(ClassDecl, NonNestedClass));
@@ -1997,7 +2032,7 @@ CXX0XAttributeList Parser::ParseCXX0XAttributes(SourceLocation *EndLoc) {
         }
         SourceLocation ParamLoc = ConsumeParen();
 
-        OwningExprResult ArgExpr = ParseCXX0XAlignArgument(ParamLoc);
+        ExprResult ArgExpr = ParseCXX0XAlignArgument(ParamLoc);
 
         MatchRHSPunctuation(tok::r_paren, ParamLoc);
 
@@ -2042,15 +2077,14 @@ CXX0XAttributeList Parser::ParseCXX0XAttributes(SourceLocation *EndLoc) {
 ///
 /// [C++0x] 'align' '(' type-id ')'
 /// [C++0x] 'align' '(' assignment-expression ')'
-Parser::OwningExprResult Parser::ParseCXX0XAlignArgument(SourceLocation Start) {
+ExprResult Parser::ParseCXX0XAlignArgument(SourceLocation Start) {
   if (isTypeIdInParens()) {
-    EnterExpressionEvaluationContext Unevaluated(Actions,
-                                                  Action::Unevaluated);
+    EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
     SourceLocation TypeLoc = Tok.getLocation();
-    TypeTy *Ty = ParseTypeName().get();
+    ParsedType Ty = ParseTypeName().get();
     SourceRange TypeRange(Start, Tok.getLocation());
-    return Actions.ActOnSizeOfAlignOfExpr(TypeLoc, false, true, Ty,
-                                              TypeRange);
+    return Actions.ActOnSizeOfAlignOfExpr(TypeLoc, false, true,
+                                          Ty.getAsOpaquePtr(), TypeRange);
   } else
     return ParseConstantExpression();
 }
diff --git a/lib/Parse/ParseExpr.cpp b/lib/Parse/ParseExpr.cpp
index e7973f73ea15..c4beab191d37 100644
--- a/lib/Parse/ParseExpr.cpp
+++ b/lib/Parse/ParseExpr.cpp
@@ -20,9 +20,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Parse/Parser.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "RAIIObjectsForParser.h"
 #include "llvm/ADT/SmallVector.h"
@@ -30,8 +30,7 @@
 using namespace clang;
 
 /// getBinOpPrecedence - Return the precedence of the specified binary operator
-/// token.  This returns:
-///
+/// token.
 static prec::Level getBinOpPrecedence(tok::TokenKind Kind,
                                       bool GreaterThanIsOperator,
                                       bool CPlusPlus0x) {
@@ -176,8 +175,8 @@ static prec::Level getBinOpPrecedence(tok::TokenKind Kind,
 ///         assignment-expression
 ///         expression ',' assignment-expression
 ///
-Parser::OwningExprResult Parser::ParseExpression() {
-  OwningExprResult LHS(ParseAssignmentExpression());
+ExprResult Parser::ParseExpression() {
+  ExprResult LHS(ParseAssignmentExpression());
   if (LHS.isInvalid()) return move(LHS);
 
   return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
@@ -188,9 +187,9 @@ Parser::OwningExprResult Parser::ParseExpression() {
 /// routine is necessary to disambiguate @try-statement from,
 /// for example, @encode-expression.
 ///
-Parser::OwningExprResult
+ExprResult
 Parser::ParseExpressionWithLeadingAt(SourceLocation AtLoc) {
-  OwningExprResult LHS(ParseObjCAtExpression(AtLoc));
+  ExprResult LHS(ParseObjCAtExpression(AtLoc));
   if (LHS.isInvalid()) return move(LHS);
 
   return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
@@ -199,9 +198,9 @@ Parser::ParseExpressionWithLeadingAt(SourceLocation AtLoc) {
 /// This routine is called when a leading '__extension__' is seen and
 /// consumed.  This is necessary because the token gets consumed in the
 /// process of disambiguating between an expression and a declaration.
-Parser::OwningExprResult
+ExprResult
 Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
-  OwningExprResult LHS(Actions, true);
+  ExprResult LHS(true);
   {
     // Silence extension warnings in the sub-expression
     ExtensionRAIIObject O(Diags);
@@ -211,27 +210,27 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
   }
 
   LHS = Actions.ActOnUnaryOp(getCurScope(), ExtLoc, tok::kw___extension__,
-                             move(LHS));
+                             LHS.take());
   if (LHS.isInvalid()) return move(LHS);
 
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
+  return ParseRHSOfBinaryExpression(LHS.take(), prec::Comma);
 }
 
 /// ParseAssignmentExpression - Parse an expr that doesn't include commas.
 ///
-Parser::OwningExprResult Parser::ParseAssignmentExpression() {
+ExprResult Parser::ParseAssignmentExpression() {
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_Expression);
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
     ConsumeCodeCompletionToken();
   }
 
   if (Tok.is(tok::kw_throw))
     return ParseThrowExpression();
 
-  OwningExprResult LHS(ParseCastExpression(false));
+  ExprResult LHS(ParseCastExpression(false));
   if (LHS.isInvalid()) return move(LHS);
 
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Assignment);
+  return ParseRHSOfBinaryExpression(LHS.take(), prec::Assignment);
 }
 
 /// ParseAssignmentExprWithObjCMessageExprStart - Parse an assignment expression
@@ -242,38 +241,38 @@ Parser::OwningExprResult Parser::ParseAssignmentExpression() {
 ///
 /// Since this handles full assignment-expression's, it handles postfix
 /// expressions and other binary operators for these expressions as well.
-Parser::OwningExprResult
+ExprResult
 Parser::ParseAssignmentExprWithObjCMessageExprStart(SourceLocation LBracLoc,
                                                     SourceLocation SuperLoc,
-                                                    TypeTy *ReceiverType,
-                                                    ExprArg ReceiverExpr) {
-  OwningExprResult R(ParseObjCMessageExpressionBody(LBracLoc, SuperLoc,
-                                                    ReceiverType,
-                                                    move(ReceiverExpr)));
+                                                    ParsedType ReceiverType,
+                                                    Expr *ReceiverExpr) {
+  ExprResult R
+    = ParseObjCMessageExpressionBody(LBracLoc, SuperLoc,
+                                     ReceiverType, ReceiverExpr);
   if (R.isInvalid()) return move(R);
-  R = ParsePostfixExpressionSuffix(move(R));
+  R = ParsePostfixExpressionSuffix(R.take());
   if (R.isInvalid()) return move(R);
-  return ParseRHSOfBinaryExpression(move(R), prec::Assignment);
+  return ParseRHSOfBinaryExpression(R.take(), prec::Assignment);
 }
 
 
-Parser::OwningExprResult Parser::ParseConstantExpression() {
+ExprResult Parser::ParseConstantExpression() {
   // C++ [basic.def.odr]p2:
   //   An expression is potentially evaluated unless it appears where an
   //   integral constant expression is required (see 5.19) [...].
   EnterExpressionEvaluationContext Unevaluated(Actions,
-                                               Action::Unevaluated);
+                                               Sema::Unevaluated);
 
-  OwningExprResult LHS(ParseCastExpression(false));
+  ExprResult LHS(ParseCastExpression(false));
   if (LHS.isInvalid()) return move(LHS);
 
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Conditional);
+  return ParseRHSOfBinaryExpression(LHS.take(), prec::Conditional);
 }
 
 /// ParseRHSOfBinaryExpression - Parse a binary expression that starts with
 /// LHS and has a precedence of at least MinPrec.
-Parser::OwningExprResult
-Parser::ParseRHSOfBinaryExpression(OwningExprResult LHS, prec::Level MinPrec) {
+ExprResult
+Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
   prec::Level NextTokPrec = getBinOpPrecedence(Tok.getKind(),
                                                GreaterThanIsOperator,
                                                getLang().CPlusPlus0x);
@@ -291,7 +290,7 @@ Parser::ParseRHSOfBinaryExpression(OwningExprResult LHS, prec::Level MinPrec) {
     ConsumeToken();
 
     // Special case handling for the ternary operator.
-    OwningExprResult TernaryMiddle(Actions, true);
+    ExprResult TernaryMiddle(true);
     if (NextTokPrec == prec::Conditional) {
       if (Tok.isNot(tok::colon)) {
         // Don't parse FOO:BAR as if it were a typo for FOO::BAR.
@@ -358,7 +357,7 @@ Parser::ParseRHSOfBinaryExpression(OwningExprResult LHS, prec::Level MinPrec) {
     // Therefore we need some special-casing here.
     // Also note that the third operand of the conditional operator is
     // an assignment-expression in C++.
-    OwningExprResult RHS(Actions);
+    ExprResult RHS;
     if (getLang().CPlusPlus && NextTokPrec <= prec::Conditional)
       RHS = ParseAssignmentExpression();
     else
@@ -385,7 +384,7 @@ Parser::ParseRHSOfBinaryExpression(OwningExprResult LHS, prec::Level MinPrec) {
       // is okay, to bind exactly as tightly.  For example, compile A=B=C=D as
       // A=(B=(C=D)), where each paren is a level of recursion here.
       // The function takes ownership of the RHS.
-      RHS = ParseRHSOfBinaryExpression(move(RHS), 
+      RHS = ParseRHSOfBinaryExpression(RHS.get(), 
                             static_cast<prec::Level>(ThisPrec + !isRightAssoc));
       if (RHS.isInvalid())
         return move(RHS);
@@ -408,11 +407,11 @@ Parser::ParseRHSOfBinaryExpression(OwningExprResult LHS, prec::Level MinPrec) {
                                      Actions.getExprRange(RHS.get()).getEnd()));
 
         LHS = Actions.ActOnBinOp(getCurScope(), OpToken.getLocation(),
-                                 OpToken.getKind(), move(LHS), move(RHS));
+                                 OpToken.getKind(), LHS.take(), RHS.take());
       } else
         LHS = Actions.ActOnConditionalOp(OpToken.getLocation(), ColonLoc,
-                                         move(LHS), move(TernaryMiddle),
-                                         move(RHS));
+                                         LHS.take(), TernaryMiddle.take(),
+                                         RHS.take());
     }
   }
 }
@@ -422,11 +421,11 @@ Parser::ParseRHSOfBinaryExpression(OwningExprResult LHS, prec::Level MinPrec) {
 /// id-expression that is the operand of address-of gets special treatment
 /// due to member pointers.
 ///
-Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
+ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
                                                      bool isAddressOfOperand,
-                                                     TypeTy *TypeOfCast) {
+                                                     ParsedType TypeOfCast) {
   bool NotCastExpr;
-  OwningExprResult Res = ParseCastExpression(isUnaryExpression,
+  ExprResult Res = ParseCastExpression(isUnaryExpression,
                                              isAddressOfOperand,
                                              NotCastExpr,
                                              TypeOfCast);
@@ -527,14 +526,14 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 ///                   '::'[opt] 'delete' '[' ']' cast-expression
 ///
 /// [GNU] unary-type-trait:
-///                   '__has_nothrow_assign'                  [TODO]
-///                   '__has_nothrow_copy'                    [TODO]
-///                   '__has_nothrow_constructor'             [TODO]
+///                   '__has_nothrow_assign'
+///                   '__has_nothrow_copy'
+///                   '__has_nothrow_constructor'
 ///                   '__has_trivial_assign'                  [TODO]
 ///                   '__has_trivial_copy'                    [TODO]
 ///                   '__has_trivial_constructor'
 ///                   '__has_trivial_destructor'
-///                   '__has_virtual_destructor'              [TODO]
+///                   '__has_virtual_destructor'
 ///                   '__is_abstract'                         [TODO]
 ///                   '__is_class'
 ///                   '__is_empty'                            [TODO]
@@ -546,11 +545,11 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 /// [GNU] binary-type-trait:
 ///                   '__is_base_of'                          [TODO]
 ///
-Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
+ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
                                                      bool isAddressOfOperand,
                                                      bool &NotCastExpr,
-                                                     TypeTy *TypeOfCast) {
-  OwningExprResult Res(Actions);
+                                                     ParsedType TypeOfCast) {
+  ExprResult Res;
   tok::TokenKind SavedKind = Tok.getKind();
   NotCastExpr = false;
 
@@ -561,16 +560,17 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   // expression, or statement expression.
   //
   // If the parsed tokens consist of a primary-expression, the cases below
-  // call ParsePostfixExpressionSuffix to handle the postfix expression
-  // suffixes.  Cases that cannot be followed by postfix exprs should
-  // return without invoking ParsePostfixExpressionSuffix.
+  // break out of the switch;  at the end we call ParsePostfixExpressionSuffix
+  // to handle the postfix expression suffixes.  Cases that cannot be followed
+  // by postfix exprs should return without invoking
+  // ParsePostfixExpressionSuffix.
   switch (SavedKind) {
   case tok::l_paren: {
     // If this expression is limited to being a unary-expression, the parent can
     // not start a cast expression.
     ParenParseOption ParenExprType =
-      isUnaryExpression ? CompoundLiteral : CastExpr;
-    TypeTy *CastTy;
+      (isUnaryExpression && !getLang().CPlusPlus)? CompoundLiteral : CastExpr;
+    ParsedType CastTy;
     SourceLocation LParenLoc = Tok.getLocation();
     SourceLocation RParenLoc;
     
@@ -597,8 +597,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
       return move(Res);
     }
 
-    // These can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   }
 
     // primary-expression
@@ -608,9 +607,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 
     Res = Actions.ActOnNumericConstant(Tok);
     ConsumeToken();
-
-    // These can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
 
   case tok::kw_true:
   case tok::kw_false:
@@ -661,9 +658,12 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
       
       Res = Actions.ActOnClassPropertyRefExpr(II, PropertyName,
                                               ILoc, PropertyLoc);
-      // These can be followed by postfix-expr pieces.
-      return ParsePostfixExpressionSuffix(move(Res));
+      break;
     }
+
+    // Make sure to pass down the right value for isAddressOfOperand.
+    if (isAddressOfOperand && isPostfixExpressionSuffixStart())
+      isAddressOfOperand = false;
    
     // Function designators are allowed to be undeclared (C99 6.5.1p2), so we
     // need to know whether or not this identifier is a function designator or
@@ -672,29 +672,23 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     CXXScopeSpec ScopeSpec;
     Name.setIdentifier(&II, ILoc);
     Res = Actions.ActOnIdExpression(getCurScope(), ScopeSpec, Name, 
-                                    Tok.is(tok::l_paren), false);
-
-    // These can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+                                    Tok.is(tok::l_paren), isAddressOfOperand);
+    break;
   }
   case tok::char_constant:     // constant: character-constant
     Res = Actions.ActOnCharacterConstant(Tok);
     ConsumeToken();
-    // These can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   case tok::kw___func__:       // primary-expression: __func__ [C99 6.4.2.2]
   case tok::kw___FUNCTION__:   // primary-expression: __FUNCTION__ [GNU]
   case tok::kw___PRETTY_FUNCTION__:  // primary-expression: __P..Y_F..N__ [GNU]
     Res = Actions.ActOnPredefinedExpr(Tok.getLocation(), SavedKind);
     ConsumeToken();
-    // These can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   case tok::string_literal:    // primary-expression: string-literal
   case tok::wide_string_literal:
     Res = ParseStringLiteralExpression();
-    if (Res.isInvalid()) return move(Res);
-    // This can be followed by postfix-expr pieces (e.g. "foo"[1]).
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   case tok::kw___builtin_va_arg:
   case tok::kw___builtin_offsetof:
   case tok::kw___builtin_choose_expr:
@@ -703,12 +697,16 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw___null:
     return Actions.ActOnGNUNullExpr(ConsumeToken());
     break;
-  case tok::plusplus:      // unary-expression: '++' unary-expression
-  case tok::minusminus: {  // unary-expression: '--' unary-expression
+  case tok::plusplus:      // unary-expression: '++' unary-expression [C99]
+  case tok::minusminus: {  // unary-expression: '--' unary-expression [C99]
+    // C++ [expr.unary] has:
+    //   unary-expression:
+    //     ++ cast-expression
+    //     -- cast-expression
     SourceLocation SavedLoc = ConsumeToken();
-    Res = ParseCastExpression(true);
+    Res = ParseCastExpression(!getLang().CPlusPlus);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, move(Res));
+      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
     return move(Res);
   }
   case tok::amp: {         // unary-expression: '&' cast-expression
@@ -716,7 +714,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(false, true);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, move(Res));
+      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
     return move(Res);
   }
 
@@ -730,7 +728,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(false);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, move(Res));
+      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
     return move(Res);
   }
 
@@ -740,7 +738,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     SourceLocation SavedLoc = ConsumeToken();
     Res = ParseCastExpression(false);
     if (!Res.isInvalid())
-      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, move(Res));
+      Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
     return move(Res);
   }
   case tok::kw_sizeof:     // unary-expression: 'sizeof' unary-expression
@@ -766,16 +764,13 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw_reinterpret_cast:
   case tok::kw_static_cast:
     Res = ParseCXXCasts();
-    // These can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   case tok::kw_typeid:
     Res = ParseCXXTypeid();
-    // This can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   case tok::kw_this:
     Res = ParseCXXThis();
-    // This can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
 
   case tok::kw_char:
   case tok::kw_wchar_t:
@@ -814,8 +809,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
                          << DS.getSourceRange());
 
     Res = ParseCXXTypeConstructExpression(DS);
-    // This can be followed by postfix-expr pieces.
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   }
 
   case tok::annot_cxxscope: { // [C++] id-expression: qualified-id
@@ -836,7 +830,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
         // type, translate it into a type and continue parsing as a
         // cast expression.
         CXXScopeSpec SS;
-        ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false);
+        ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
         AnnotateTemplateIdTokenAsType(&SS);
         return ParseCastExpression(isUnaryExpression, isAddressOfOperand,
                                    NotCastExpr, TypeOfCast);
@@ -845,7 +839,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 
     // Parse as an id-expression.
     Res = ParseCXXIdExpression(isAddressOfOperand);
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
   }
 
   case tok::annot_template_id: { // [C++]          template-id
@@ -865,7 +859,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 
   case tok::kw_operator: // [C++] id-expression: operator/conversion-function-id
     Res = ParseCXXIdExpression(isAddressOfOperand);
-    return ParsePostfixExpressionSuffix(move(Res));
+    break;
 
   case tok::coloncolon: {
     // ::foo::bar -> global qualified name etc.   If TryAnnotateTypeOrScopeToken
@@ -906,6 +900,10 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::kw___has_trivial_copy:
   case tok::kw___has_trivial_assign:
   case tok::kw___has_trivial_destructor:
+  case tok::kw___has_nothrow_assign:
+  case tok::kw___has_nothrow_copy:
+  case tok::kw___has_nothrow_constructor:
+  case tok::kw___has_virtual_destructor:
     return ParseUnaryTypeTrait();
 
   case tok::at: {
@@ -915,7 +913,7 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
   case tok::caret:
     return ParsePostfixExpressionSuffix(ParseBlockLiteralExpression());
   case tok::code_completion:
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_Expression);
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Expression);
     ConsumeCodeCompletionToken();
     return ParseCastExpression(isUnaryExpression, isAddressOfOperand, 
                                NotCastExpr, TypeOfCast);
@@ -929,8 +927,9 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     return ExprError();
   }
 
-  // unreachable.
-  abort();
+  // These can be followed by postfix-expr pieces.
+  if (Res.isInvalid()) return move(Res);
+  return ParsePostfixExpressionSuffix(Res.get());
 }
 
 /// ParsePostfixExpressionSuffix - Once the leading part of a postfix-expression
@@ -951,8 +950,8 @@ Parser::OwningExprResult Parser::ParseCastExpression(bool isUnaryExpression,
 ///         argument-expression
 ///         argument-expression-list ',' assignment-expression
 ///
-Parser::OwningExprResult
-Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
+ExprResult
+Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
   // Now that the primary-expression piece of the postfix-expression has been
   // parsed, see if there are any postfix-expression pieces here.
   SourceLocation Loc;
@@ -972,13 +971,13 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
         return move(LHS);
           
       Loc = ConsumeBracket();
-      OwningExprResult Idx(ParseExpression());
+      ExprResult Idx(ParseExpression());
 
       SourceLocation RLoc = Tok.getLocation();
 
       if (!LHS.isInvalid() && !Idx.isInvalid() && Tok.is(tok::r_square)) {
-        LHS = Actions.ActOnArraySubscriptExpr(getCurScope(), move(LHS), Loc,
-                                              move(Idx), RLoc);
+        LHS = Actions.ActOnArraySubscriptExpr(getCurScope(), LHS.take(), Loc,
+                                              Idx.take(), RLoc);
       } else
         LHS = ExprError();
 
@@ -1004,7 +1003,7 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
       }
       
       if (Tok.isNot(tok::r_paren)) {
-        if (ParseExpressionList(ArgExprs, CommaLocs, &Action::CodeCompleteCall,
+        if (ParseExpressionList(ArgExprs, CommaLocs, &Sema::CodeCompleteCall,
                                 LHS.get())) {
           SkipUntil(tok::r_paren);
           return ExprError();
@@ -1020,7 +1019,7 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
       if (!LHS.isInvalid()) {
         assert((ArgExprs.size() == 0 || ArgExprs.size()-1 == CommaLocs.size())&&
                "Unexpected number of commas!");
-        LHS = Actions.ActOnCallExpr(getCurScope(), move(LHS), Loc,
+        LHS = Actions.ActOnCallExpr(getCurScope(), LHS.take(), Loc,
                                     move_arg(ArgExprs), CommaLocs.data(),
                                     Tok.getLocation());
       }
@@ -1036,10 +1035,10 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
       SourceLocation OpLoc = ConsumeToken();  // Eat the "." or "->" token.
 
       CXXScopeSpec SS;
-      Action::TypeTy *ObjectType = 0;
+      ParsedType ObjectType;
       bool MayBePseudoDestructor = false;
       if (getLang().CPlusPlus && !LHS.isInvalid()) {
-        LHS = Actions.ActOnStartCXXMemberReference(getCurScope(), move(LHS),
+        LHS = Actions.ActOnStartCXXMemberReference(getCurScope(), LHS.take(),
                                                    OpLoc, OpKind, ObjectType,
                                                    MayBePseudoDestructor);
         if (LHS.isInvalid())
@@ -1048,7 +1047,7 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
         ParseOptionalCXXScopeSpecifier(SS, ObjectType, false,
                                        &MayBePseudoDestructor);
         if (SS.isNotEmpty())
-          ObjectType = 0;
+          ObjectType = ParsedType();
       }
 
       if (Tok.is(tok::code_completion)) {
@@ -1059,8 +1058,8 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
         ConsumeCodeCompletionToken();
       }
       
-      if (MayBePseudoDestructor) {
-        LHS = ParseCXXPseudoDestructor(move(LHS), OpLoc, OpKind, SS, 
+      if (MayBePseudoDestructor && !LHS.isInvalid()) {
+        LHS = ParseCXXPseudoDestructor(LHS.take(), OpLoc, OpKind, SS, 
                                        ObjectType);
         break;
       }
@@ -1080,7 +1079,7 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
         return ExprError();
       
       if (!LHS.isInvalid())
-        LHS = Actions.ActOnMemberAccessExpr(getCurScope(), move(LHS), OpLoc, 
+        LHS = Actions.ActOnMemberAccessExpr(getCurScope(), LHS.take(), OpLoc, 
                                             OpKind, SS, Name, ObjCImpDecl,
                                             Tok.is(tok::l_paren));
       break;
@@ -1089,7 +1088,7 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
     case tok::minusminus:  // postfix-expression: postfix-expression '--'
       if (!LHS.isInvalid()) {
         LHS = Actions.ActOnPostfixUnaryOp(getCurScope(), Tok.getLocation(),
-                                          Tok.getKind(), move(LHS));
+                                          Tok.getKind(), LHS.take());
       }
       ConsumeToken();
       break;
@@ -1114,17 +1113,17 @@ Parser::ParsePostfixExpressionSuffix(OwningExprResult LHS) {
 ///           typeof ( type-name )
 /// [GNU/C++] typeof unary-expression
 ///
-Parser::OwningExprResult
+ExprResult
 Parser::ParseExprAfterTypeofSizeofAlignof(const Token &OpTok,
                                           bool &isCastExpr,
-                                          TypeTy *&CastTy,
+                                          ParsedType &CastTy,
                                           SourceRange &CastRange) {
 
   assert((OpTok.is(tok::kw_typeof)    || OpTok.is(tok::kw_sizeof) ||
           OpTok.is(tok::kw___alignof) || OpTok.is(tok::kw_alignof)) &&
           "Not a typeof/sizeof/alignof expression!");
 
-  OwningExprResult Operand(Actions);
+  ExprResult Operand;
 
   // If the operand doesn't start with an '(', it must be an expression.
   if (Tok.isNot(tok::l_paren)) {
@@ -1141,7 +1140,7 @@ Parser::ParseExprAfterTypeofSizeofAlignof(const Token &OpTok,
     // The GNU typeof and alignof extensions also behave as unevaluated
     // operands.
     EnterExpressionEvaluationContext Unevaluated(Actions,
-                                                 Action::Unevaluated);
+                                                 Sema::Unevaluated);
     Operand = ParseCastExpression(true/*isUnaryExpression*/);
   } else {
     // If it starts with a '(', we know that it is either a parenthesized
@@ -1158,10 +1157,9 @@ Parser::ParseExprAfterTypeofSizeofAlignof(const Token &OpTok,
     // The GNU typeof and alignof extensions also behave as unevaluated
     // operands.
     EnterExpressionEvaluationContext Unevaluated(Actions,
-                                                 Action::Unevaluated);
+                                                 Sema::Unevaluated);
     Operand = ParseParenExpression(ExprType, true/*stopIfCastExpr*/, 
-                                   0/*TypeOfCast*/,
-                                   CastTy, RParenLoc);
+                                   ParsedType(), CastTy, RParenLoc);
     CastRange = SourceRange(LParenLoc, RParenLoc);
 
     // If ParseParenExpression parsed a '(typename)' sequence only, then this is
@@ -1171,10 +1169,14 @@ Parser::ParseExprAfterTypeofSizeofAlignof(const Token &OpTok,
       return ExprEmpty();
     }
 
-    // If this is a parenthesized expression, it is the start of a
-    // unary-expression, but doesn't include any postfix pieces.  Parse these
-    // now if present.
-    Operand = ParsePostfixExpressionSuffix(move(Operand));
+    if (getLang().CPlusPlus || OpTok.isNot(tok::kw_typeof)) {
+      // GNU typeof in C requires the expression to be parenthesized. Not so for
+      // sizeof/alignof or in C++. Therefore, the parenthesized expression is
+      // the start of a unary-expression, but doesn't include any postfix 
+      // pieces. Parse these now if present.
+      if (!Operand.isInvalid())
+        Operand = ParsePostfixExpressionSuffix(Operand.get());
+    }
   }
 
   // If we get here, the operand to the typeof/sizeof/alignof was an expresion.
@@ -1190,7 +1192,7 @@ Parser::ParseExprAfterTypeofSizeofAlignof(const Token &OpTok,
 /// [GNU]   '__alignof' unary-expression
 /// [GNU]   '__alignof' '(' type-name ')'
 /// [C++0x] 'alignof' '(' type-id ')'
-Parser::OwningExprResult Parser::ParseSizeofAlignofExpression() {
+ExprResult Parser::ParseSizeofAlignofExpression() {
   assert((Tok.is(tok::kw_sizeof) || Tok.is(tok::kw___alignof)
           || Tok.is(tok::kw_alignof)) &&
          "Not a sizeof/alignof expression!");
@@ -1198,9 +1200,9 @@ Parser::OwningExprResult Parser::ParseSizeofAlignofExpression() {
   ConsumeToken();
 
   bool isCastExpr;
-  TypeTy *CastTy;
+  ParsedType CastTy;
   SourceRange CastRange;
-  OwningExprResult Operand = ParseExprAfterTypeofSizeofAlignof(OpTok,
+  ExprResult Operand = ParseExprAfterTypeofSizeofAlignof(OpTok,
                                                                isCastExpr,
                                                                CastTy,
                                                                CastRange);
@@ -1208,7 +1210,8 @@ Parser::OwningExprResult Parser::ParseSizeofAlignofExpression() {
   if (isCastExpr)
     return Actions.ActOnSizeOfAlignOfExpr(OpTok.getLocation(),
                                           OpTok.is(tok::kw_sizeof),
-                                          /*isType=*/true, CastTy,
+                                          /*isType=*/true,
+                                          CastTy.getAsOpaquePtr(),
                                           CastRange);
 
   // If we get here, the operand to the sizeof/alignof was an expresion.
@@ -1234,8 +1237,8 @@ Parser::OwningExprResult Parser::ParseSizeofAlignofExpression() {
 /// [GNU]   offsetof-member-designator '.' identifier
 /// [GNU]   offsetof-member-designator '[' expression ']'
 ///
-Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
-  OwningExprResult Res(Actions);
+ExprResult Parser::ParseBuiltinPrimaryExpression() {
+  ExprResult Res;
   const IdentifierInfo *BuiltinII = Tok.getIdentifierInfo();
 
   tok::TokenKind T = Tok.getKind();
@@ -1252,7 +1255,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
   switch (T) {
   default: assert(0 && "Not a builtin primary expression!");
   case tok::kw___builtin_va_arg: {
-    OwningExprResult Expr(ParseAssignmentExpression());
+    ExprResult Expr(ParseAssignmentExpression());
     if (Expr.isInvalid()) {
       SkipUntil(tok::r_paren);
       return ExprError();
@@ -1270,7 +1273,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     if (Ty.isInvalid())
       Res = ExprError();
     else
-      Res = Actions.ActOnVAArg(StartLoc, move(Expr), Ty.get(), ConsumeParen());
+      Res = Actions.ActOnVAArg(StartLoc, Expr.take(), Ty.get(), ConsumeParen());
     break;
   }
   case tok::kw___builtin_offsetof: {
@@ -1292,9 +1295,9 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     }
 
     // Keep track of the various subcomponents we see.
-    llvm::SmallVector<Action::OffsetOfComponent, 4> Comps;
+    llvm::SmallVector<Sema::OffsetOfComponent, 4> Comps;
 
-    Comps.push_back(Action::OffsetOfComponent());
+    Comps.push_back(Sema::OffsetOfComponent());
     Comps.back().isBrackets = false;
     Comps.back().U.IdentInfo = Tok.getIdentifierInfo();
     Comps.back().LocStart = Comps.back().LocEnd = ConsumeToken();
@@ -1303,7 +1306,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     while (1) {
       if (Tok.is(tok::period)) {
         // offsetof-member-designator: offsetof-member-designator '.' identifier
-        Comps.push_back(Action::OffsetOfComponent());
+        Comps.push_back(Sema::OffsetOfComponent());
         Comps.back().isBrackets = false;
         Comps.back().LocStart = ConsumeToken();
 
@@ -1317,7 +1320,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
 
       } else if (Tok.is(tok::l_square)) {
         // offsetof-member-designator: offsetof-member-design '[' expression ']'
-        Comps.push_back(Action::OffsetOfComponent());
+        Comps.push_back(Sema::OffsetOfComponent());
         Comps.back().isBrackets = true;
         Comps.back().LocStart = ConsumeBracket();
         Res = ParseExpression();
@@ -1346,7 +1349,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     break;
   }
   case tok::kw___builtin_choose_expr: {
-    OwningExprResult Cond(ParseAssignmentExpression());
+    ExprResult Cond(ParseAssignmentExpression());
     if (Cond.isInvalid()) {
       SkipUntil(tok::r_paren);
       return move(Cond);
@@ -1354,7 +1357,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "",tok::r_paren))
       return ExprError();
 
-    OwningExprResult Expr1(ParseAssignmentExpression());
+    ExprResult Expr1(ParseAssignmentExpression());
     if (Expr1.isInvalid()) {
       SkipUntil(tok::r_paren);
       return move(Expr1);
@@ -1362,7 +1365,7 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "",tok::r_paren))
       return ExprError();
 
-    OwningExprResult Expr2(ParseAssignmentExpression());
+    ExprResult Expr2(ParseAssignmentExpression());
     if (Expr2.isInvalid()) {
       SkipUntil(tok::r_paren);
       return move(Expr2);
@@ -1371,8 +1374,8 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
       Diag(Tok, diag::err_expected_rparen);
       return ExprError();
     }
-    Res = Actions.ActOnChooseExpr(StartLoc, move(Cond), move(Expr1),
-                                  move(Expr2), ConsumeParen());
+    Res = Actions.ActOnChooseExpr(StartLoc, Cond.take(), Expr1.take(),
+                                  Expr2.take(), ConsumeParen());
     break;
   }
   case tok::kw___builtin_types_compatible_p:
@@ -1396,9 +1399,12 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
     break;
   }
 
+  if (Res.isInvalid())
+    return ExprError();
+
   // These can be followed by postfix-expr pieces because they are
   // primary-expressions.
-  return ParsePostfixExpressionSuffix(move(Res));
+  return ParsePostfixExpressionSuffix(Res.take());
 }
 
 /// ParseParenExpression - This parses the unit that starts with a '(' token,
@@ -1415,25 +1421,25 @@ Parser::OwningExprResult Parser::ParseBuiltinPrimaryExpression() {
 ///       cast-expression: [C99 6.5.4]
 ///         '(' type-name ')' cast-expression
 ///
-Parser::OwningExprResult
+ExprResult
 Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
-                             TypeTy *TypeOfCast, TypeTy *&CastTy,
+                             ParsedType TypeOfCast, ParsedType &CastTy,
                              SourceLocation &RParenLoc) {
   assert(Tok.is(tok::l_paren) && "Not a paren expr!");
   GreaterThanIsOperatorScope G(GreaterThanIsOperator, true);
   SourceLocation OpenLoc = ConsumeParen();
-  OwningExprResult Result(Actions, true);
+  ExprResult Result(true);
   bool isAmbiguousTypeId;
-  CastTy = 0;
+  CastTy = ParsedType();
 
   if (ExprType >= CompoundStmt && Tok.is(tok::l_brace)) {
     Diag(Tok, diag::ext_gnu_statement_expr);
-    OwningStmtResult Stmt(ParseCompoundStatement(0, true));
+    StmtResult Stmt(ParseCompoundStatement(0, true));
     ExprType = CompoundStmt;
 
     // If the substmt parsed correctly, build the AST node.
     if (!Stmt.isInvalid() && Tok.is(tok::r_paren))
-      Result = Actions.ActOnStmtExpr(OpenLoc, move(Stmt), Tok.getLocation());
+      Result = Actions.ActOnStmtExpr(OpenLoc, Stmt.take(), Tok.getLocation());
 
   } else if (ExprType >= CompoundLiteral &&
              isTypeIdInParens(isAmbiguousTypeId)) {
@@ -1473,7 +1479,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       // Note that this doesn't parse the subsequent cast-expression, it just
       // returns the parsed type to the callee.
       if (stopIfCastExpr)
-        return OwningExprResult(Actions);
+        return ExprResult();
       
       // Reject the cast of super idiom in ObjC.
       if (Tok.is(tok::identifier) && getLang().ObjC1 &&
@@ -1490,7 +1496,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       Result = ParseCastExpression(false, false, CastTy);
       if (!Result.isInvalid())
         Result = Actions.ActOnCastExpr(getCurScope(), OpenLoc, CastTy, RParenLoc,
-                                       move(Result));
+                                       Result.take());
       return move(Result);
     }
 
@@ -1510,7 +1516,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     Result = ParseExpression();
     ExprType = SimpleExpr;
     if (!Result.isInvalid() && Tok.is(tok::r_paren))
-      Result = Actions.ActOnParenExpr(OpenLoc, Tok.getLocation(), move(Result));
+      Result = Actions.ActOnParenExpr(OpenLoc, Tok.getLocation(), Result.take());
   }
 
   // Match the ')'.
@@ -1534,16 +1540,16 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
 ///         '(' type-name ')' '{' initializer-list '}'
 ///         '(' type-name ')' '{' initializer-list ',' '}'
 ///
-Parser::OwningExprResult
-Parser::ParseCompoundLiteralExpression(TypeTy *Ty,
+ExprResult
+Parser::ParseCompoundLiteralExpression(ParsedType Ty,
                                        SourceLocation LParenLoc,
                                        SourceLocation RParenLoc) {
   assert(Tok.is(tok::l_brace) && "Not a compound literal!");
   if (!getLang().C99)   // Compound literals don't exist in C90.
     Diag(LParenLoc, diag::ext_c99_compound_literal);
-  OwningExprResult Result = ParseInitializer();
+  ExprResult Result = ParseInitializer();
   if (!Result.isInvalid() && Ty)
-    return Actions.ActOnCompoundLiteral(LParenLoc, Ty, RParenLoc, move(Result));
+    return Actions.ActOnCompoundLiteral(LParenLoc, Ty, RParenLoc, Result.take());
   return move(Result);
 }
 
@@ -1553,7 +1559,7 @@ Parser::ParseCompoundLiteralExpression(TypeTy *Ty,
 ///
 ///       primary-expression: [C99 6.5.1]
 ///         string-literal
-Parser::OwningExprResult Parser::ParseStringLiteralExpression() {
+ExprResult Parser::ParseStringLiteralExpression() {
   assert(isTokenStringLiteral() && "Not a string literal!");
 
   // String concat.  Note that keywords like __func__ and __FUNCTION__ are not
@@ -1579,12 +1585,13 @@ Parser::OwningExprResult Parser::ParseStringLiteralExpression() {
 /// [C++]   assignment-expression
 /// [C++]   expression-list , assignment-expression
 ///
-bool Parser::ParseExpressionList(ExprListTy &Exprs, CommaLocsTy &CommaLocs,
-                                 void (Action::*Completer)(Scope *S, 
-                                                           void *Data,
-                                                           ExprTy **Args,
+bool Parser::ParseExpressionList(llvm::SmallVectorImpl<Expr*> &Exprs,
+                            llvm::SmallVectorImpl<SourceLocation> &CommaLocs,
+                                 void (Sema::*Completer)(Scope *S, 
+                                                           Expr *Data,
+                                                           Expr **Args,
                                                            unsigned NumArgs),
-                                 void *Data) {
+                                 Expr *Data) {
   while (1) {
     if (Tok.is(tok::code_completion)) {
       if (Completer)
@@ -1592,7 +1599,7 @@ bool Parser::ParseExpressionList(ExprListTy &Exprs, CommaLocsTy &CommaLocs,
       ConsumeCodeCompletionToken();
     }
     
-    OwningExprResult Expr(ParseAssignmentExpression());
+    ExprResult Expr(ParseAssignmentExpression());
     if (Expr.isInvalid())
       return true;
 
@@ -1642,7 +1649,7 @@ void Parser::ParseBlockId() {
 /// [clang] block-args:
 /// [clang]   '(' parameter-list ')'
 ///
-Parser::OwningExprResult Parser::ParseBlockLiteralExpression() {
+ExprResult Parser::ParseBlockLiteralExpression() {
   assert(Tok.is(tok::caret) && "block literal starts with ^");
   SourceLocation CaretLoc = ConsumeToken();
 
@@ -1717,7 +1724,7 @@ Parser::OwningExprResult Parser::ParseBlockLiteralExpression() {
   }
 
 
-  OwningExprResult Result(Actions, true);
+  ExprResult Result(true);
   if (!Tok.is(tok::l_brace)) {
     // Saw something like: ^expr
     Diag(Tok, diag::err_expected_expression);
@@ -1725,9 +1732,9 @@ Parser::OwningExprResult Parser::ParseBlockLiteralExpression() {
     return ExprError();
   }
 
-  OwningStmtResult Stmt(ParseCompoundStatementBody());
+  StmtResult Stmt(ParseCompoundStatementBody());
   if (!Stmt.isInvalid())
-    Result = Actions.ActOnBlockStmtExpr(CaretLoc, move(Stmt), getCurScope());
+    Result = Actions.ActOnBlockStmtExpr(CaretLoc, Stmt.take(), getCurScope());
   else
     Actions.ActOnBlockError(CaretLoc, getCurScope());
   return move(Result);
diff --git a/lib/Parse/ParseExprCXX.cpp b/lib/Parse/ParseExprCXX.cpp
index 579d3bde4912..5041a212924a 100644
--- a/lib/Parse/ParseExprCXX.cpp
+++ b/lib/Parse/ParseExprCXX.cpp
@@ -13,8 +13,8 @@
 
 #include "clang/Parse/ParseDiagnostic.h"
 #include "clang/Parse/Parser.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace clang;
@@ -57,14 +57,14 @@ using namespace clang;
 ///
 /// \returns true if there was an error parsing a scope specifier
 bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
-                                            Action::TypeTy *ObjectType,
+                                            ParsedType ObjectType,
                                             bool EnteringContext,
                                             bool *MayBePseudoDestructor) {
   assert(getLang().CPlusPlus &&
          "Call sites of this function should be guarded by checking for C++");
 
   if (Tok.is(tok::annot_cxxscope)) {
-    SS.setScopeRep(Tok.getAnnotationValue());
+    SS.setScopeRep(static_cast<NestedNameSpecifier*>(Tok.getAnnotationValue()));
     SS.setRange(Tok.getAnnotationRange());
     ConsumeToken();
     return false;
@@ -104,7 +104,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
       //
       // To implement this, we clear out the object type as soon as we've
       // seen a leading '::' or part of a nested-name-specifier.
-      ObjectType = 0;
+      ObjectType = ParsedType();
       
       if (Tok.is(tok::code_completion)) {
         // Code completion for a nested-name-specifier, where the code
@@ -212,13 +212,13 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
           HasScopeSpecifier = true;
         }
 
-        if (TypeToken.getAnnotationValue())
-          SS.setScopeRep(
-            Actions.ActOnCXXNestedNameSpecifier(getCurScope(), SS,
-                                                TypeToken.getAnnotationValue(),
+        if (ParsedType T = getTypeAnnotation(TypeToken)) {
+          CXXScopeTy *Scope =
+            Actions.ActOnCXXNestedNameSpecifier(getCurScope(), SS, T,
                                                 TypeToken.getAnnotationRange(),
-                                                CCLoc));
-        else
+                                                CCLoc);
+          SS.setScopeRep(Scope);
+        } else
           SS.setScopeRep(0);
         SS.setEndLoc(CCLoc);
         continue;
@@ -294,6 +294,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
       TemplateName.setIdentifier(&II, Tok.getLocation());
       bool MemberOfUnknownSpecialization;
       if (TemplateNameKind TNK = Actions.isTemplateName(getCurScope(), SS, 
+                                              /*hasTemplateKeyword=*/false,
                                                         TemplateName,
                                                         ObjectType,
                                                         EnteringContext,
@@ -396,40 +397,27 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
 /// the only place where a qualified-id naming a non-static class member may
 /// appear.
 ///
-Parser::OwningExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
+ExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
   // qualified-id:
   //   '::'[opt] nested-name-specifier 'template'[opt] unqualified-id
   //   '::' unqualified-id
   //
   CXXScopeSpec SS;
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false);
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(), false);
   
   UnqualifiedId Name;
   if (ParseUnqualifiedId(SS, 
                          /*EnteringContext=*/false, 
                          /*AllowDestructorName=*/false, 
                          /*AllowConstructorName=*/false, 
-                         /*ObjectType=*/0,
+                         /*ObjectType=*/ ParsedType(),
                          Name))
     return ExprError();
 
   // This is only the direct operand of an & operator if it is not
   // followed by a postfix-expression suffix.
-  if (isAddressOfOperand) {
-    switch (Tok.getKind()) {
-    case tok::l_square:
-    case tok::l_paren:
-    case tok::arrow:
-    case tok::period:
-    case tok::plusplus:
-    case tok::minusminus:
-      isAddressOfOperand = false;
-      break;
-
-    default:
-      break;
-    }
-  }
+  if (isAddressOfOperand && isPostfixExpressionSuffixStart())
+    isAddressOfOperand = false;
   
   return Actions.ActOnIdExpression(getCurScope(), SS, Name, Tok.is(tok::l_paren),
                                    isAddressOfOperand);
@@ -445,7 +433,7 @@ Parser::OwningExprResult Parser::ParseCXXIdExpression(bool isAddressOfOperand) {
 ///         'reinterpret_cast' '<' type-name '>' '(' expression ')'
 ///         'const_cast' '<' type-name '>' '(' expression ')'
 ///
-Parser::OwningExprResult Parser::ParseCXXCasts() {
+ExprResult Parser::ParseCXXCasts() {
   tok::TokenKind Kind = Tok.getKind();
   const char *CastName = 0;     // For error messages
 
@@ -474,7 +462,7 @@ Parser::OwningExprResult Parser::ParseCXXCasts() {
   if (ExpectAndConsume(tok::l_paren, diag::err_expected_lparen_after, CastName))
     return ExprError();
 
-  OwningExprResult Result = ParseExpression();
+  ExprResult Result = ParseExpression();
 
   // Match the ')'.
   RParenLoc = MatchRHSPunctuation(tok::r_paren, LParenLoc);
@@ -483,7 +471,7 @@ Parser::OwningExprResult Parser::ParseCXXCasts() {
     Result = Actions.ActOnCXXNamedCast(OpLoc, Kind,
                                        LAngleBracketLoc, CastTy.get(),
                                        RAngleBracketLoc,
-                                       LParenLoc, move(Result), RParenLoc);
+                                       LParenLoc, Result.take(), RParenLoc);
 
   return move(Result);
 }
@@ -494,7 +482,7 @@ Parser::OwningExprResult Parser::ParseCXXCasts() {
 ///         'typeid' '(' expression ')'
 ///         'typeid' '(' type-id ')'
 ///
-Parser::OwningExprResult Parser::ParseCXXTypeid() {
+ExprResult Parser::ParseCXXTypeid() {
   assert(Tok.is(tok::kw_typeid) && "Not 'typeid'!");
 
   SourceLocation OpLoc = ConsumeToken();
@@ -506,7 +494,7 @@ Parser::OwningExprResult Parser::ParseCXXTypeid() {
       "typeid"))
     return ExprError();
 
-  OwningExprResult Result(Actions);
+  ExprResult Result;
 
   if (isTypeIdInParens()) {
     TypeResult Ty = ParseTypeName();
@@ -518,7 +506,7 @@ Parser::OwningExprResult Parser::ParseCXXTypeid() {
       return ExprError();
 
     Result = Actions.ActOnCXXTypeid(OpLoc, LParenLoc, /*isType=*/true,
-                                    Ty.get(), RParenLoc);
+                                    Ty.get().getAsOpaquePtr(), RParenLoc);
   } else {
     // C++0x [expr.typeid]p3:
     //   When typeid is applied to an expression other than an lvalue of a
@@ -529,7 +517,7 @@ Parser::OwningExprResult Parser::ParseCXXTypeid() {
     // polymorphic class type until after we've parsed the expression, so
     // we the expression is potentially potentially evaluated.
     EnterExpressionEvaluationContext Unevaluated(Actions,
-                                       Action::PotentiallyPotentiallyEvaluated);
+                                       Sema::PotentiallyPotentiallyEvaluated);
     Result = ParseExpression();
 
     // Match the ')'.
@@ -560,11 +548,11 @@ Parser::OwningExprResult Parser::ParseCXXTypeid() {
 ///                 ~type-name 
 ///         ::[opt] nested-name-specifier[opt] ~type-name
 ///       
-Parser::OwningExprResult 
+ExprResult 
 Parser::ParseCXXPseudoDestructor(ExprArg Base, SourceLocation OpLoc,
                                  tok::TokenKind OpKind,
                                  CXXScopeSpec &SS,
-                                 Action::TypeTy *ObjectType) {
+                                 ParsedType ObjectType) {
   // We're parsing either a pseudo-destructor-name or a dependent
   // member access that has the same form as a
   // pseudo-destructor-name. We parse both in the same way and let
@@ -612,7 +600,8 @@ Parser::ParseCXXPseudoDestructor(ExprArg Base, SourceLocation OpLoc,
                                    /*TemplateKWLoc*/SourceLocation()))
     return ExprError();
 
-  return Actions.ActOnPseudoDestructorExpr(getCurScope(), move(Base), OpLoc, OpKind,
+  return Actions.ActOnPseudoDestructorExpr(getCurScope(), Base,
+                                           OpLoc, OpKind,
                                            SS, FirstTypeName, CCLoc,
                                            TildeLoc, SecondTypeName,
                                            Tok.is(tok::l_paren));
@@ -623,7 +612,7 @@ Parser::ParseCXXPseudoDestructor(ExprArg Base, SourceLocation OpLoc,
 ///       boolean-literal: [C++ 2.13.5]
 ///         'true'
 ///         'false'
-Parser::OwningExprResult Parser::ParseCXXBoolLiteral() {
+ExprResult Parser::ParseCXXBoolLiteral() {
   tok::TokenKind Kind = Tok.getKind();
   return Actions.ActOnCXXBoolLiteral(ConsumeToken(), Kind);
 }
@@ -632,7 +621,7 @@ Parser::OwningExprResult Parser::ParseCXXBoolLiteral() {
 ///
 ///       throw-expression: [C++ 15]
 ///         'throw' assignment-expression[opt]
-Parser::OwningExprResult Parser::ParseThrowExpression() {
+ExprResult Parser::ParseThrowExpression() {
   assert(Tok.is(tok::kw_throw) && "Not throw!");
   SourceLocation ThrowLoc = ConsumeToken();           // Eat the throw token.
 
@@ -646,12 +635,12 @@ Parser::OwningExprResult Parser::ParseThrowExpression() {
   case tok::r_brace:
   case tok::colon:
   case tok::comma:
-    return Actions.ActOnCXXThrow(ThrowLoc, ExprArg(Actions));
+    return Actions.ActOnCXXThrow(ThrowLoc, 0);
 
   default:
-    OwningExprResult Expr(ParseAssignmentExpression());
+    ExprResult Expr(ParseAssignmentExpression());
     if (Expr.isInvalid()) return move(Expr);
-    return Actions.ActOnCXXThrow(ThrowLoc, move(Expr));
+    return Actions.ActOnCXXThrow(ThrowLoc, Expr.take());
   }
 }
 
@@ -660,7 +649,7 @@ Parser::OwningExprResult Parser::ParseThrowExpression() {
 /// C++ 9.3.2: In the body of a non-static member function, the keyword this is
 /// a non-lvalue expression whose value is the address of the object for which
 /// the function is called.
-Parser::OwningExprResult Parser::ParseCXXThis() {
+ExprResult Parser::ParseCXXThis() {
   assert(Tok.is(tok::kw_this) && "Not 'this'!");
   SourceLocation ThisLoc = ConsumeToken();
   return Actions.ActOnCXXThis(ThisLoc);
@@ -675,10 +664,10 @@ Parser::OwningExprResult Parser::ParseCXXThis() {
 ///         simple-type-specifier '(' expression-list[opt] ')'      [C++ 5.2.3]
 ///         typename-specifier '(' expression-list[opt] ')'         [TODO]
 ///
-Parser::OwningExprResult
+ExprResult
 Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
   Declarator DeclaratorInfo(DS, Declarator::TypeNameContext);
-  TypeTy *TypeRep = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo).get();
+  ParsedType TypeRep = Actions.ActOnTypeName(getCurScope(), DeclaratorInfo).get();
 
   assert(Tok.is(tok::l_paren) && "Expected '('!");
   SourceLocation LParenLoc = ConsumeParen();
@@ -728,27 +717,27 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
 /// converted to a boolean value.
 ///
 /// \returns true if there was a parsing, false otherwise.
-bool Parser::ParseCXXCondition(OwningExprResult &ExprResult,
-                               DeclPtrTy &DeclResult,
+bool Parser::ParseCXXCondition(ExprResult &ExprOut,
+                               Decl *&DeclOut,
                                SourceLocation Loc,
                                bool ConvertToBoolean) {
   if (Tok.is(tok::code_completion)) {
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_Condition);
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition);
     ConsumeCodeCompletionToken();
   }
 
   if (!isCXXConditionDeclaration()) {
     // Parse the expression.
-    ExprResult = ParseExpression(); // expression
-    DeclResult = DeclPtrTy();
-    if (ExprResult.isInvalid())
+    ExprOut = ParseExpression(); // expression
+    DeclOut = 0;
+    if (ExprOut.isInvalid())
       return true;
 
     // If required, convert to a boolean value.
     if (ConvertToBoolean)
-      ExprResult
-        = Actions.ActOnBooleanCondition(getCurScope(), Loc, move(ExprResult));
-    return ExprResult.isInvalid();
+      ExprOut
+        = Actions.ActOnBooleanCondition(getCurScope(), Loc, ExprOut.get());
+    return ExprOut.isInvalid();
   }
 
   // type-specifier-seq
@@ -762,7 +751,7 @@ bool Parser::ParseCXXCondition(OwningExprResult &ExprResult,
   // simple-asm-expr[opt]
   if (Tok.is(tok::kw_asm)) {
     SourceLocation Loc;
-    OwningExprResult AsmLabel(ParseSimpleAsm(&Loc));
+    ExprResult AsmLabel(ParseSimpleAsm(&Loc));
     if (AsmLabel.isInvalid()) {
       SkipUntil(tok::semi);
       return true;
@@ -779,17 +768,17 @@ bool Parser::ParseCXXCondition(OwningExprResult &ExprResult,
   }
 
   // Type-check the declaration itself.
-  Action::DeclResult Dcl = Actions.ActOnCXXConditionDeclaration(getCurScope(), 
+  DeclResult Dcl = Actions.ActOnCXXConditionDeclaration(getCurScope(), 
                                                                 DeclaratorInfo);
-  DeclResult = Dcl.get();
-  ExprResult = ExprError();
+  DeclOut = Dcl.get();
+  ExprOut = ExprError();
   
   // '=' assignment-expression
   if (Tok.is(tok::equal)) {
     SourceLocation EqualLoc = ConsumeToken();
-    OwningExprResult AssignExpr(ParseAssignmentExpression());
+    ExprResult AssignExpr(ParseAssignmentExpression());
     if (!AssignExpr.isInvalid()) 
-      Actions.AddInitializerToDecl(DeclResult, move(AssignExpr));
+      Actions.AddInitializerToDecl(DeclOut, AssignExpr.take());
   } else {
     // FIXME: C++0x allows a braced-init-list
     Diag(Tok, diag::err_expected_equal_after_declarator);
@@ -874,7 +863,7 @@ void Parser::ParseCXXSimpleTypeSpecifier(DeclSpec &DS) {
   // type-name
   case tok::annot_typename: {
     DS.SetTypeSpecType(DeclSpec::TST_typename, Loc, PrevSpec, DiagID,
-                       Tok.getAnnotationValue());
+                       getTypeAnnotation(Tok));
     break;
   }
 
@@ -1002,7 +991,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
                                           IdentifierInfo *Name,
                                           SourceLocation NameLoc,
                                           bool EnteringContext,
-                                          TypeTy *ObjectType,
+                                          ParsedType ObjectType,
                                           UnqualifiedId &Id,
                                           bool AssumeTemplateId,
                                           SourceLocation TemplateKWLoc) {
@@ -1023,8 +1012,9 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
         return true;
     } else {
       bool MemberOfUnknownSpecialization;
-      TNK = Actions.isTemplateName(getCurScope(), SS, Id, ObjectType, 
-                                   EnteringContext, Template,
+      TNK = Actions.isTemplateName(getCurScope(), SS,
+                                   TemplateKWLoc.isValid(), Id,
+                                   ObjectType, EnteringContext, Template,
                                    MemberOfUnknownSpecialization);
       
       if (TNK == TNK_Non_template && MemberOfUnknownSpecialization &&
@@ -1059,7 +1049,8 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
     UnqualifiedId TemplateName;
     bool MemberOfUnknownSpecialization;
     TemplateName.setIdentifier(Name, NameLoc);
-    TNK = Actions.isTemplateName(getCurScope(), SS, TemplateName, ObjectType, 
+    TNK = Actions.isTemplateName(getCurScope(), SS, TemplateKWLoc.isValid(),
+                                 TemplateName, ObjectType, 
                                  EnteringContext, Template,
                                  MemberOfUnknownSpecialization);
     break;
@@ -1076,11 +1067,12 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
       if (TNK == TNK_Non_template)
         return true;
     } else {
-      TNK = Actions.isTemplateName(getCurScope(), SS, TemplateName, ObjectType, 
+      TNK = Actions.isTemplateName(getCurScope(), SS, TemplateKWLoc.isValid(),
+                                   TemplateName, ObjectType, 
                                    EnteringContext, Template,
                                    MemberOfUnknownSpecialization);
       
-      if (TNK == TNK_Non_template && Id.DestructorName == 0) {
+      if (TNK == TNK_Non_template && !Id.DestructorName.get()) {
         Diag(NameLoc, diag::err_destructor_template_id)
           << Name << SS.getRange();
         return true;        
@@ -1124,7 +1116,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
       TemplateId->TemplateNameLoc = Id.StartLocation;
     }
 
-    TemplateId->Template = Template.getAs<void*>();
+    TemplateId->Template = Template;
     TemplateId->Kind = TNK;
     TemplateId->LAngleLoc = LAngleLoc;
     TemplateId->RAngleLoc = RAngleLoc;
@@ -1142,7 +1134,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
                                      TemplateArgs.size());
   
   // Constructor and destructor names.
-  Action::TypeResult Type
+  TypeResult Type
     = Actions.ActOnTemplateIdType(Template, NameLoc,
                                   LAngleLoc, TemplateArgsPtr,
                                   RAngleLoc);
@@ -1198,7 +1190,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
 ///
 /// \returns true if parsing fails, false otherwise.
 bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
-                                        TypeTy *ObjectType,
+                                        ParsedType ObjectType,
                                         UnqualifiedId &Result) {
   assert(Tok.is(tok::kw_operator) && "Expected 'operator' keyword");
   
@@ -1334,7 +1326,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
   ParseDeclaratorInternal(D, /*DirectDeclParser=*/0);
   
   // Finish up the type.
-  Action::TypeResult Ty = Actions.ActOnTypeName(getCurScope(), D);
+  TypeResult Ty = Actions.ActOnTypeName(getCurScope(), D);
   if (Ty.isInvalid())
     return true;
   
@@ -1377,7 +1369,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
 bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, bool EnteringContext,
                                 bool AllowDestructorName,
                                 bool AllowConstructorName,
-                                TypeTy *ObjectType,
+                                ParsedType ObjectType,
                                 UnqualifiedId &Result) {
 
   // Handle 'A::template B'. This is for template-ids which have not
@@ -1511,17 +1503,17 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, bool EnteringContext,
     SourceLocation ClassNameLoc = ConsumeToken();
     
     if (TemplateSpecified || Tok.is(tok::less)) {
-      Result.setDestructorName(TildeLoc, 0, ClassNameLoc);
+      Result.setDestructorName(TildeLoc, ParsedType(), ClassNameLoc);
       return ParseUnqualifiedIdTemplateId(SS, ClassName, ClassNameLoc,
                                           EnteringContext, ObjectType, Result,
                                           TemplateSpecified, TemplateKWLoc);
     }
     
     // Note that this is a destructor name.
-    Action::TypeTy *Ty = Actions.getDestructorName(TildeLoc, *ClassName, 
-                                                   ClassNameLoc, getCurScope(),
-                                                   SS, ObjectType,
-                                                   EnteringContext);
+    ParsedType Ty = Actions.getDestructorName(TildeLoc, *ClassName, 
+                                              ClassNameLoc, getCurScope(),
+                                              SS, ObjectType,
+                                              EnteringContext);
     if (!Ty)
       return true;
 
@@ -1561,7 +1553,7 @@ bool Parser::ParseUnqualifiedId(CXXScopeSpec &SS, bool EnteringContext,
 ///                   '(' expression-list[opt] ')'
 /// [C++0x]           braced-init-list                                   [TODO]
 ///
-Parser::OwningExprResult
+ExprResult
 Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
   assert(Tok.is(tok::kw_new) && "expected 'new' token");
   ConsumeToken();   // Consume 'new'
@@ -1665,7 +1657,7 @@ void Parser::ParseDirectNewDeclarator(Declarator &D) {
   bool first = true;
   while (Tok.is(tok::l_square)) {
     SourceLocation LLoc = ConsumeBracket();
-    OwningExprResult Size(first ? ParseExpression()
+    ExprResult Size(first ? ParseExpression()
                                 : ParseConstantExpression());
     if (Size.isInvalid()) {
       // Recover
@@ -1694,7 +1686,8 @@ void Parser::ParseDirectNewDeclarator(Declarator &D) {
 ///        new-placement:
 ///                   '(' expression-list ')'
 ///
-bool Parser::ParseExpressionListOrTypeId(ExprListTy &PlacementArgs,
+bool Parser::ParseExpressionListOrTypeId(
+                                   llvm::SmallVectorImpl<Expr*> &PlacementArgs,
                                          Declarator &D) {
   // The '(' was already consumed.
   if (isTypeIdInParens()) {
@@ -1721,7 +1714,7 @@ bool Parser::ParseExpressionListOrTypeId(ExprListTy &PlacementArgs,
 ///        delete-expression:
 ///                   '::'[opt] 'delete' cast-expression
 ///                   '::'[opt] 'delete' '[' ']' cast-expression
-Parser::OwningExprResult
+ExprResult
 Parser::ParseCXXDeleteExpression(bool UseGlobal, SourceLocation Start) {
   assert(Tok.is(tok::kw_delete) && "Expected 'delete' keyword");
   ConsumeToken(); // Consume 'delete'
@@ -1736,11 +1729,11 @@ Parser::ParseCXXDeleteExpression(bool UseGlobal, SourceLocation Start) {
       return ExprError();
   }
 
-  OwningExprResult Operand(ParseCastExpression(false));
+  ExprResult Operand(ParseCastExpression(false));
   if (Operand.isInvalid())
     return move(Operand);
 
-  return Actions.ActOnCXXDelete(Start, UseGlobal, ArrayDelete, move(Operand));
+  return Actions.ActOnCXXDelete(Start, UseGlobal, ArrayDelete, Operand.take());
 }
 
 static UnaryTypeTrait UnaryTypeTraitFromTokKind(tok::TokenKind kind) {
@@ -1772,7 +1765,7 @@ static UnaryTypeTrait UnaryTypeTraitFromTokKind(tok::TokenKind kind) {
 ///       primary-expression:
 /// [GNU]             unary-type-trait '(' type-id ')'
 ///
-Parser::OwningExprResult Parser::ParseUnaryTypeTrait() {
+ExprResult Parser::ParseUnaryTypeTrait() {
   UnaryTypeTrait UTT = UnaryTypeTraitFromTokKind(Tok.getKind());
   SourceLocation Loc = ConsumeToken();
 
@@ -1796,17 +1789,17 @@ Parser::OwningExprResult Parser::ParseUnaryTypeTrait() {
 /// ParseCXXAmbiguousParenExpression - We have parsed the left paren of a
 /// parenthesized ambiguous type-id. This uses tentative parsing to disambiguate
 /// based on the context past the parens.
-Parser::OwningExprResult
+ExprResult
 Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
-                                         TypeTy *&CastTy,
+                                         ParsedType &CastTy,
                                          SourceLocation LParenLoc,
                                          SourceLocation &RParenLoc) {
   assert(getLang().CPlusPlus && "Should only be called for C++!");
   assert(ExprType == CastExpr && "Compound literals are not ambiguous!");
   assert(isTypeIdInParens() && "Not a type-id!");
 
-  OwningExprResult Result(Actions, true);
-  CastTy = 0;
+  ExprResult Result(true);
+  CastTy = ParsedType();
 
   // We need to disambiguate a very ugly part of the C++ syntax:
   //
@@ -1851,7 +1844,8 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
       // will be consumed.
       Result = ParseCastExpression(false/*isUnaryExpression*/,
                                    false/*isAddressofOperand*/,
-                                   NotCastExpr, 0/*TypeOfCast*/);
+                                   NotCastExpr,
+                                   ParsedType()/*TypeOfCast*/);
     }
 
     // If we parsed a cast-expression, it's really a type-id, otherwise it's
@@ -1894,7 +1888,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
     // Result is what ParseCastExpression returned earlier.
     if (!Result.isInvalid())
       Result = Actions.ActOnCastExpr(getCurScope(), LParenLoc, CastTy, RParenLoc,
-                                     move(Result));
+                                     Result.take());
     return move(Result);
   }
 
@@ -1904,7 +1898,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
   ExprType = SimpleExpr;
   Result = ParseExpression();
   if (!Result.isInvalid() && Tok.is(tok::r_paren))
-    Result = Actions.ActOnParenExpr(LParenLoc, Tok.getLocation(), move(Result));
+    Result = Actions.ActOnParenExpr(LParenLoc, Tok.getLocation(), Result.take());
 
   // Match the ')'.
   if (Result.isInvalid()) {
diff --git a/lib/Parse/ParseInit.cpp b/lib/Parse/ParseInit.cpp
index 8451aebc04bd..4347294141ac 100644
--- a/lib/Parse/ParseInit.cpp
+++ b/lib/Parse/ParseInit.cpp
@@ -11,10 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Parse/Designator.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/Scope.h"
+#include "clang/Sema/Designator.h"
+#include "clang/Sema/Scope.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
@@ -71,7 +71,7 @@ static void CheckArrayDesignatorSyntax(Parser &P, SourceLocation Loc,
 /// initializer (because it is an expression).  We need to consider this case
 /// when parsing array designators.
 ///
-Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
+ExprResult Parser::ParseInitializerWithPotentialDesignator() {
 
   // If this is the old-style GNU extension:
   //   designation ::= identifier ':'
@@ -137,7 +137,7 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
     //   [4][foo bar]      -> obsolete GNU designation with objc message send.
     //
     SourceLocation StartLoc = ConsumeBracket();
-    OwningExprResult Idx(Actions);
+    ExprResult Idx;
 
     // If Objective-C is enabled and this is a typename (class message
     // send) or send to 'super', parse this as a message send
@@ -149,8 +149,9 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
           NextToken().isNot(tok::period) && getCurScope()->isInObjcMethodScope()) {
         CheckArrayDesignatorSyntax(*this, StartLoc, Desig);
         return ParseAssignmentExprWithObjCMessageExprStart(StartLoc,
-                                                           ConsumeToken(), 0, 
-                                                           ExprArg(Actions));
+                                                           ConsumeToken(),
+                                                           ParsedType(), 
+                                                           0);
       }
 
       // Parse the receiver, which is either a type or an expression.
@@ -167,35 +168,35 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
         CheckArrayDesignatorSyntax(*this, StartLoc, Desig);
         return ParseAssignmentExprWithObjCMessageExprStart(StartLoc, 
                                                            SourceLocation(), 
-                                                           TypeOrExpr, 
-                                                           ExprArg(Actions));
+                                   ParsedType::getFromOpaquePtr(TypeOrExpr),
+                                                           0);
       }
 
       // If the receiver was an expression, we still don't know
       // whether we have a message send or an array designator; just
       // adopt the expression for further analysis below.
       // FIXME: potentially-potentially evaluated expression above?
-      Idx = OwningExprResult(Actions, TypeOrExpr);
+      Idx = ExprResult(static_cast<Expr*>(TypeOrExpr));
     } else if (getLang().ObjC1 && Tok.is(tok::identifier)) {
       IdentifierInfo *II = Tok.getIdentifierInfo();
       SourceLocation IILoc = Tok.getLocation();
-      TypeTy *ReceiverType;
+      ParsedType ReceiverType;
       // Three cases. This is a message send to a type: [type foo]
       // This is a message send to super:  [super foo]
       // This is a message sent to an expr:  [super.bar foo]
-      switch (Action::ObjCMessageKind Kind
+      switch (Sema::ObjCMessageKind Kind
                 = Actions.getObjCMessageKind(getCurScope(), II, IILoc, 
                                              II == Ident_super,
                                              NextToken().is(tok::period),
                                              ReceiverType)) {
-      case Action::ObjCSuperMessage:
-      case Action::ObjCClassMessage:
+      case Sema::ObjCSuperMessage:
+      case Sema::ObjCClassMessage:
         CheckArrayDesignatorSyntax(*this, StartLoc, Desig);
-        if (Kind == Action::ObjCSuperMessage)
+        if (Kind == Sema::ObjCSuperMessage)
           return ParseAssignmentExprWithObjCMessageExprStart(StartLoc,
                                                              ConsumeToken(),
-                                                             0,
-                                                             ExprArg(Actions));
+                                                             ParsedType(),
+                                                             0);
         ConsumeToken(); // the identifier
         if (!ReceiverType) {
           SkipUntil(tok::r_square);
@@ -205,9 +206,9 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
         return ParseAssignmentExprWithObjCMessageExprStart(StartLoc, 
                                                            SourceLocation(), 
                                                            ReceiverType, 
-                                                           ExprArg(Actions));
+                                                           0);
 
-      case Action::ObjCInstanceMessage:
+      case Sema::ObjCInstanceMessage:
         // Fall through; we'll just parse the expression and
         // (possibly) treat this like an Objective-C message send
         // later.
@@ -239,7 +240,8 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
       CheckArrayDesignatorSyntax(*this, Tok.getLocation(), Desig);
       return ParseAssignmentExprWithObjCMessageExprStart(StartLoc,
                                                          SourceLocation(),
-                                                         0, move(Idx));
+                                                         ParsedType(),
+                                                         Idx.take());
     }
 
     // If this is a normal array designator, remember it.
@@ -250,7 +252,7 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
       Diag(Tok, diag::ext_gnu_array_range);
       SourceLocation EllipsisLoc = ConsumeToken();
 
-      OwningExprResult RHS(ParseConstantExpression());
+      ExprResult RHS(ParseConstantExpression());
       if (RHS.isInvalid()) {
         SkipUntil(tok::r_square);
         return move(RHS);
@@ -307,7 +309,7 @@ Parser::OwningExprResult Parser::ParseInitializerWithPotentialDesignator() {
 ///         designation[opt] initializer
 ///         initializer-list ',' designation[opt] initializer
 ///
-Parser::OwningExprResult Parser::ParseBraceInitializer() {
+ExprResult Parser::ParseBraceInitializer() {
   SourceLocation LBraceLoc = ConsumeBrace();
 
   /// InitExprs - This is the actual list of expressions contained in the
@@ -319,7 +321,7 @@ Parser::OwningExprResult Parser::ParseBraceInitializer() {
     if (!getLang().CPlusPlus)
       Diag(LBraceLoc, diag::ext_gnu_empty_initializer);
     // Match the '}'.
-    return Actions.ActOnInitList(LBraceLoc, Action::MultiExprArg(Actions),
+    return Actions.ActOnInitList(LBraceLoc, MultiExprArg(Actions),
                                  ConsumeBrace());
   }
 
@@ -330,7 +332,7 @@ Parser::OwningExprResult Parser::ParseBraceInitializer() {
 
     // If we know that this cannot be a designation, just parse the nested
     // initializer directly.
-    OwningExprResult SubElt(Actions);
+    ExprResult SubElt;
     if (MayBeDesignationStart(Tok.getKind(), PP))
       SubElt = ParseInitializerWithPotentialDesignator();
     else
diff --git a/lib/Parse/ParseObjc.cpp b/lib/Parse/ParseObjc.cpp
index 68473a551d1a..6861ce940f76 100644
--- a/lib/Parse/ParseObjc.cpp
+++ b/lib/Parse/ParseObjc.cpp
@@ -11,10 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Parse/Parser.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
 #include "clang/Parse/ParseDiagnostic.h"
+#include "clang/Parse/Parser.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/PrettyDeclStackTrace.h"
+#include "clang/Sema/Scope.h"
 #include "llvm/ADT/SmallVector.h"
 using namespace clang;
 
@@ -27,7 +28,7 @@ using namespace clang;
 /// [OBJC]  objc-protocol-definition
 /// [OBJC]  objc-method-definition
 /// [OBJC]  '@' 'end'
-Parser::DeclPtrTy Parser::ParseObjCAtDirectives() {
+Decl *Parser::ParseObjCAtDirectives() {
   SourceLocation AtLoc = ConsumeToken(); // the "@"
 
   if (Tok.is(tok::code_completion)) {
@@ -55,7 +56,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtDirectives() {
   default:
     Diag(AtLoc, diag::err_unexpected_at);
     SkipUntil(tok::semi);
-    return DeclPtrTy();
+    return 0;
   }
 }
 
@@ -63,7 +64,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtDirectives() {
 /// objc-class-declaration:
 ///    '@' 'class' identifier-list ';'
 ///
-Parser::DeclPtrTy Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
+Decl *Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
   ConsumeToken(); // the identifier "class"
   llvm::SmallVector<IdentifierInfo *, 8> ClassNames;
   llvm::SmallVector<SourceLocation, 8> ClassLocs;
@@ -73,7 +74,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_expected_ident);
       SkipUntil(tok::semi);
-      return DeclPtrTy();
+      return 0;
     }
     ClassNames.push_back(Tok.getIdentifierInfo());
     ClassLocs.push_back(Tok.getLocation());
@@ -87,7 +88,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
 
   // Consume the ';'.
   if (ExpectAndConsume(tok::semi, diag::err_expected_semi_after, "@class"))
-    return DeclPtrTy();
+    return 0;
 
   return Actions.ActOnForwardClassDeclaration(atLoc, ClassNames.data(),
                                               ClassLocs.data(),
@@ -122,7 +123,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtClassDeclaration(SourceLocation atLoc) {
 ///     __attribute__((unavailable))
 ///     __attribute__((objc_exception)) - used by NSException on 64-bit
 ///
-Parser::DeclPtrTy Parser::ParseObjCAtInterfaceDeclaration(
+Decl *Parser::ParseObjCAtInterfaceDeclaration(
   SourceLocation atLoc, AttributeList *attrList) {
   assert(Tok.isObjCAtKeyword(tok::objc_interface) &&
          "ParseObjCAtInterfaceDeclaration(): Expected @interface");
@@ -136,7 +137,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtInterfaceDeclaration(
 
   if (Tok.isNot(tok::identifier)) {
     Diag(Tok, diag::err_expected_ident); // missing class or category name.
-    return DeclPtrTy();
+    return 0;
   }
 
   // We have a class or category name - consume it.
@@ -159,27 +160,27 @@ Parser::DeclPtrTy Parser::ParseObjCAtInterfaceDeclaration(
     }
     else if (!getLang().ObjC2) {
       Diag(Tok, diag::err_expected_ident); // missing category name.
-      return DeclPtrTy();
+      return 0;
     }
     if (Tok.isNot(tok::r_paren)) {
       Diag(Tok, diag::err_expected_rparen);
       SkipUntil(tok::r_paren, false); // don't stop at ';'
-      return DeclPtrTy();
+      return 0;
     }
     rparenLoc = ConsumeParen();
     // Next, we need to check for any protocol references.
     SourceLocation LAngleLoc, EndProtoLoc;
-    llvm::SmallVector<DeclPtrTy, 8> ProtocolRefs;
+    llvm::SmallVector<Decl *, 8> ProtocolRefs;
     llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
     if (Tok.is(tok::less) &&
         ParseObjCProtocolReferences(ProtocolRefs, ProtocolLocs, true,
                                     LAngleLoc, EndProtoLoc))
-      return DeclPtrTy();
+      return 0;
 
     if (attrList) // categories don't support attributes.
       Diag(Tok, diag::err_objc_no_attributes_on_category);
 
-    DeclPtrTy CategoryType =
+    Decl *CategoryType =
     Actions.ActOnStartCategoryInterface(atLoc,
                                         nameId, nameLoc,
                                         categoryId, categoryLoc,
@@ -209,21 +210,21 @@ Parser::DeclPtrTy Parser::ParseObjCAtInterfaceDeclaration(
 
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_expected_ident); // missing super class name.
-      return DeclPtrTy();
+      return 0;
     }
     superClassId = Tok.getIdentifierInfo();
     superClassLoc = ConsumeToken();
   }
   // Next, we need to check for any protocol references.
-  llvm::SmallVector<Action::DeclPtrTy, 8> ProtocolRefs;
+  llvm::SmallVector<Decl *, 8> ProtocolRefs;
   llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
   SourceLocation LAngleLoc, EndProtoLoc;
   if (Tok.is(tok::less) &&
       ParseObjCProtocolReferences(ProtocolRefs, ProtocolLocs, true,
                                   LAngleLoc, EndProtoLoc))
-    return DeclPtrTy();
+    return 0;
 
-  DeclPtrTy ClsType =
+  Decl *ClsType =
     Actions.ActOnStartClassInterface(atLoc, nameId, nameLoc,
                                      superClassId, superClassLoc,
                                      ProtocolRefs.data(), ProtocolRefs.size(),
@@ -241,30 +242,30 @@ Parser::DeclPtrTy Parser::ParseObjCAtInterfaceDeclaration(
 /// it's used, but instead it's been lifted to here to support VS2005.
 struct Parser::ObjCPropertyCallback : FieldCallback {
   Parser &P;
-  DeclPtrTy IDecl;
-  llvm::SmallVectorImpl<DeclPtrTy> &Props;
+  Decl *IDecl;
+  llvm::SmallVectorImpl<Decl *> &Props;
   ObjCDeclSpec &OCDS;
   SourceLocation AtLoc;
   tok::ObjCKeywordKind MethodImplKind;
         
-  ObjCPropertyCallback(Parser &P, DeclPtrTy IDecl,
-                       llvm::SmallVectorImpl<DeclPtrTy> &Props,
+  ObjCPropertyCallback(Parser &P, Decl *IDecl,
+                       llvm::SmallVectorImpl<Decl *> &Props,
                        ObjCDeclSpec &OCDS, SourceLocation AtLoc,
                        tok::ObjCKeywordKind MethodImplKind) :
     P(P), IDecl(IDecl), Props(Props), OCDS(OCDS), AtLoc(AtLoc),
     MethodImplKind(MethodImplKind) {
   }
 
-  DeclPtrTy invoke(FieldDeclarator &FD) {
+  Decl *invoke(FieldDeclarator &FD) {
     if (FD.D.getIdentifier() == 0) {
       P.Diag(AtLoc, diag::err_objc_property_requires_field_name)
         << FD.D.getSourceRange();
-      return DeclPtrTy();
+      return 0;
     }
     if (FD.BitfieldSize) {
       P.Diag(AtLoc, diag::err_objc_property_bitfield)
         << FD.D.getSourceRange();
-      return DeclPtrTy();
+      return 0;
     }
 
     // Install the property declarator into interfaceDecl.
@@ -282,7 +283,7 @@ struct Parser::ObjCPropertyCallback : FieldCallback {
                                                      P.PP.getSelectorTable(),
                                                      FD.D.getIdentifier());
     bool isOverridingProperty = false;
-    DeclPtrTy Property =
+    Decl *Property =
       P.Actions.ActOnProperty(P.getCurScope(), AtLoc, FD, OCDS,
                               GetterSel, SetterSel, IDecl,
                               &isOverridingProperty,
@@ -306,10 +307,10 @@ struct Parser::ObjCPropertyCallback : FieldCallback {
 ///     @required
 ///     @optional
 ///
-void Parser::ParseObjCInterfaceDeclList(DeclPtrTy interfaceDecl,
+void Parser::ParseObjCInterfaceDeclList(Decl *interfaceDecl,
                                         tok::ObjCKeywordKind contextKey) {
-  llvm::SmallVector<DeclPtrTy, 32> allMethods;
-  llvm::SmallVector<DeclPtrTy, 16> allProperties;
+  llvm::SmallVector<Decl *, 32> allMethods;
+  llvm::SmallVector<Decl *, 16> allProperties;
   llvm::SmallVector<DeclGroupPtrTy, 8> allTUVariables;
   tok::ObjCKeywordKind MethodImplKind = tok::objc_not_keyword;
 
@@ -318,7 +319,7 @@ void Parser::ParseObjCInterfaceDeclList(DeclPtrTy interfaceDecl,
   while (1) {
     // If this is a method prototype, parse it.
     if (Tok.is(tok::minus) || Tok.is(tok::plus)) {
-      DeclPtrTy methodPrototype =
+      Decl *methodPrototype =
         ParseObjCMethodPrototype(interfaceDecl, MethodImplKind);
       allMethods.push_back(methodPrototype);
       // Consume the ';' here, since ParseObjCMethodPrototype() is re-used for
@@ -329,10 +330,10 @@ void Parser::ParseObjCInterfaceDeclList(DeclPtrTy interfaceDecl,
     }
     if (Tok.is(tok::l_paren)) {
       Diag(Tok, diag::err_expected_minus_or_plus);
-      DeclPtrTy methodPrototype = ParseObjCMethodDecl(Tok.getLocation(), 
-                                                      tok::minus, 
-                                                      interfaceDecl,
-                                                      MethodImplKind);
+      ParseObjCMethodDecl(Tok.getLocation(), 
+                          tok::minus, 
+                          interfaceDecl,
+                          MethodImplKind);
       continue;
     }
     // Ignore excess semicolons.
@@ -348,8 +349,8 @@ void Parser::ParseObjCInterfaceDeclList(DeclPtrTy interfaceDecl,
     // Code completion within an Objective-C interface.
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
-                                  ObjCImpDecl? Action::CCC_ObjCImplementation
-                                             : Action::CCC_ObjCInterface);
+                                  ObjCImpDecl? Sema::PCC_ObjCImplementation
+                                             : Sema::PCC_ObjCInterface);
       ConsumeCodeCompletionToken();
     }
     
@@ -468,8 +469,8 @@ void Parser::ParseObjCInterfaceDeclList(DeclPtrTy interfaceDecl,
 ///     copy
 ///     nonatomic
 ///
-void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, DeclPtrTy ClassDecl,
-                                        DeclPtrTy *Methods, 
+void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, Decl *ClassDecl,
+                                        Decl **Methods, 
                                         unsigned NumMethods) {
   assert(Tok.getKind() == tok::l_paren);
   SourceLocation LHSLoc = ConsumeParen(); // consume '('
@@ -562,14 +563,13 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS, DeclPtrTy ClassDecl,
 ///   objc-method-attributes:         [OBJC2]
 ///     __attribute__((deprecated))
 ///
-Parser::DeclPtrTy Parser::ParseObjCMethodPrototype(DeclPtrTy IDecl,
-                                          tok::ObjCKeywordKind MethodImplKind) {
+Decl *Parser::ParseObjCMethodPrototype(Decl *IDecl,
+                                       tok::ObjCKeywordKind MethodImplKind) {
   assert((Tok.is(tok::minus) || Tok.is(tok::plus)) && "expected +/-");
 
   tok::TokenKind methodType = Tok.getKind();
   SourceLocation mLoc = ConsumeToken();
-
-  DeclPtrTy MDecl = ParseObjCMethodDecl(mLoc, methodType, IDecl,MethodImplKind);
+  Decl *MDecl = ParseObjCMethodDecl(mLoc, methodType, IDecl,MethodImplKind);
   // Since this rule is used for both method declarations and definitions,
   // the caller is (optionally) responsible for consuming the ';'.
   return MDecl;
@@ -584,9 +584,31 @@ Parser::DeclPtrTy Parser::ParseObjCMethodPrototype(DeclPtrTy IDecl,
 ///       in out inout bycopy byref oneway int char float double void _Bool
 ///
 IdentifierInfo *Parser::ParseObjCSelectorPiece(SourceLocation &SelectorLoc) {
+
   switch (Tok.getKind()) {
   default:
     return 0;
+  case tok::ampamp:
+  case tok::ampequal:
+  case tok::amp:
+  case tok::pipe:
+  case tok::tilde:
+  case tok::exclaim:
+  case tok::exclaimequal:
+  case tok::pipepipe:
+  case tok::pipeequal:
+  case tok::caret:
+  case tok::caretequal: {
+    std::string ThisTok(PP.getSpelling(Tok));
+    if (isalpha(ThisTok[0])) {
+      IdentifierInfo *II = &PP.getIdentifierTable().get(ThisTok.data());
+      Tok.setKind(tok::identifier);
+      SelectorLoc = ConsumeToken();
+      return II;
+    }
+    return 0; 
+  }
+      
   case tok::identifier:
   case tok::kw_asm:
   case tok::kw_auto:
@@ -680,8 +702,13 @@ bool Parser::isTokIdentifier_in() const {
 ///     objc-type-qualifier
 ///     objc-type-qualifiers objc-type-qualifier
 ///
-void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS) {
+void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS, bool IsParameter) {
   while (1) {
+    if (Tok.is(tok::code_completion)) {
+      Actions.CodeCompleteObjCPassingType(getCurScope(), DS);
+      ConsumeCodeCompletionToken();
+    }
+    
     if (Tok.isNot(tok::identifier))
       return;
 
@@ -715,16 +742,16 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS) {
 ///     '(' objc-type-qualifiers[opt] type-name ')'
 ///     '(' objc-type-qualifiers[opt] ')'
 ///
-Parser::TypeTy *Parser::ParseObjCTypeName(ObjCDeclSpec &DS) {
+ParsedType Parser::ParseObjCTypeName(ObjCDeclSpec &DS, bool IsParameter) {
   assert(Tok.is(tok::l_paren) && "expected (");
 
   SourceLocation LParenLoc = ConsumeParen();
   SourceLocation TypeStartLoc = Tok.getLocation();
 
   // Parse type qualifiers, in, inout, etc.
-  ParseObjCTypeQualifierList(DS);
+  ParseObjCTypeQualifierList(DS, IsParameter);
 
-  TypeTy *Ty = 0;
+  ParsedType Ty;
   if (isTypeSpecifierQualifier()) {
     TypeResult TypeSpec = ParseTypeName();
     if (!TypeSpec.isInvalid())
@@ -773,23 +800,23 @@ Parser::TypeTy *Parser::ParseObjCTypeName(ObjCDeclSpec &DS) {
 ///   objc-keyword-attributes:         [OBJC2]
 ///     __attribute__((unused))
 ///
-Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
-                                              tok::TokenKind mType,
-                                              DeclPtrTy IDecl,
-                                          tok::ObjCKeywordKind MethodImplKind) {
+Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
+                                  tok::TokenKind mType,
+                                  Decl *IDecl,
+                                  tok::ObjCKeywordKind MethodImplKind) {
   ParsingDeclRAIIObject PD(*this);
 
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, 
-                                       /*ReturnType=*/0, IDecl);
+                                       /*ReturnType=*/ ParsedType(), IDecl);
     ConsumeCodeCompletionToken();
   }
 
   // Parse the return type if present.
-  TypeTy *ReturnType = 0;
+  ParsedType ReturnType;
   ObjCDeclSpec DSRet;
   if (Tok.is(tok::l_paren))
-    ReturnType = ParseObjCTypeName(DSRet);
+    ReturnType = ParseObjCTypeName(DSRet, false);
 
   // If attributes exist before the method, parse them.
   llvm::OwningPtr<AttributeList> MethodAttrs;
@@ -812,7 +839,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
       << SourceRange(mLoc, Tok.getLocation());
     // Skip until we get a ; or {}.
     SkipUntil(tok::r_brace);
-    return DeclPtrTy();
+    return 0;
   }
 
   llvm::SmallVector<DeclaratorChunk::ParamInfo, 8> CParamInfo;
@@ -823,7 +850,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                                           ParseGNUAttributes()));
 
     Selector Sel = PP.getSelectorTable().getNullarySelector(SelIdent);
-    DeclPtrTy Result
+    Decl *Result
          = Actions.ActOnMethodDeclaration(mLoc, Tok.getLocation(),
                                           mType, IDecl, DSRet, ReturnType, Sel,
                                           0, 
@@ -835,10 +862,10 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   }
 
   llvm::SmallVector<IdentifierInfo *, 12> KeyIdents;
-  llvm::SmallVector<Action::ObjCArgInfo, 12> ArgInfos;
+  llvm::SmallVector<Sema::ObjCArgInfo, 12> ArgInfos;
 
   while (1) {
-    Action::ObjCArgInfo ArgInfo;
+    Sema::ObjCArgInfo ArgInfo;
 
     // Each iteration parses a single keyword argument.
     if (Tok.isNot(tok::colon)) {
@@ -847,9 +874,9 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     }
     ConsumeToken(); // Eat the ':'.
 
-    ArgInfo.Type = 0;
+    ArgInfo.Type = ParsedType();
     if (Tok.is(tok::l_paren)) // Parse the argument type if present.
-      ArgInfo.Type = ParseObjCTypeName(ArgInfo.DeclSpec);
+      ArgInfo.Type = ParseObjCTypeName(ArgInfo.DeclSpec, true);
 
     // If attributes exist before the argument name, parse them.
     ArgInfo.ArgAttrs = 0;
@@ -918,7 +945,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     Declarator ParmDecl(DS, Declarator::PrototypeContext);
     ParseDeclarator(ParmDecl);
     IdentifierInfo *ParmII = ParmDecl.getIdentifier();
-    DeclPtrTy Param = Actions.ActOnParamDeclarator(getCurScope(), ParmDecl);
+    Decl *Param = Actions.ActOnParamDeclarator(getCurScope(), ParmDecl);
     CParamInfo.push_back(DeclaratorChunk::ParamInfo(ParmII,
                                                     ParmDecl.getIdentifierLoc(), 
                                                     Param,
@@ -933,10 +960,10 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                                         ParseGNUAttributes()));
 
   if (KeyIdents.size() == 0)
-    return DeclPtrTy();
+    return 0;
   Selector Sel = PP.getSelectorTable().getSelector(KeyIdents.size(),
                                                    &KeyIdents[0]);
-  DeclPtrTy Result
+  Decl *Result
        = Actions.ActOnMethodDeclaration(mLoc, Tok.getLocation(),
                                         mType, IDecl, DSRet, ReturnType, Sel,
                                         &ArgInfos[0], 
@@ -946,7 +973,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
   PD.complete(Result);
   
   // Delete referenced AttributeList objects.
-  for (llvm::SmallVectorImpl<Action::ObjCArgInfo>::iterator
+  for (llvm::SmallVectorImpl<Sema::ObjCArgInfo>::iterator
        I = ArgInfos.begin(), E = ArgInfos.end(); I != E; ++I)
     delete I->ArgAttrs;
   
@@ -957,7 +984,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDecl(SourceLocation mLoc,
 ///     '<' identifier-list '>'
 ///
 bool Parser::
-ParseObjCProtocolReferences(llvm::SmallVectorImpl<Action::DeclPtrTy> &Protocols,
+ParseObjCProtocolReferences(llvm::SmallVectorImpl<Decl *> &Protocols,
                             llvm::SmallVectorImpl<SourceLocation> &ProtocolLocs,
                             bool WarnOnDeclarations,
                             SourceLocation &LAngleLoc, SourceLocation &EndLoc) {
@@ -1024,11 +1051,11 @@ ParseObjCProtocolReferences(llvm::SmallVectorImpl<Action::DeclPtrTy> &Protocols,
 ///   objc-instance-variable-decl:
 ///     struct-declaration
 ///
-void Parser::ParseObjCClassInstanceVariables(DeclPtrTy interfaceDecl,
+void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
                                              tok::ObjCKeywordKind visibility,
                                              SourceLocation atLoc) {
   assert(Tok.is(tok::l_brace) && "expected {");
-  llvm::SmallVector<DeclPtrTy, 32> AllIvarDecls;
+  llvm::SmallVector<Decl *, 32> AllIvarDecls;
 
   ParseScope ClassScope(this, Scope::DeclScope|Scope::ClassScope);
 
@@ -1071,24 +1098,24 @@ void Parser::ParseObjCClassInstanceVariables(DeclPtrTy interfaceDecl,
 
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
-                                       Action::CCC_ObjCInstanceVariableList);
+                                       Sema::PCC_ObjCInstanceVariableList);
       ConsumeCodeCompletionToken();
     }
     
     struct ObjCIvarCallback : FieldCallback {
       Parser &P;
-      DeclPtrTy IDecl;
+      Decl *IDecl;
       tok::ObjCKeywordKind visibility;
-      llvm::SmallVectorImpl<DeclPtrTy> &AllIvarDecls;
+      llvm::SmallVectorImpl<Decl *> &AllIvarDecls;
 
-      ObjCIvarCallback(Parser &P, DeclPtrTy IDecl, tok::ObjCKeywordKind V,
-                       llvm::SmallVectorImpl<DeclPtrTy> &AllIvarDecls) :
+      ObjCIvarCallback(Parser &P, Decl *IDecl, tok::ObjCKeywordKind V,
+                       llvm::SmallVectorImpl<Decl *> &AllIvarDecls) :
         P(P), IDecl(IDecl), visibility(V), AllIvarDecls(AllIvarDecls) {
       }
 
-      DeclPtrTy invoke(FieldDeclarator &FD) {
+      Decl *invoke(FieldDeclarator &FD) {
         // Install the declarator into the interface decl.
-        DeclPtrTy Field
+        Decl *Field
           = P.Actions.ActOnIvar(P.getCurScope(),
                                 FD.D.getDeclSpec().getSourceRange().getBegin(),
                                 IDecl, FD.D, FD.BitfieldSize, visibility);
@@ -1097,7 +1124,7 @@ void Parser::ParseObjCClassInstanceVariables(DeclPtrTy interfaceDecl,
         return Field;
       }
     } Callback(*this, interfaceDecl, visibility, AllIvarDecls);
-
+    
     // Parse all the comma separated declarators.
     DeclSpec DS;
     ParseStructDeclaration(DS, Callback);
@@ -1111,6 +1138,7 @@ void Parser::ParseObjCClassInstanceVariables(DeclPtrTy interfaceDecl,
     }
   }
   SourceLocation RBraceLoc = MatchRHSPunctuation(tok::r_brace, LBraceLoc);
+  Actions.ActOnLastBitfield(RBraceLoc, interfaceDecl, AllIvarDecls);
   // Call ActOnFields() even if we don't have any decls. This is useful
   // for code rewriting tools that need to be aware of the empty list.
   Actions.ActOnFields(getCurScope(), atLoc, interfaceDecl,
@@ -1135,7 +1163,7 @@ void Parser::ParseObjCClassInstanceVariables(DeclPtrTy interfaceDecl,
 ///   "@protocol identifier ;" should be resolved as "@protocol
 ///   identifier-list ;": objc-interface-decl-list may not start with a
 ///   semicolon in the first alternative if objc-protocol-refs are omitted.
-Parser::DeclPtrTy Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
+Decl *Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
                                                       AttributeList *attrList) {
   assert(Tok.isObjCAtKeyword(tok::objc_protocol) &&
          "ParseObjCAtProtocolDeclaration(): Expected @protocol");
@@ -1148,7 +1176,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
 
   if (Tok.isNot(tok::identifier)) {
     Diag(Tok, diag::err_expected_ident); // missing protocol name.
-    return DeclPtrTy();
+    return 0;
   }
   // Save the protocol name, then consume it.
   IdentifierInfo *protocolName = Tok.getIdentifierInfo();
@@ -1171,7 +1199,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
       if (Tok.isNot(tok::identifier)) {
         Diag(Tok, diag::err_expected_ident);
         SkipUntil(tok::semi);
-        return DeclPtrTy();
+        return 0;
       }
       ProtocolRefs.push_back(IdentifierLocPair(Tok.getIdentifierInfo(),
                                                Tok.getLocation()));
@@ -1182,7 +1210,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
     }
     // Consume the ';'.
     if (ExpectAndConsume(tok::semi, diag::err_expected_semi_after, "@protocol"))
-      return DeclPtrTy();
+      return 0;
 
     return Actions.ActOnForwardProtocolDeclaration(AtLoc,
                                                    &ProtocolRefs[0],
@@ -1193,14 +1221,14 @@ Parser::DeclPtrTy Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
   // Last, and definitely not least, parse a protocol declaration.
   SourceLocation LAngleLoc, EndProtoLoc;
 
-  llvm::SmallVector<DeclPtrTy, 8> ProtocolRefs;
+  llvm::SmallVector<Decl *, 8> ProtocolRefs;
   llvm::SmallVector<SourceLocation, 8> ProtocolLocs;
   if (Tok.is(tok::less) &&
       ParseObjCProtocolReferences(ProtocolRefs, ProtocolLocs, false,
                                   LAngleLoc, EndProtoLoc))
-    return DeclPtrTy();
+    return 0;
 
-  DeclPtrTy ProtoType =
+  Decl *ProtoType =
     Actions.ActOnStartProtocolInterface(AtLoc, protocolName, nameLoc,
                                         ProtocolRefs.data(),
                                         ProtocolRefs.size(),
@@ -1220,7 +1248,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc,
 ///
 ///   objc-category-implementation-prologue:
 ///     @implementation identifier ( identifier )
-Parser::DeclPtrTy Parser::ParseObjCAtImplementationDeclaration(
+Decl *Parser::ParseObjCAtImplementationDeclaration(
   SourceLocation atLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_implementation) &&
          "ParseObjCAtImplementationDeclaration(): Expected @implementation");
@@ -1234,7 +1262,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtImplementationDeclaration(
 
   if (Tok.isNot(tok::identifier)) {
     Diag(Tok, diag::err_expected_ident); // missing class or category name.
-    return DeclPtrTy();
+    return 0;
   }
   // We have a class or category name - consume it.
   IdentifierInfo *nameId = Tok.getIdentifierInfo();
@@ -1256,20 +1284,20 @@ Parser::DeclPtrTy Parser::ParseObjCAtImplementationDeclaration(
       categoryLoc = ConsumeToken();
     } else {
       Diag(Tok, diag::err_expected_ident); // missing category name.
-      return DeclPtrTy();
+      return 0;
     }
     if (Tok.isNot(tok::r_paren)) {
       Diag(Tok, diag::err_expected_rparen);
       SkipUntil(tok::r_paren, false); // don't stop at ';'
-      return DeclPtrTy();
+      return 0;
     }
     rparenLoc = ConsumeParen();
-    DeclPtrTy ImplCatType = Actions.ActOnStartCategoryImplementation(
+    Decl *ImplCatType = Actions.ActOnStartCategoryImplementation(
                                     atLoc, nameId, nameLoc, categoryId,
                                     categoryLoc);
     ObjCImpDecl = ImplCatType;
     PendingObjCImpDecl.push_back(ObjCImpDecl);
-    return DeclPtrTy();
+    return 0;
   }
   // We have a class implementation
   SourceLocation superClassLoc;
@@ -1279,12 +1307,12 @@ Parser::DeclPtrTy Parser::ParseObjCAtImplementationDeclaration(
     ConsumeToken();
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_expected_ident); // missing super class name.
-      return DeclPtrTy();
+      return 0;
     }
     superClassId = Tok.getIdentifierInfo();
     superClassLoc = ConsumeToken(); // Consume super class name
   }
-  DeclPtrTy ImplClsType = Actions.ActOnStartClassImplementation(
+  Decl *ImplClsType = Actions.ActOnStartClassImplementation(
                                   atLoc, nameId, nameLoc,
                                   superClassId, superClassLoc);
 
@@ -1294,17 +1322,17 @@ Parser::DeclPtrTy Parser::ParseObjCAtImplementationDeclaration(
   ObjCImpDecl = ImplClsType;
   PendingObjCImpDecl.push_back(ObjCImpDecl);
   
-  return DeclPtrTy();
+  return 0;
 }
 
-Parser::DeclPtrTy Parser::ParseObjCAtEndDeclaration(SourceRange atEnd) {
+Decl *Parser::ParseObjCAtEndDeclaration(SourceRange atEnd) {
   assert(Tok.isObjCAtKeyword(tok::objc_end) &&
          "ParseObjCAtEndDeclaration(): Expected @end");
-  DeclPtrTy Result = ObjCImpDecl;
+  Decl *Result = ObjCImpDecl;
   ConsumeToken(); // the "end" identifier
   if (ObjCImpDecl) {
     Actions.ActOnAtEnd(getCurScope(), atEnd, ObjCImpDecl);
-    ObjCImpDecl = DeclPtrTy();
+    ObjCImpDecl = 0;
     PendingObjCImpDecl.pop_back();
   }
   else {
@@ -1314,10 +1342,11 @@ Parser::DeclPtrTy Parser::ParseObjCAtEndDeclaration(SourceRange atEnd) {
   return Result;
 }
 
-Parser::DeclGroupPtrTy Parser::RetrievePendingObjCImpDecl() {
+Parser::DeclGroupPtrTy Parser::FinishPendingObjCActions() {
+  Actions.DiagnoseUseOfUnimplementedSelectors();
   if (PendingObjCImpDecl.empty())
-    return Actions.ConvertDeclToDeclGroup(DeclPtrTy());
-  DeclPtrTy ImpDecl = PendingObjCImpDecl.pop_back_val();
+    return Actions.ConvertDeclToDeclGroup(0);
+  Decl *ImpDecl = PendingObjCImpDecl.pop_back_val();
   Actions.ActOnAtEnd(getCurScope(), SourceRange(), ImpDecl);
   return Actions.ConvertDeclToDeclGroup(ImpDecl);
 }
@@ -1325,25 +1354,25 @@ Parser::DeclGroupPtrTy Parser::RetrievePendingObjCImpDecl() {
 ///   compatibility-alias-decl:
 ///     @compatibility_alias alias-name  class-name ';'
 ///
-Parser::DeclPtrTy Parser::ParseObjCAtAliasDeclaration(SourceLocation atLoc) {
+Decl *Parser::ParseObjCAtAliasDeclaration(SourceLocation atLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_compatibility_alias) &&
          "ParseObjCAtAliasDeclaration(): Expected @compatibility_alias");
   ConsumeToken(); // consume compatibility_alias
   if (Tok.isNot(tok::identifier)) {
     Diag(Tok, diag::err_expected_ident);
-    return DeclPtrTy();
+    return 0;
   }
   IdentifierInfo *aliasId = Tok.getIdentifierInfo();
   SourceLocation aliasLoc = ConsumeToken(); // consume alias-name
   if (Tok.isNot(tok::identifier)) {
     Diag(Tok, diag::err_expected_ident);
-    return DeclPtrTy();
+    return 0;
   }
   IdentifierInfo *classId = Tok.getIdentifierInfo();
   SourceLocation classLoc = ConsumeToken(); // consume class-name;
   if (Tok.isNot(tok::semi)) {
     Diag(Tok, diag::err_expected_semi_after) << "@compatibility_alias";
-    return DeclPtrTy();
+    return 0;
   }
   return Actions.ActOnCompatiblityAlias(atLoc, aliasId, aliasLoc,
                                         classId, classLoc);
@@ -1360,7 +1389,7 @@ Parser::DeclPtrTy Parser::ParseObjCAtAliasDeclaration(SourceLocation atLoc) {
 ///     identifier
 ///     identifier '=' identifier
 ///
-Parser::DeclPtrTy Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
+Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_synthesize) &&
          "ParseObjCPropertyDynamic(): Expected '@synthesize'");
   SourceLocation loc = ConsumeToken(); // consume synthesize
@@ -1374,7 +1403,7 @@ Parser::DeclPtrTy Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_synthesized_property_name);
       SkipUntil(tok::semi);
-      return DeclPtrTy();
+      return 0;
     }
     
     IdentifierInfo *propertyIvar = 0;
@@ -1409,7 +1438,7 @@ Parser::DeclPtrTy Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
   }
   else
     ConsumeToken(); // consume ';'
-  return DeclPtrTy();
+  return 0;
 }
 
 ///   property-dynamic:
@@ -1419,7 +1448,7 @@ Parser::DeclPtrTy Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) {
 ///     identifier
 ///     property-list ',' identifier
 ///
-Parser::DeclPtrTy Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
+Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_dynamic) &&
          "ParseObjCPropertyDynamic(): Expected '@dynamic'");
   SourceLocation loc = ConsumeToken(); // consume dynamic
@@ -1432,7 +1461,7 @@ Parser::DeclPtrTy Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
     if (Tok.isNot(tok::identifier)) {
       Diag(Tok, diag::err_expected_ident);
       SkipUntil(tok::semi);
-      return DeclPtrTy();
+      return 0;
     }
     
     IdentifierInfo *propertyId = Tok.getIdentifierInfo();
@@ -1450,14 +1479,14 @@ Parser::DeclPtrTy Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) {
   }
   else
     ConsumeToken(); // consume ';'
-  return DeclPtrTy();
+  return 0;
 }
 
 ///  objc-throw-statement:
 ///    throw expression[opt];
 ///
-Parser::OwningStmtResult Parser::ParseObjCThrowStmt(SourceLocation atLoc) {
-  OwningExprResult Res(Actions);
+StmtResult Parser::ParseObjCThrowStmt(SourceLocation atLoc) {
+  ExprResult Res;
   ConsumeToken(); // consume throw
   if (Tok.isNot(tok::semi)) {
     Res = ParseExpression();
@@ -1468,13 +1497,13 @@ Parser::OwningStmtResult Parser::ParseObjCThrowStmt(SourceLocation atLoc) {
   }
   // consume ';'
   ExpectAndConsume(tok::semi, diag::err_expected_semi_after, "@throw");
-  return Actions.ActOnObjCAtThrowStmt(atLoc, move(Res), getCurScope());
+  return Actions.ActOnObjCAtThrowStmt(atLoc, Res.take(), getCurScope());
 }
 
 /// objc-synchronized-statement:
 ///   @synchronized '(' expression ')' compound-statement
 ///
-Parser::OwningStmtResult
+StmtResult
 Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
   ConsumeToken(); // consume synchronized
   if (Tok.isNot(tok::l_paren)) {
@@ -1482,7 +1511,7 @@ Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
     return StmtError();
   }
   ConsumeParen();  // '('
-  OwningExprResult Res(ParseExpression());
+  ExprResult Res(ParseExpression());
   if (Res.isInvalid()) {
     SkipUntil(tok::semi);
     return StmtError();
@@ -1500,12 +1529,12 @@ Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
   // statements can always hold declarations.
   ParseScope BodyScope(this, Scope::DeclScope);
 
-  OwningStmtResult SynchBody(ParseCompoundStatementBody());
+  StmtResult SynchBody(ParseCompoundStatementBody());
 
   BodyScope.Exit();
   if (SynchBody.isInvalid())
     SynchBody = Actions.ActOnNullStmt(Tok.getLocation());
-  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, move(Res), move(SynchBody));
+  return Actions.ActOnObjCAtSynchronizedStmt(atLoc, Res.take(), SynchBody.take());
 }
 
 ///  objc-try-catch-statement:
@@ -1519,7 +1548,7 @@ Parser::ParseObjCSynchronizedStmt(SourceLocation atLoc) {
 ///     parameter-declaration
 ///     '...' [OBJC2]
 ///
-Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
+StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
   bool catch_or_finally_seen = false;
 
   ConsumeToken(); // consume try
@@ -1528,9 +1557,9 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
     return StmtError();
   }
   StmtVector CatchStmts(Actions);
-  OwningStmtResult FinallyStmt(Actions);
+  StmtResult FinallyStmt;
   ParseScope TryScope(this, Scope::DeclScope);
-  OwningStmtResult TryBody(ParseCompoundStatementBody());
+  StmtResult TryBody(ParseCompoundStatementBody());
   TryScope.Exit();
   if (TryBody.isInvalid())
     TryBody = Actions.ActOnNullStmt(Tok.getLocation());
@@ -1546,7 +1575,7 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
 
     SourceLocation AtCatchFinallyLoc = ConsumeToken();
     if (Tok.isObjCAtKeyword(tok::objc_catch)) {
-      DeclPtrTy FirstPart;
+      Decl *FirstPart = 0;
       ConsumeToken(); // consume catch
       if (Tok.is(tok::l_paren)) {
         ConsumeParen();
@@ -1573,7 +1602,7 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
         else // Skip over garbage, until we get to ')'.  Eat the ')'.
           SkipUntil(tok::r_paren, true, false);
 
-        OwningStmtResult CatchBody(Actions, true);
+        StmtResult CatchBody(true);
         if (Tok.is(tok::l_brace))
           CatchBody = ParseCompoundStatementBody();
         else
@@ -1581,10 +1610,10 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
         if (CatchBody.isInvalid())
           CatchBody = Actions.ActOnNullStmt(Tok.getLocation());
         
-        OwningStmtResult Catch = Actions.ActOnObjCAtCatchStmt(AtCatchFinallyLoc,
+        StmtResult Catch = Actions.ActOnObjCAtCatchStmt(AtCatchFinallyLoc,
                                                               RParenLoc, 
                                                               FirstPart, 
-                                                              move(CatchBody));
+                                                              CatchBody.take());
         if (!Catch.isInvalid())
           CatchStmts.push_back(Catch.release());
         
@@ -1599,7 +1628,7 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
       ConsumeToken(); // consume finally
       ParseScope FinallyScope(this, Scope::DeclScope);
 
-      OwningStmtResult FinallyBody(Actions, true);
+      StmtResult FinallyBody(true);
       if (Tok.is(tok::l_brace))
         FinallyBody = ParseCompoundStatementBody();
       else
@@ -1607,7 +1636,7 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
       if (FinallyBody.isInvalid())
         FinallyBody = Actions.ActOnNullStmt(Tok.getLocation());
       FinallyStmt = Actions.ActOnObjCAtFinallyStmt(AtCatchFinallyLoc,
-                                                   move(FinallyBody));
+                                                   FinallyBody.take());
       catch_or_finally_seen = true;
       break;
     }
@@ -1617,19 +1646,18 @@ Parser::OwningStmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
     return StmtError();
   }
   
-  return Actions.ActOnObjCAtTryStmt(atLoc, move(TryBody), 
+  return Actions.ActOnObjCAtTryStmt(atLoc, TryBody.take(), 
                                     move_arg(CatchStmts),
-                                    move(FinallyStmt));
+                                    FinallyStmt.take());
 }
 
 ///   objc-method-def: objc-method-proto ';'[opt] '{' body '}'
 ///
-Parser::DeclPtrTy Parser::ParseObjCMethodDefinition() {
-  DeclPtrTy MDecl = ParseObjCMethodPrototype(ObjCImpDecl);
+Decl *Parser::ParseObjCMethodDefinition() {
+  Decl *MDecl = ParseObjCMethodPrototype(ObjCImpDecl);
 
-  PrettyStackTraceActionsDecl CrashInfo(MDecl, Tok.getLocation(), Actions,
-                                        PP.getSourceManager(),
-                                        "parsing Objective-C method");
+  PrettyDeclStackTraceEntry CrashInfo(Actions, MDecl, Tok.getLocation(),
+                                      "parsing Objective-C method");
 
   // parse optional ';'
   if (Tok.is(tok::semi)) {
@@ -1649,7 +1677,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDefinition() {
 
     // If we didn't find the '{', bail out.
     if (Tok.isNot(tok::l_brace))
-      return DeclPtrTy();
+      return 0;
   }
   SourceLocation BraceLoc = Tok.getLocation();
 
@@ -1661,7 +1689,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDefinition() {
   // specified Declarator for the method.
   Actions.ActOnStartOfObjCMethodDef(getCurScope(), MDecl);
 
-  OwningStmtResult FnBody(ParseCompoundStatementBody());
+  StmtResult FnBody(ParseCompoundStatementBody());
 
   // If the function body could not be parsed, make a bogus compoundstmt.
   if (FnBody.isInvalid())
@@ -1669,7 +1697,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDefinition() {
                                        MultiStmtArg(Actions), false);
 
   // TODO: Pass argument information.
-  Actions.ActOnFinishFunctionBody(MDecl, move(FnBody));
+  Actions.ActOnFinishFunctionBody(MDecl, FnBody.take());
 
   // Leave the function body scope.
   BodyScope.Exit();
@@ -1677,7 +1705,7 @@ Parser::DeclPtrTy Parser::ParseObjCMethodDefinition() {
   return MDecl;
 }
 
-Parser::OwningStmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
+StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteObjCAtStatement(getCurScope());
     ConsumeCodeCompletionToken();
@@ -1693,7 +1721,7 @@ Parser::OwningStmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
   if (Tok.isObjCAtKeyword(tok::objc_synchronized))
     return ParseObjCSynchronizedStmt(AtLoc);
   
-  OwningExprResult Res(ParseExpressionWithLeadingAt(AtLoc));
+  ExprResult Res(ParseExpressionWithLeadingAt(AtLoc));
   if (Res.isInvalid()) {
     // If the expression is invalid, skip ahead to the next semicolon. Not
     // doing this opens us up to the possibility of infinite loops if
@@ -1704,10 +1732,10 @@ Parser::OwningStmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc) {
   
   // Otherwise, eat the semicolon.
   ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
-  return Actions.ActOnExprStmt(Actions.MakeFullExpr(Res));
+  return Actions.ActOnExprStmt(Actions.MakeFullExpr(Res.take()));
 }
 
-Parser::OwningExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
+ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
   switch (Tok.getKind()) {
   case tok::code_completion:
     Actions.CodeCompleteObjCAtExpression(getCurScope());
@@ -1764,7 +1792,7 @@ bool Parser::ParseObjCXXMessageReceiver(bool &IsExpr, void *&TypeOrExpr) {
   if (!isCXXSimpleTypeSpecifier()) {
     //   objc-receiver:
     //     expression
-    OwningExprResult Receiver = ParseExpression();
+    ExprResult Receiver = ParseExpression();
     if (Receiver.isInvalid())
       return true;
 
@@ -1793,11 +1821,11 @@ bool Parser::ParseObjCXXMessageReceiver(bool &IsExpr, void *&TypeOrExpr) {
     // postfix-expression suffix, followed by the (optional)
     // right-hand side of the binary expression. We have an
     // instance method.
-    OwningExprResult Receiver = ParseCXXTypeConstructExpression(DS);
+    ExprResult Receiver = ParseCXXTypeConstructExpression(DS);
     if (!Receiver.isInvalid())
-      Receiver = ParsePostfixExpressionSuffix(move(Receiver));
+      Receiver = ParsePostfixExpressionSuffix(Receiver.take());
     if (!Receiver.isInvalid())
-      Receiver = ParseRHSOfBinaryExpression(move(Receiver), prec::Comma);
+      Receiver = ParseRHSOfBinaryExpression(Receiver.take(), prec::Comma);
     if (Receiver.isInvalid())
       return true;
 
@@ -1815,7 +1843,7 @@ bool Parser::ParseObjCXXMessageReceiver(bool &IsExpr, void *&TypeOrExpr) {
     return true;
 
   IsExpr = false;
-  TypeOrExpr = Type.get();
+  TypeOrExpr = Type.get().getAsOpaquePtr();
   return false;
 }
 
@@ -1840,7 +1868,7 @@ bool Parser::isSimpleObjCMessageExpression() {
 ///     class-name
 ///     type-name
 ///
-Parser::OwningExprResult Parser::ParseObjCMessageExpression() {
+ExprResult Parser::ParseObjCMessageExpression() {
   assert(Tok.is(tok::l_square) && "'[' expected");
   SourceLocation LBracLoc = ConsumeBracket(); // consume '['
 
@@ -1860,8 +1888,8 @@ Parser::OwningExprResult Parser::ParseObjCMessageExpression() {
     // get in Objective-C.
     if (Tok.is(tok::identifier) && Tok.getIdentifierInfo() == Ident_super &&
         NextToken().isNot(tok::period) && getCurScope()->isInObjcMethodScope())
-      return ParseObjCMessageExpressionBody(LBracLoc, ConsumeToken(), 0, 
-                                            ExprArg(Actions));
+      return ParseObjCMessageExpressionBody(LBracLoc, ConsumeToken(),
+                                            ParsedType(), 0);
 
     // Parse the receiver, which is either a type or an expression.
     bool IsExpr;
@@ -1872,26 +1900,28 @@ Parser::OwningExprResult Parser::ParseObjCMessageExpression() {
     }
 
     if (IsExpr)
-      return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(), 0,
-                                         OwningExprResult(Actions, TypeOrExpr));
+      return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(),
+                                            ParsedType(),
+                                            static_cast<Expr*>(TypeOrExpr));
 
     return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(), 
-                                          TypeOrExpr, ExprArg(Actions));
+                              ParsedType::getFromOpaquePtr(TypeOrExpr),
+                                          0);
   }
   
   if (Tok.is(tok::identifier)) {
     IdentifierInfo *Name = Tok.getIdentifierInfo();
     SourceLocation NameLoc = Tok.getLocation();
-    TypeTy *ReceiverType;
+    ParsedType ReceiverType;
     switch (Actions.getObjCMessageKind(getCurScope(), Name, NameLoc,
                                        Name == Ident_super,
                                        NextToken().is(tok::period),
                                        ReceiverType)) {
-    case Action::ObjCSuperMessage:
-      return ParseObjCMessageExpressionBody(LBracLoc, ConsumeToken(), 0,
-                                            ExprArg(Actions));
+    case Sema::ObjCSuperMessage:
+      return ParseObjCMessageExpressionBody(LBracLoc, ConsumeToken(),
+                                            ParsedType(), 0);
 
-    case Action::ObjCClassMessage:
+    case Sema::ObjCClassMessage:
       if (!ReceiverType) {
         SkipUntil(tok::r_square);
         return ExprError();
@@ -1900,24 +1930,23 @@ Parser::OwningExprResult Parser::ParseObjCMessageExpression() {
       ConsumeToken(); // the type name
 
       return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(), 
-                                            ReceiverType,
-                                            ExprArg(Actions));
+                                            ReceiverType, 0);
         
-    case Action::ObjCInstanceMessage:
+    case Sema::ObjCInstanceMessage:
       // Fall through to parse an expression.
       break;
     }
   }
   
   // Otherwise, an arbitrary expression can be the receiver of a send.
-  OwningExprResult Res(ParseExpression());
+  ExprResult Res(ParseExpression());
   if (Res.isInvalid()) {
     SkipUntil(tok::r_square);
     return move(Res);
   }
 
-  return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(), 0, 
-                                        move(Res));
+  return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(),
+                                        ParsedType(), Res.take());
 }
 
 /// \brief Parse the remainder of an Objective-C message following the
@@ -1958,10 +1987,10 @@ Parser::OwningExprResult Parser::ParseObjCMessageExpression() {
 ///     assignment-expression
 ///     nonempty-expr-list , assignment-expression
 ///
-Parser::OwningExprResult
+ExprResult
 Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                        SourceLocation SuperLoc,
-                                       TypeTy *ReceiverType,
+                                       ParsedType ReceiverType,
                                        ExprArg ReceiverExpr) {
   if (Tok.is(tok::code_completion)) {
     if (SuperLoc.isValid())
@@ -1969,7 +1998,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
     else if (ReceiverType)
       Actions.CodeCompleteObjCClassMessage(getCurScope(), ReceiverType, 0, 0);
     else
-      Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr.get(), 
+      Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                               0, 0);
     ConsumeCodeCompletionToken();
   }
@@ -1999,7 +2028,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
 
       ConsumeToken(); // Eat the ':'.
       ///  Parse the expression after ':'
-      OwningExprResult Res(ParseAssignmentExpression());
+      ExprResult Res(ParseAssignmentExpression());
       if (Res.isInvalid()) {
         // We must manually skip to a ']', otherwise the expression skipper will
         // stop at the ']' when it skips to the ';'.  We want it to skip beyond
@@ -2022,7 +2051,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
                                                KeyIdents.data(), 
                                                KeyIdents.size());
         else
-          Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr.get(),
+          Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr,
                                                   KeyIdents.data(), 
                                                   KeyIdents.size());
         ConsumeCodeCompletionToken();
@@ -2038,7 +2067,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
     while (Tok.is(tok::comma)) {
       ConsumeToken(); // Eat the ','.
       ///  Parse the expression after ','
-      OwningExprResult Res(ParseAssignmentExpression());
+      ExprResult Res(ParseAssignmentExpression());
       if (Res.isInvalid()) {
         // We must manually skip to a ']', otherwise the expression skipper will
         // stop at the ']' when it skips to the ';'.  We want it to skip beyond
@@ -2082,24 +2111,24 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
   if (SuperLoc.isValid())
     return Actions.ActOnSuperMessage(getCurScope(), SuperLoc, Sel,
                                      LBracLoc, SelectorLoc, RBracLoc,
-                                     Action::MultiExprArg(Actions, 
-                                                          KeyExprs.take(),
-                                                          KeyExprs.size()));
+                                     MultiExprArg(Actions, 
+                                                  KeyExprs.take(),
+                                                  KeyExprs.size()));
   else if (ReceiverType)
     return Actions.ActOnClassMessage(getCurScope(), ReceiverType, Sel,
                                      LBracLoc, SelectorLoc, RBracLoc,
-                                     Action::MultiExprArg(Actions, 
-                                                          KeyExprs.take(), 
-                                                          KeyExprs.size()));
-  return Actions.ActOnInstanceMessage(getCurScope(), move(ReceiverExpr), Sel,
+                                     MultiExprArg(Actions, 
+                                                  KeyExprs.take(), 
+                                                  KeyExprs.size()));
+  return Actions.ActOnInstanceMessage(getCurScope(), ReceiverExpr, Sel,
                                       LBracLoc, SelectorLoc, RBracLoc,
-                                      Action::MultiExprArg(Actions, 
-                                                           KeyExprs.take(), 
-                                                           KeyExprs.size()));
+                                      MultiExprArg(Actions, 
+                                                   KeyExprs.take(), 
+                                                   KeyExprs.size()));
 }
 
-Parser::OwningExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
-  OwningExprResult Res(ParseStringLiteralExpression());
+ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
+  ExprResult Res(ParseStringLiteralExpression());
   if (Res.isInvalid()) return move(Res);
 
   // @"foo" @"bar" is a valid concatenated string.  Eat any subsequent string
@@ -2117,7 +2146,7 @@ Parser::OwningExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
     if (!isTokenStringLiteral())
       return ExprError(Diag(Tok, diag::err_objc_concat_string));
 
-    OwningExprResult Lit(ParseStringLiteralExpression());
+    ExprResult Lit(ParseStringLiteralExpression());
     if (Lit.isInvalid())
       return move(Lit);
 
@@ -2130,7 +2159,7 @@ Parser::OwningExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
 
 ///    objc-encode-expression:
 ///      @encode ( type-name )
-Parser::OwningExprResult
+ExprResult
 Parser::ParseObjCEncodeExpression(SourceLocation AtLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_encode) && "Not an @encode expression!");
 
@@ -2154,7 +2183,7 @@ Parser::ParseObjCEncodeExpression(SourceLocation AtLoc) {
 
 ///     objc-protocol-expression
 ///       @protocol ( protocol-name )
-Parser::OwningExprResult
+ExprResult
 Parser::ParseObjCProtocolExpression(SourceLocation AtLoc) {
   SourceLocation ProtoLoc = ConsumeToken();
 
@@ -2177,8 +2206,7 @@ Parser::ParseObjCProtocolExpression(SourceLocation AtLoc) {
 
 ///     objc-selector-expression
 ///       @selector '(' objc-keyword-selector ')'
-Parser::OwningExprResult
-Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
+ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
   SourceLocation SelectorLoc = ConsumeToken();
 
   if (Tok.isNot(tok::l_paren))
@@ -2187,21 +2215,43 @@ Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) {
   llvm::SmallVector<IdentifierInfo *, 12> KeyIdents;
   SourceLocation LParenLoc = ConsumeParen();
   SourceLocation sLoc;
+  
+  if (Tok.is(tok::code_completion)) {
+    Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents.data(),
+                                     KeyIdents.size());
+    ConsumeCodeCompletionToken();
+    MatchRHSPunctuation(tok::r_paren, LParenLoc);
+    return ExprError();
+  }
+  
   IdentifierInfo *SelIdent = ParseObjCSelectorPiece(sLoc);
-  if (!SelIdent && Tok.isNot(tok::colon)) // missing selector name.
+  if (!SelIdent &&  // missing selector name.
+      Tok.isNot(tok::colon) && Tok.isNot(tok::coloncolon))
     return ExprError(Diag(Tok, diag::err_expected_ident));
 
   KeyIdents.push_back(SelIdent);
   unsigned nColons = 0;
   if (Tok.isNot(tok::r_paren)) {
     while (1) {
-      if (Tok.isNot(tok::colon))
+      if (Tok.is(tok::coloncolon)) { // Handle :: in C++.
+        ++nColons;
+        KeyIdents.push_back(0);
+      } else if (Tok.isNot(tok::colon))
         return ExprError(Diag(Tok, diag::err_expected_colon));
 
-      nColons++;
+      ++nColons;
       ConsumeToken(); // Eat the ':'.
       if (Tok.is(tok::r_paren))
         break;
+      
+      if (Tok.is(tok::code_completion)) {
+        Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents.data(),
+                                         KeyIdents.size());
+        ConsumeCodeCompletionToken();
+        MatchRHSPunctuation(tok::r_paren, LParenLoc);
+        return ExprError();
+      }
+
       // Check for another keyword selector.
       SourceLocation Loc;
       SelIdent = ParseObjCSelectorPiece(Loc);
diff --git a/lib/Parse/ParsePragma.cpp b/lib/Parse/ParsePragma.cpp
index 64a4c16b77c3..ddba09ae0f95 100644
--- a/lib/Parse/ParsePragma.cpp
+++ b/lib/Parse/ParsePragma.cpp
@@ -13,11 +13,63 @@
 
 #include "ParsePragma.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Parse/Action.h"
 #include "clang/Parse/Parser.h"
+#include "clang/Lex/Preprocessor.h"
 using namespace clang;
 
+
+// #pragma GCC visibility comes in two variants:
+//   'push' '(' [visibility] ')'
+//   'pop'
+void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP, Token &VisTok) {
+  SourceLocation VisLoc = VisTok.getLocation();
+
+  Token Tok;
+  PP.Lex(Tok);
+
+  const IdentifierInfo *PushPop = Tok.getIdentifierInfo();
+
+  bool IsPush;
+  const IdentifierInfo *VisType;
+  if (PushPop && PushPop->isStr("pop")) {
+    IsPush = false;
+    VisType = 0;
+  } else if (PushPop && PushPop->isStr("push")) {
+    IsPush = true;
+    PP.Lex(Tok);
+    if (Tok.isNot(tok::l_paren)) {
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_lparen)
+        << "visibility";
+      return;
+    }
+    PP.Lex(Tok);
+    VisType = Tok.getIdentifierInfo();
+    if (!VisType) {
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier)
+        << "visibility";
+      return;
+    }
+    PP.Lex(Tok);
+    if (Tok.isNot(tok::r_paren)) {
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_rparen)
+        << "visibility";
+      return;
+    }
+  } else {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier)
+      << "visibility";
+    return;
+  }
+  PP.Lex(Tok);
+  if (Tok.isNot(tok::eom)) {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
+      << "visibility";
+    return;
+  }
+
+  Actions.ActOnPragmaVisibility(IsPush, VisType, VisLoc);
+}
+
 // #pragma pack(...) comes in the following delicious flavors:
 //   pack '(' [integer] ')'
 //   pack '(' 'show' ')'
@@ -32,9 +84,9 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP, Token &PackTok) {
     return;
   }
 
-  Action::PragmaPackKind Kind = Action::PPK_Default;
+  Sema::PragmaPackKind Kind = Sema::PPK_Default;
   IdentifierInfo *Name = 0;
-  Action::OwningExprResult Alignment(Actions);
+  ExprResult Alignment;
   SourceLocation LParenLoc = Tok.getLocation();
   PP.Lex(Tok);
   if (Tok.is(tok::numeric_constant)) {
@@ -46,13 +98,13 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP, Token &PackTok) {
   } else if (Tok.is(tok::identifier)) {
     const IdentifierInfo *II = Tok.getIdentifierInfo();
     if (II->isStr("show")) {
-      Kind = Action::PPK_Show;
+      Kind = Sema::PPK_Show;
       PP.Lex(Tok);
     } else {
       if (II->isStr("push")) {
-        Kind = Action::PPK_Push;
+        Kind = Sema::PPK_Push;
       } else if (II->isStr("pop")) {
-        Kind = Action::PPK_Pop;
+        Kind = Sema::PPK_Pop;
       } else {
         PP.Diag(Tok.getLocation(), diag::warn_pragma_pack_invalid_action);
         return;
@@ -110,46 +162,52 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP, Token &PackTok) {
                           LParenLoc, RParenLoc);
 }
 
-// #pragma 'options' 'align' '=' {'native','natural','mac68k','power','reset'}
-void PragmaOptionsHandler::HandlePragma(Preprocessor &PP, Token &OptionsTok) {
-  SourceLocation OptionsLoc = OptionsTok.getLocation();
-
+// #pragma 'align' '=' {'native','natural','mac68k','power','reset'}
+// #pragma 'options 'align' '=' {'native','natural','mac68k','power','reset'}
+static void ParseAlignPragma(Sema &Actions, Preprocessor &PP, Token &FirstTok,
+                             bool IsOptions) {
   Token Tok;
-  PP.Lex(Tok);
-  if (Tok.isNot(tok::identifier) || !Tok.getIdentifierInfo()->isStr("align")) {
-    PP.Diag(Tok.getLocation(), diag::warn_pragma_options_expected_align);
-    return;
+
+  if (IsOptions) {
+    PP.Lex(Tok);
+    if (Tok.isNot(tok::identifier) ||
+        !Tok.getIdentifierInfo()->isStr("align")) {
+      PP.Diag(Tok.getLocation(), diag::warn_pragma_options_expected_align);
+      return;
+    }
   }
 
   PP.Lex(Tok);
   if (Tok.isNot(tok::equal)) {
-    PP.Diag(Tok.getLocation(), diag::warn_pragma_options_expected_equal);
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_align_expected_equal)
+      << IsOptions;
     return;
   }
 
   PP.Lex(Tok);
   if (Tok.isNot(tok::identifier)) {
     PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier)
-      << "options";
+      << (IsOptions ? "options" : "align");
     return;
   }
 
-  Action::PragmaOptionsAlignKind Kind = Action::POAK_Natural;
+  Sema::PragmaOptionsAlignKind Kind = Sema::POAK_Natural;
   const IdentifierInfo *II = Tok.getIdentifierInfo();
   if (II->isStr("native"))
-    Kind = Action::POAK_Native;
+    Kind = Sema::POAK_Native;
   else if (II->isStr("natural"))
-    Kind = Action::POAK_Natural;
+    Kind = Sema::POAK_Natural;
   else if (II->isStr("packed"))
-    Kind = Action::POAK_Packed;
+    Kind = Sema::POAK_Packed;
   else if (II->isStr("power"))
-    Kind = Action::POAK_Power;
+    Kind = Sema::POAK_Power;
   else if (II->isStr("mac68k"))
-    Kind = Action::POAK_Mac68k;
+    Kind = Sema::POAK_Mac68k;
   else if (II->isStr("reset"))
-    Kind = Action::POAK_Reset;
+    Kind = Sema::POAK_Reset;
   else {
-    PP.Diag(Tok.getLocation(), diag::warn_pragma_options_invalid_option);
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_align_invalid_option)
+      << IsOptions;
     return;
   }
 
@@ -157,11 +215,19 @@ void PragmaOptionsHandler::HandlePragma(Preprocessor &PP, Token &OptionsTok) {
   PP.Lex(Tok);
   if (Tok.isNot(tok::eom)) {
     PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
-      << "options";
+      << (IsOptions ? "options" : "align");
     return;
   }
 
-  Actions.ActOnPragmaOptionsAlign(Kind, OptionsLoc, KindLoc);
+  Actions.ActOnPragmaOptionsAlign(Kind, FirstTok.getLocation(), KindLoc);
+}
+
+void PragmaAlignHandler::HandlePragma(Preprocessor &PP, Token &AlignTok) {
+  ParseAlignPragma(Actions, PP, AlignTok, /*IsOptions=*/false);
+}
+
+void PragmaOptionsHandler::HandlePragma(Preprocessor &PP, Token &OptionsTok) {
+  ParseAlignPragma(Actions, PP, OptionsTok, /*IsOptions=*/true);
 }
 
 // #pragma unused(identifier)
diff --git a/lib/Parse/ParsePragma.h b/lib/Parse/ParsePragma.h
index 929ec46c4ff5..0feaa9919e8d 100644
--- a/lib/Parse/ParsePragma.h
+++ b/lib/Parse/ParsePragma.h
@@ -17,41 +17,58 @@
 #include "clang/Lex/Pragma.h"
 
 namespace clang {
-  class Action;
+  class Sema;
   class Parser;
 
+class PragmaAlignHandler : public PragmaHandler {
+  Sema &Actions;
+public:
+  explicit PragmaAlignHandler(Sema &A) : PragmaHandler("align"), Actions(A) {}
+
+  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+};
+
+class PragmaGCCVisibilityHandler : public PragmaHandler {
+  Sema &Actions;
+public:
+  explicit PragmaGCCVisibilityHandler(Sema &A) : PragmaHandler("visibility"),
+                                                 Actions(A) {}
+
+  virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
+};
+
 class PragmaOptionsHandler : public PragmaHandler {
-  Action &Actions;
+  Sema &Actions;
 public:
-  explicit PragmaOptionsHandler(Action &A) : PragmaHandler("options"),
-                                             Actions(A) {}
+  explicit PragmaOptionsHandler(Sema &A) : PragmaHandler("options"),
+                                           Actions(A) {}
 
   virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
 };
 
 class PragmaPackHandler : public PragmaHandler {
-  Action &Actions;
+  Sema &Actions;
 public:
-  explicit PragmaPackHandler(Action &A) : PragmaHandler("pack"),
-                                          Actions(A) {}
+  explicit PragmaPackHandler(Sema &A) : PragmaHandler("pack"),
+                                        Actions(A) {}
 
   virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
 };
 
 class PragmaUnusedHandler : public PragmaHandler {
-  Action &Actions;
+  Sema &Actions;
   Parser &parser;
 public:
-  PragmaUnusedHandler(Action &A, Parser& p)
+  PragmaUnusedHandler(Sema &A, Parser& p)
     : PragmaHandler("unused"), Actions(A), parser(p) {}
 
   virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
 };
 
 class PragmaWeakHandler : public PragmaHandler {
-  Action &Actions;
+  Sema &Actions;
 public:
-  explicit PragmaWeakHandler(Action &A)
+  explicit PragmaWeakHandler(Sema &A)
     : PragmaHandler("weak"), Actions(A) {}
 
   virtual void HandlePragma(Preprocessor &PP, Token &FirstToken);
diff --git a/lib/Parse/ParseStmt.cpp b/lib/Parse/ParseStmt.cpp
index c908ed9caec6..6c240e608c2e 100644
--- a/lib/Parse/ParseStmt.cpp
+++ b/lib/Parse/ParseStmt.cpp
@@ -14,8 +14,9 @@
 
 #include "clang/Parse/Parser.h"
 #include "RAIIObjectsForParser.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/PrettyDeclStackTrace.h"
+#include "clang/Sema/Scope.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "clang/Basic/SourceManager.h"
@@ -73,10 +74,10 @@ using namespace clang;
 /// [OBC]   '@' 'throw' expression ';'
 /// [OBC]   '@' 'throw' ';'
 ///
-Parser::OwningStmtResult
+StmtResult
 Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
   const char *SemiError = 0;
-  OwningStmtResult Res(Actions);
+  StmtResult Res;
   
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
 
@@ -98,7 +99,7 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
     }
 
   case tok::code_completion:
-    Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_Statement);
+    Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement);
     ConsumeCodeCompletionToken();
     return ParseStatementOrDeclaration(OnlyStatement);
       
@@ -125,7 +126,7 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
 
     // FIXME: Use the attributes
     // expression[opt] ';'
-    OwningExprResult Expr(ParseExpression());
+    ExprResult Expr(ParseExpression());
     if (Expr.isInvalid()) {
       // If the expression is invalid, skip ahead to the next semicolon or '}'.
       // Not doing this opens us up to the possibility of infinite loops if
@@ -137,7 +138,7 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
     }
     // Otherwise, eat the semicolon.
     ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
-    return Actions.ActOnExprStmt(Actions.MakeFullExpr(Expr));
+    return Actions.ActOnExprStmt(Actions.MakeFullExpr(Expr.get()));
   }
 
   case tok::kw_case:                // C99 6.8.1: labeled-statement
@@ -217,7 +218,7 @@ Parser::ParseStatementOrDeclaration(bool OnlyStatement) {
 ///         identifier ':' statement
 /// [GNU]   identifier ':' attributes[opt] statement
 ///
-Parser::OwningStmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
+StmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
   assert(Tok.is(tok::identifier) && Tok.getIdentifierInfo() &&
          "Not an identifier!");
 
@@ -234,7 +235,7 @@ Parser::OwningStmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
   if (Tok.is(tok::kw___attribute))
     AttrList.reset(addAttributeLists(AttrList.take(), ParseGNUAttributes()));
 
-  OwningStmtResult SubStmt(ParseStatement());
+  StmtResult SubStmt(ParseStatement());
 
   // Broken substmt shouldn't prevent the label from being added to the AST.
   if (SubStmt.isInvalid())
@@ -243,7 +244,7 @@ Parser::OwningStmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
   // FIXME: use attributes?
   return Actions.ActOnLabelStmt(IdentTok.getLocation(),
                                 IdentTok.getIdentifierInfo(),
-                                ColonLoc, move(SubStmt));
+                                ColonLoc, SubStmt.get());
 }
 
 /// ParseCaseStatement
@@ -251,7 +252,7 @@ Parser::OwningStmtResult Parser::ParseLabeledStatement(AttributeList *Attr) {
 ///         'case' constant-expression ':' statement
 /// [GNU]   'case' constant-expression '...' constant-expression ':' statement
 ///
-Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
+StmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
   assert(Tok.is(tok::kw_case) && "Not a case stmt!");
   // FIXME: Use attributes?
   delete Attr;
@@ -272,7 +273,7 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
 
   // TopLevelCase - This is the highest level we have parsed.  'case 1' in the
   // example above.
-  OwningStmtResult TopLevelCase(Actions, true);
+  StmtResult TopLevelCase(true);
 
   // DeepestParsedCaseStmt - This is the deepest statement we have parsed, which
   // gets updated each time a new case is parsed, and whose body is unset so
@@ -293,7 +294,7 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
     /// expression.
     ColonProtectionRAIIObject ColonProtection(*this);
     
-    OwningExprResult LHS(ParseConstantExpression());
+    ExprResult LHS(ParseConstantExpression());
     if (LHS.isInvalid()) {
       SkipUntil(tok::colon);
       return StmtError();
@@ -301,7 +302,7 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
 
     // GNU case range extension.
     SourceLocation DotDotDotLoc;
-    OwningExprResult RHS(Actions);
+    ExprResult RHS;
     if (Tok.is(tok::ellipsis)) {
       Diag(Tok, diag::ext_gnu_case_range);
       DotDotDotLoc = ConsumeToken();
@@ -323,9 +324,9 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
 
     SourceLocation ColonLoc = ConsumeToken();
 
-    OwningStmtResult Case =
-      Actions.ActOnCaseStmt(CaseLoc, move(LHS), DotDotDotLoc,
-                            move(RHS), ColonLoc);
+    StmtResult Case =
+      Actions.ActOnCaseStmt(CaseLoc, LHS.get(), DotDotDotLoc,
+                            RHS.get(), ColonLoc);
 
     // If we had a sema error parsing this case, then just ignore it and
     // continue parsing the sub-stmt.
@@ -336,11 +337,11 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
     } else {
       // If this is the first case statement we parsed, it becomes TopLevelCase.
       // Otherwise we link it into the current chain.
-      StmtTy *NextDeepest = Case.get();
+      Stmt *NextDeepest = Case.get();
       if (TopLevelCase.isInvalid())
         TopLevelCase = move(Case);
       else
-        Actions.ActOnCaseStmtBody(DeepestParsedCaseStmt, move(Case));
+        Actions.ActOnCaseStmtBody(DeepestParsedCaseStmt, Case.get());
       DeepestParsedCaseStmt = NextDeepest;
     }
 
@@ -350,7 +351,7 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
   assert(!TopLevelCase.isInvalid() && "Should have parsed at least one case!");
 
   // If we found a non-case statement, start by parsing it.
-  OwningStmtResult SubStmt(Actions);
+  StmtResult SubStmt;
 
   if (Tok.isNot(tok::r_brace)) {
     SubStmt = ParseStatement();
@@ -367,7 +368,7 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
     SubStmt = Actions.ActOnNullStmt(SourceLocation());
 
   // Install the body into the most deeply-nested case.
-  Actions.ActOnCaseStmtBody(DeepestParsedCaseStmt, move(SubStmt));
+  Actions.ActOnCaseStmtBody(DeepestParsedCaseStmt, SubStmt.get());
 
   // Return the top level parsed statement tree.
   return move(TopLevelCase);
@@ -378,7 +379,7 @@ Parser::OwningStmtResult Parser::ParseCaseStatement(AttributeList *Attr) {
 ///         'default' ':' statement
 /// Note that this does not parse the 'statement' at the end.
 ///
-Parser::OwningStmtResult Parser::ParseDefaultStatement(AttributeList *Attr) {
+StmtResult Parser::ParseDefaultStatement(AttributeList *Attr) {
   //FIXME: Use attributes?
   delete Attr;
 
@@ -399,12 +400,12 @@ Parser::OwningStmtResult Parser::ParseDefaultStatement(AttributeList *Attr) {
     return StmtError();
   }
 
-  OwningStmtResult SubStmt(ParseStatement());
+  StmtResult SubStmt(ParseStatement());
   if (SubStmt.isInvalid())
     return StmtError();
 
   return Actions.ActOnDefaultStmt(DefaultLoc, ColonLoc,
-                                  move(SubStmt), getCurScope());
+                                  SubStmt.get(), getCurScope());
 }
 
 
@@ -435,7 +436,7 @@ Parser::OwningStmtResult Parser::ParseDefaultStatement(AttributeList *Attr) {
 /// [OMP]   barrier-directive
 /// [OMP]   flush-directive
 ///
-Parser::OwningStmtResult Parser::ParseCompoundStatement(AttributeList *Attr,
+StmtResult Parser::ParseCompoundStatement(AttributeList *Attr,
                                                         bool isStmtExpr) {
   //FIXME: Use attributes?
   delete Attr;
@@ -455,7 +456,7 @@ Parser::OwningStmtResult Parser::ParseCompoundStatement(AttributeList *Attr,
 /// ActOnCompoundStmt action.  This expects the '{' to be the current token, and
 /// consume the '}' at the end of the block.  It does not manipulate the scope
 /// stack.
-Parser::OwningStmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
+StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   PrettyStackTraceLoc CrashInfo(PP.getSourceManager(),
                                 Tok.getLocation(),
                                 "in compound statement ('{}')");
@@ -468,7 +469,7 @@ Parser::OwningStmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   typedef StmtVector StmtsTy;
   StmtsTy Stmts(Actions);
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
-    OwningStmtResult R(Actions);
+    StmtResult R;
     if (Tok.isNot(tok::kw___extension__)) {
       R = ParseStatementOrDeclaration(false);
     } else {
@@ -496,7 +497,7 @@ Parser::OwningStmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
         R = Actions.ActOnDeclStmt(Res, DeclStart, DeclEnd);
       } else {
         // Otherwise this was a unary __extension__ marker.
-        OwningExprResult Res(ParseExpressionWithLeadingExtension(ExtLoc));
+        ExprResult Res(ParseExpressionWithLeadingExtension(ExtLoc));
 
         if (Res.isInvalid()) {
           SkipUntil(tok::semi);
@@ -507,7 +508,7 @@ Parser::OwningStmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
         // Eat the semicolon at the end of stmt and convert the expr into a
         // statement.
         ExpectAndConsume(tok::semi, diag::err_expected_semi_after_expr);
-        R = Actions.ActOnExprStmt(Actions.MakeFullExpr(Res));
+        R = Actions.ActOnExprStmt(Actions.MakeFullExpr(Res.get()));
       }
     }
 
@@ -518,6 +519,7 @@ Parser::OwningStmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   // We broke out of the while loop because we found a '}' or EOF.
   if (Tok.isNot(tok::r_brace)) {
     Diag(Tok, diag::err_expected_rbrace);
+    Diag(LBraceLoc, diag::note_matching) << "{";
     return StmtError();
   }
 
@@ -537,8 +539,8 @@ Parser::OwningStmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
 /// should try to recover harder.  It returns false if the condition is
 /// successfully parsed.  Note that a successful parse can still have semantic
 /// errors in the condition.
-bool Parser::ParseParenExprOrCondition(OwningExprResult &ExprResult,
-                                       DeclPtrTy &DeclResult,
+bool Parser::ParseParenExprOrCondition(ExprResult &ExprResult,
+                                       Decl *&DeclResult,
                                        SourceLocation Loc,
                                        bool ConvertToBoolean) {
   bool ParseError = false;
@@ -549,18 +551,18 @@ bool Parser::ParseParenExprOrCondition(OwningExprResult &ExprResult,
                                    ConvertToBoolean);
   else {
     ExprResult = ParseExpression();
-    DeclResult = DeclPtrTy();
+    DeclResult = 0;
     
     // If required, convert to a boolean value.
     if (!ExprResult.isInvalid() && ConvertToBoolean)
       ExprResult
-        = Actions.ActOnBooleanCondition(getCurScope(), Loc, move(ExprResult));
+        = Actions.ActOnBooleanCondition(getCurScope(), Loc, ExprResult.get());
   }
 
   // If the parser was confused by the condition and we don't have a ')', try to
   // recover by skipping ahead to a semi and bailing out.  If condexp is
   // semantically invalid but we have well formed code, keep going.
-  if (ExprResult.isInvalid() && !DeclResult.get() && Tok.isNot(tok::r_paren)) {
+  if (ExprResult.isInvalid() && !DeclResult && Tok.isNot(tok::r_paren)) {
     SkipUntil(tok::semi);
     // Skipping may have stopped if it found the containing ')'.  If so, we can
     // continue parsing the if statement.
@@ -581,7 +583,7 @@ bool Parser::ParseParenExprOrCondition(OwningExprResult &ExprResult,
 /// [C++]   'if' '(' condition ')' statement
 /// [C++]   'if' '(' condition ')' statement 'else' statement
 ///
-Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
+StmtResult Parser::ParseIfStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -611,12 +613,12 @@ Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
   ParseScope IfScope(this, Scope::DeclScope | Scope::ControlScope, C99orCXX);
 
   // Parse the condition.
-  OwningExprResult CondExp(Actions);
-  DeclPtrTy CondVar;
+  ExprResult CondExp;
+  Decl *CondVar = 0;
   if (ParseParenExprOrCondition(CondExp, CondVar, IfLoc, true))
     return StmtError();
 
-  FullExprArg FullCondExp(Actions.MakeFullExpr(CondExp));
+  FullExprArg FullCondExp(Actions.MakeFullExpr(CondExp.get()));
 
   // C99 6.8.4p3 - In C99, the body of the if statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
@@ -641,7 +643,7 @@ Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
 
   // Read the 'then' stmt.
   SourceLocation ThenStmtLoc = Tok.getLocation();
-  OwningStmtResult ThenStmt(ParseStatement());
+  StmtResult ThenStmt(ParseStatement());
 
   // Pop the 'if' scope if needed.
   InnerScope.Exit();
@@ -649,7 +651,7 @@ Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
   // If it has an else, parse it.
   SourceLocation ElseLoc;
   SourceLocation ElseStmtLoc;
-  OwningStmtResult ElseStmt(Actions);
+  StmtResult ElseStmt;
 
   if (Tok.is(tok::kw_else)) {
     ElseLoc = ConsumeToken();
@@ -667,13 +669,7 @@ Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
     ParseScope InnerScope(this, Scope::DeclScope,
                           C99orCXX && Tok.isNot(tok::l_brace));
 
-    // Regardless of whether or not InnerScope actually pushed a scope, set the
-    // ElseScope flag for the innermost scope so we can diagnose use of the if
-    // condition variable in C++.
-    unsigned OldFlags = getCurScope()->getFlags();
-    getCurScope()->setFlags(OldFlags | Scope::ElseScope);
     ElseStmt = ParseStatement();
-    getCurScope()->setFlags(OldFlags);
     
     // Pop the 'else' scope if needed.
     InnerScope.Exit();
@@ -683,7 +679,7 @@ Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
 
   // If the condition was invalid, discard the if statement.  We could recover
   // better by replacing it with a valid expr, but don't do that yet.
-  if (CondExp.isInvalid() && !CondVar.get())
+  if (CondExp.isInvalid() && !CondVar)
     return StmtError();
 
   // If the then or else stmt is invalid and the other is valid (and present),
@@ -702,15 +698,15 @@ Parser::OwningStmtResult Parser::ParseIfStatement(AttributeList *Attr) {
   if (ElseStmt.isInvalid())
     ElseStmt = Actions.ActOnNullStmt(ElseStmtLoc);
 
-  return Actions.ActOnIfStmt(IfLoc, FullCondExp, CondVar, move(ThenStmt),
-                             ElseLoc, move(ElseStmt));
+  return Actions.ActOnIfStmt(IfLoc, FullCondExp, CondVar, ThenStmt.get(),
+                             ElseLoc, ElseStmt.get());
 }
 
 /// ParseSwitchStatement
 ///       switch-statement:
 ///         'switch' '(' expression ')' statement
 /// [C++]   'switch' '(' condition ')' statement
-Parser::OwningStmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
+StmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -743,13 +739,13 @@ Parser::OwningStmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
   ParseScope SwitchScope(this, ScopeFlags);
 
   // Parse the condition.
-  OwningExprResult Cond(Actions);
-  DeclPtrTy CondVar;
+  ExprResult Cond;
+  Decl *CondVar = 0;
   if (ParseParenExprOrCondition(Cond, CondVar, SwitchLoc, false))
     return StmtError();
 
-  OwningStmtResult Switch
-    = Actions.ActOnStartOfSwitchStmt(SwitchLoc, move(Cond), CondVar);
+  StmtResult Switch
+    = Actions.ActOnStartOfSwitchStmt(SwitchLoc, Cond.get(), CondVar);
 
   if (Switch.isInvalid()) {
     // Skip the switch body. 
@@ -779,7 +775,7 @@ Parser::OwningStmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
                         C99orCXX && Tok.isNot(tok::l_brace));
 
   // Read the body statement.
-  OwningStmtResult Body(ParseStatement());
+  StmtResult Body(ParseStatement());
 
   // Pop the scopes.
   InnerScope.Exit();
@@ -789,14 +785,14 @@ Parser::OwningStmtResult Parser::ParseSwitchStatement(AttributeList *Attr) {
     // FIXME: Remove the case statement list from the Switch statement.
     Body = Actions.ActOnNullStmt(Tok.getLocation());
   
-  return Actions.ActOnFinishSwitchStmt(SwitchLoc, move(Switch), move(Body));
+  return Actions.ActOnFinishSwitchStmt(SwitchLoc, Switch.get(), Body.get());
 }
 
 /// ParseWhileStatement
 ///       while-statement: [C99 6.8.5.1]
 ///         'while' '(' expression ')' statement
 /// [C++]   'while' '(' condition ')' statement
-Parser::OwningStmtResult Parser::ParseWhileStatement(AttributeList *Attr) {
+StmtResult Parser::ParseWhileStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -833,12 +829,12 @@ Parser::OwningStmtResult Parser::ParseWhileStatement(AttributeList *Attr) {
   ParseScope WhileScope(this, ScopeFlags);
 
   // Parse the condition.
-  OwningExprResult Cond(Actions);
-  DeclPtrTy CondVar;
+  ExprResult Cond;
+  Decl *CondVar = 0;
   if (ParseParenExprOrCondition(Cond, CondVar, WhileLoc, true))
     return StmtError();
 
-  FullExprArg FullCond(Actions.MakeFullExpr(Cond));
+  FullExprArg FullCond(Actions.MakeFullExpr(Cond.get()));
 
   // C99 6.8.5p5 - In C99, the body of the if statement is a scope, even if
   // there is no compound stmt.  C90 does not have this clause.  We only do this
@@ -855,23 +851,23 @@ Parser::OwningStmtResult Parser::ParseWhileStatement(AttributeList *Attr) {
                         C99orCXX && Tok.isNot(tok::l_brace));
 
   // Read the body statement.
-  OwningStmtResult Body(ParseStatement());
+  StmtResult Body(ParseStatement());
 
   // Pop the body scope if needed.
   InnerScope.Exit();
   WhileScope.Exit();
 
-  if ((Cond.isInvalid() && !CondVar.get()) || Body.isInvalid())
+  if ((Cond.isInvalid() && !CondVar) || Body.isInvalid())
     return StmtError();
 
-  return Actions.ActOnWhileStmt(WhileLoc, FullCond, CondVar, move(Body));
+  return Actions.ActOnWhileStmt(WhileLoc, FullCond, CondVar, Body.get());
 }
 
 /// ParseDoStatement
 ///       do-statement: [C99 6.8.5.2]
 ///         'do' statement 'while' '(' expression ')' ';'
 /// Note: this lets the caller parse the end ';'.
-Parser::OwningStmtResult Parser::ParseDoStatement(AttributeList *Attr) {
+StmtResult Parser::ParseDoStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -901,7 +897,7 @@ Parser::OwningStmtResult Parser::ParseDoStatement(AttributeList *Attr) {
                         Tok.isNot(tok::l_brace));
 
   // Read the body statement.
-  OwningStmtResult Body(ParseStatement());
+  StmtResult Body(ParseStatement());
 
   // Pop the body scope if needed.
   InnerScope.Exit();
@@ -924,15 +920,15 @@ Parser::OwningStmtResult Parser::ParseDoStatement(AttributeList *Attr) {
 
   // Parse the parenthesized condition.
   SourceLocation LPLoc = ConsumeParen();
-  OwningExprResult Cond = ParseExpression();
+  ExprResult Cond = ParseExpression();
   SourceLocation RPLoc = MatchRHSPunctuation(tok::r_paren, LPLoc);
   DoScope.Exit();
 
   if (Cond.isInvalid() || Body.isInvalid())
     return StmtError();
 
-  return Actions.ActOnDoStmt(DoLoc, move(Body), WhileLoc, LPLoc,
-                             move(Cond), RPLoc);
+  return Actions.ActOnDoStmt(DoLoc, Body.get(), WhileLoc, LPLoc,
+                             Cond.get(), RPLoc);
 }
 
 /// ParseForStatement
@@ -948,7 +944,7 @@ Parser::OwningStmtResult Parser::ParseDoStatement(AttributeList *Attr) {
 /// [C++]   expression-statement
 /// [C++]   simple-declaration
 ///
-Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
+StmtResult Parser::ParseForStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -988,20 +984,20 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
   ParseScope ForScope(this, ScopeFlags);
 
   SourceLocation LParenLoc = ConsumeParen();
-  OwningExprResult Value(Actions);
+  ExprResult Value;
 
   bool ForEach = false;
-  OwningStmtResult FirstPart(Actions);
+  StmtResult FirstPart;
   bool SecondPartIsInvalid = false;
   FullExprArg SecondPart(Actions);
-  OwningExprResult Collection(Actions);
+  ExprResult Collection;
   FullExprArg ThirdPart(Actions);
-  DeclPtrTy SecondVar;
+  Decl *SecondVar = 0;
   
   if (Tok.is(tok::code_completion)) {
     Actions.CodeCompleteOrdinaryName(getCurScope(), 
-                                     C99orCXXorObjC? Action::CCC_ForInit
-                                                   : Action::CCC_Expression);
+                                     C99orCXXorObjC? Sema::PCC_ForInit
+                                                   : Sema::PCC_Expression);
     ConsumeCodeCompletionToken();
   }
   
@@ -1029,6 +1025,11 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
       Actions.ActOnForEachDeclStmt(DG);
       // ObjC: for (id x in expr)
       ConsumeToken(); // consume 'in'
+      
+      if (Tok.is(tok::code_completion)) {
+        Actions.CodeCompleteObjCForCollection(getCurScope(), DG);
+        ConsumeCodeCompletionToken();
+      }
       Collection = ParseExpression();
     } else {
       Diag(Tok, diag::err_expected_semi_for);
@@ -1039,12 +1040,17 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
 
     // Turn the expression into a stmt.
     if (!Value.isInvalid())
-      FirstPart = Actions.ActOnExprStmt(Actions.MakeFullExpr(Value));
+      FirstPart = Actions.ActOnExprStmt(Actions.MakeFullExpr(Value.get()));
 
     if (Tok.is(tok::semi)) {
       ConsumeToken();
     } else if ((ForEach = isTokIdentifier_in())) {
       ConsumeToken(); // consume 'in'
+      
+      if (Tok.is(tok::code_completion)) {
+        Actions.CodeCompleteObjCForCollection(getCurScope(), DeclGroupPtrTy());
+        ConsumeCodeCompletionToken();
+      }
       Collection = ParseExpression();
     } else {
       if (!Value.isInvalid()) Diag(Tok, diag::err_expected_semi_for);
@@ -1052,36 +1058,36 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
     }
   }
   if (!ForEach) {
-    assert(!SecondPart->get() && "Shouldn't have a second expression yet.");
+    assert(!SecondPart.get() && "Shouldn't have a second expression yet.");
     // Parse the second part of the for specifier.
     if (Tok.is(tok::semi)) {  // for (...;;
       // no second part.
     } else {
-      OwningExprResult Second(Actions);
+      ExprResult Second;
       if (getLang().CPlusPlus)
         ParseCXXCondition(Second, SecondVar, ForLoc, true);
       else {
         Second = ParseExpression();
         if (!Second.isInvalid())
           Second = Actions.ActOnBooleanCondition(getCurScope(), ForLoc, 
-                                                 move(Second));
+                                                 Second.get());
       }
       SecondPartIsInvalid = Second.isInvalid();
-      SecondPart = Actions.MakeFullExpr(Second);
+      SecondPart = Actions.MakeFullExpr(Second.get());
     }
 
     if (Tok.is(tok::semi)) {
       ConsumeToken();
     } else {
-      if (!SecondPartIsInvalid || SecondVar.get()) 
+      if (!SecondPartIsInvalid || SecondVar) 
         Diag(Tok, diag::err_expected_semi_for);
       SkipUntil(tok::semi);
     }
 
     // Parse the third part of the for specifier.
     if (Tok.isNot(tok::r_paren)) {   // for (...;...;)
-      OwningExprResult Third = ParseExpression();
-      ThirdPart = Actions.MakeFullExpr(Third);
+      ExprResult Third = ParseExpression();
+      ThirdPart = Actions.MakeFullExpr(Third.take());
     }
   }
   // Match the ')'.
@@ -1102,7 +1108,7 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
                         C99orCXXorObjC && Tok.isNot(tok::l_brace));
 
   // Read the body statement.
-  OwningStmtResult Body(ParseStatement());
+  StmtResult Body(ParseStatement());
 
   // Pop the body scope if needed.
   InnerScope.Exit();
@@ -1114,14 +1120,14 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
     return StmtError();
 
   if (!ForEach)
-    return Actions.ActOnForStmt(ForLoc, LParenLoc, move(FirstPart), SecondPart,
-                                SecondVar, ThirdPart, RParenLoc, move(Body));
+    return Actions.ActOnForStmt(ForLoc, LParenLoc, FirstPart.take(), SecondPart,
+                                SecondVar, ThirdPart, RParenLoc, Body.take());
 
   // FIXME: It isn't clear how to communicate the late destruction of 
   // C++ temporaries used to create the collection.
-  return Actions.ActOnObjCForCollectionStmt(ForLoc, LParenLoc, move(FirstPart), 
-                                            move(Collection), RParenLoc, 
-                                            move(Body));
+  return Actions.ActOnObjCForCollectionStmt(ForLoc, LParenLoc, FirstPart.take(), 
+                                            Collection.take(), RParenLoc, 
+                                            Body.take());
 }
 
 /// ParseGotoStatement
@@ -1131,14 +1137,14 @@ Parser::OwningStmtResult Parser::ParseForStatement(AttributeList *Attr) {
 ///
 /// Note: this lets the caller parse the end ';'.
 ///
-Parser::OwningStmtResult Parser::ParseGotoStatement(AttributeList *Attr) {
+StmtResult Parser::ParseGotoStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
   assert(Tok.is(tok::kw_goto) && "Not a goto stmt!");
   SourceLocation GotoLoc = ConsumeToken();  // eat the 'goto'.
 
-  OwningStmtResult Res(Actions);
+  StmtResult Res;
   if (Tok.is(tok::identifier)) {
     Res = Actions.ActOnGotoStmt(GotoLoc, Tok.getLocation(),
                                 Tok.getIdentifierInfo());
@@ -1147,12 +1153,12 @@ Parser::OwningStmtResult Parser::ParseGotoStatement(AttributeList *Attr) {
     // GNU indirect goto extension.
     Diag(Tok, diag::ext_gnu_indirect_goto);
     SourceLocation StarLoc = ConsumeToken();
-    OwningExprResult R(ParseExpression());
+    ExprResult R(ParseExpression());
     if (R.isInvalid()) {  // Skip to the semicolon, but don't consume it.
       SkipUntil(tok::semi, false, true);
       return StmtError();
     }
-    Res = Actions.ActOnIndirectGotoStmt(GotoLoc, StarLoc, move(R));
+    Res = Actions.ActOnIndirectGotoStmt(GotoLoc, StarLoc, R.take());
   } else {
     Diag(Tok, diag::err_expected_ident);
     return StmtError();
@@ -1167,7 +1173,7 @@ Parser::OwningStmtResult Parser::ParseGotoStatement(AttributeList *Attr) {
 ///
 /// Note: this lets the caller parse the end ';'.
 ///
-Parser::OwningStmtResult Parser::ParseContinueStatement(AttributeList *Attr) {
+StmtResult Parser::ParseContinueStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -1181,7 +1187,7 @@ Parser::OwningStmtResult Parser::ParseContinueStatement(AttributeList *Attr) {
 ///
 /// Note: this lets the caller parse the end ';'.
 ///
-Parser::OwningStmtResult Parser::ParseBreakStatement(AttributeList *Attr) {
+StmtResult Parser::ParseBreakStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
@@ -1192,14 +1198,14 @@ Parser::OwningStmtResult Parser::ParseBreakStatement(AttributeList *Attr) {
 /// ParseReturnStatement
 ///       jump-statement:
 ///         'return' expression[opt] ';'
-Parser::OwningStmtResult Parser::ParseReturnStatement(AttributeList *Attr) {
+StmtResult Parser::ParseReturnStatement(AttributeList *Attr) {
   // FIXME: Use attributes?
   delete Attr;
 
   assert(Tok.is(tok::kw_return) && "Not a return stmt!");
   SourceLocation ReturnLoc = ConsumeToken();  // eat the 'return'.
 
-  OwningExprResult R(Actions);
+  ExprResult R;
   if (Tok.isNot(tok::semi)) {
     if (Tok.is(tok::code_completion)) {
       Actions.CodeCompleteReturn(getCurScope());
@@ -1214,12 +1220,12 @@ Parser::OwningStmtResult Parser::ParseReturnStatement(AttributeList *Attr) {
       return StmtError();
     }
   }
-  return Actions.ActOnReturnStmt(ReturnLoc, move(R));
+  return Actions.ActOnReturnStmt(ReturnLoc, R.take());
 }
 
 /// FuzzyParseMicrosoftAsmStatement. When -fms-extensions is enabled, this
 /// routine is called to skip/ignore tokens that comprise the MS asm statement.
-Parser::OwningStmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
+StmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
   if (Tok.is(tok::l_brace)) {
     unsigned short savedBraceCount = BraceCount;
     do {
@@ -1240,16 +1246,16 @@ Parser::OwningStmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
   }
   Token t;
   t.setKind(tok::string_literal);
-  t.setLiteralData("\"FIXME: not done\"");
+  t.setLiteralData("\"/*FIXME: not done*/\"");
   t.clearFlag(Token::NeedsCleaning);
-  t.setLength(17);
-  OwningExprResult AsmString(Actions.ActOnStringLiteral(&t, 1));
+  t.setLength(21);
+  ExprResult AsmString(Actions.ActOnStringLiteral(&t, 1));
   ExprVector Constraints(Actions);
   ExprVector Exprs(Actions);
   ExprVector Clobbers(Actions);
   return Actions.ActOnAsmStmt(Tok.getLocation(), true, true, 0, 0, 0,
                               move_arg(Constraints), move_arg(Exprs),
-                              move(AsmString), move_arg(Clobbers),
+                              AsmString.take(), move_arg(Clobbers),
                               Tok.getLocation(), true);
 }
 
@@ -1280,7 +1286,7 @@ Parser::OwningStmtResult Parser::FuzzyParseMicrosoftAsmStatement() {
 ///         assembly-instruction ';'[opt]
 ///         assembly-instruction-list ';' assembly-instruction ';'[opt]
 ///
-Parser::OwningStmtResult Parser::ParseAsmStatement(bool &msAsm) {
+StmtResult Parser::ParseAsmStatement(bool &msAsm) {
   assert(Tok.is(tok::kw_asm) && "Not an asm stmt");
   SourceLocation AsmLoc = ConsumeToken();
 
@@ -1307,7 +1313,7 @@ Parser::OwningStmtResult Parser::ParseAsmStatement(bool &msAsm) {
   }
   Loc = ConsumeParen();
 
-  OwningExprResult AsmString(ParseAsmStringLiteral());
+  ExprResult AsmString(ParseAsmStringLiteral());
   if (AsmString.isInvalid())
     return StmtError();
 
@@ -1322,7 +1328,7 @@ Parser::OwningStmtResult Parser::ParseAsmStatement(bool &msAsm) {
     return Actions.ActOnAsmStmt(AsmLoc, /*isSimple*/ true, isVolatile,
                                 /*NumOutputs*/ 0, /*NumInputs*/ 0, 0, 
                                 move_arg(Constraints), move_arg(Exprs),
-                                move(AsmString), move_arg(Clobbers),
+                                AsmString.take(), move_arg(Clobbers),
                                 RParenLoc);
   }
 
@@ -1367,17 +1373,19 @@ Parser::OwningStmtResult Parser::ParseAsmStatement(bool &msAsm) {
     if (!AteExtraColon)
       ConsumeToken();
 
-    // Parse the asm-string list for clobbers.
-    while (1) {
-      OwningExprResult Clobber(ParseAsmStringLiteral());
+    // Parse the asm-string list for clobbers if present.
+    if (Tok.isNot(tok::r_paren)) {
+      while (1) {
+        ExprResult Clobber(ParseAsmStringLiteral());
 
-      if (Clobber.isInvalid())
-        break;
+        if (Clobber.isInvalid())
+          break;
 
-      Clobbers.push_back(Clobber.release());
+        Clobbers.push_back(Clobber.release());
 
-      if (Tok.isNot(tok::comma)) break;
-      ConsumeToken();
+        if (Tok.isNot(tok::comma)) break;
+        ConsumeToken();
+      }
     }
   }
 
@@ -1385,7 +1393,7 @@ Parser::OwningStmtResult Parser::ParseAsmStatement(bool &msAsm) {
   return Actions.ActOnAsmStmt(AsmLoc, false, isVolatile,
                               NumOutputs, NumInputs, Names.data(),
                               move_arg(Constraints), move_arg(Exprs),
-                              move(AsmString), move_arg(Clobbers),
+                              AsmString.take(), move_arg(Clobbers),
                               RParenLoc);
 }
 
@@ -1428,7 +1436,7 @@ bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
     } else
       Names.push_back(0);
 
-    OwningExprResult Constraint(ParseAsmStringLiteral());
+    ExprResult Constraint(ParseAsmStringLiteral());
     if (Constraint.isInvalid()) {
         SkipUntil(tok::r_paren);
         return true;
@@ -1443,7 +1451,7 @@ bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
 
     // Read the parenthesized expression.
     SourceLocation OpenLoc = ConsumeParen();
-    OwningExprResult Res(ParseExpression());
+    ExprResult Res(ParseExpression());
     MatchRHSPunctuation(tok::r_paren, OpenLoc);
     if (Res.isInvalid()) {
       SkipUntil(tok::r_paren);
@@ -1458,25 +1466,24 @@ bool Parser::ParseAsmOperandsOpt(llvm::SmallVectorImpl<IdentifierInfo *> &Names,
   return true;
 }
 
-Parser::DeclPtrTy Parser::ParseFunctionStatementBody(DeclPtrTy Decl) {
+Decl *Parser::ParseFunctionStatementBody(Decl *Decl) {
   assert(Tok.is(tok::l_brace));
   SourceLocation LBraceLoc = Tok.getLocation();
 
-  PrettyStackTraceActionsDecl CrashInfo(Decl, LBraceLoc, Actions,
-                                        PP.getSourceManager(),
-                                        "parsing function body");
+  PrettyDeclStackTraceEntry CrashInfo(Actions, Decl, LBraceLoc,
+                                      "parsing function body");
 
   // Do not enter a scope for the brace, as the arguments are in the same scope
   // (the function body) as the body itself.  Instead, just read the statement
   // list and put it into a CompoundStmt for safe keeping.
-  OwningStmtResult FnBody(ParseCompoundStatementBody());
+  StmtResult FnBody(ParseCompoundStatementBody());
 
   // If the function body could not be parsed, make a bogus compoundstmt.
   if (FnBody.isInvalid())
     FnBody = Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc,
                                        MultiStmtArg(Actions), false);
 
-  return Actions.ActOnFinishFunctionBody(Decl, move(FnBody));
+  return Actions.ActOnFinishFunctionBody(Decl, FnBody.take());
 }
 
 /// ParseFunctionTryBlock - Parse a C++ function-try-block.
@@ -1484,27 +1491,26 @@ Parser::DeclPtrTy Parser::ParseFunctionStatementBody(DeclPtrTy Decl) {
 ///       function-try-block:
 ///         'try' ctor-initializer[opt] compound-statement handler-seq
 ///
-Parser::DeclPtrTy Parser::ParseFunctionTryBlock(DeclPtrTy Decl) {
+Decl *Parser::ParseFunctionTryBlock(Decl *Decl) {
   assert(Tok.is(tok::kw_try) && "Expected 'try'");
   SourceLocation TryLoc = ConsumeToken();
 
-  PrettyStackTraceActionsDecl CrashInfo(Decl, TryLoc, Actions,
-                                        PP.getSourceManager(),
-                                        "parsing function try block");
+  PrettyDeclStackTraceEntry CrashInfo(Actions, Decl, TryLoc,
+                                      "parsing function try block");
 
   // Constructor initializer list?
   if (Tok.is(tok::colon))
     ParseConstructorInitializer(Decl);
 
   SourceLocation LBraceLoc = Tok.getLocation();
-  OwningStmtResult FnBody(ParseCXXTryBlockCommon(TryLoc));
+  StmtResult FnBody(ParseCXXTryBlockCommon(TryLoc));
   // If we failed to parse the try-catch, we just give the function an empty
   // compound statement as the body.
   if (FnBody.isInvalid())
     FnBody = Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc,
                                        MultiStmtArg(Actions), false);
 
-  return Actions.ActOnFinishFunctionBody(Decl, move(FnBody));
+  return Actions.ActOnFinishFunctionBody(Decl, FnBody.take());
 }
 
 /// ParseCXXTryBlock - Parse a C++ try-block.
@@ -1512,7 +1518,7 @@ Parser::DeclPtrTy Parser::ParseFunctionTryBlock(DeclPtrTy Decl) {
 ///       try-block:
 ///         'try' compound-statement handler-seq
 ///
-Parser::OwningStmtResult Parser::ParseCXXTryBlock(AttributeList* Attr) {
+StmtResult Parser::ParseCXXTryBlock(AttributeList* Attr) {
   // FIXME: Add attributes?
   delete Attr;
 
@@ -1534,11 +1540,11 @@ Parser::OwningStmtResult Parser::ParseCXXTryBlock(AttributeList* Attr) {
 ///       handler-seq:
 ///         handler handler-seq[opt]
 ///
-Parser::OwningStmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
+StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
   if (Tok.isNot(tok::l_brace))
     return StmtError(Diag(Tok, diag::err_expected_lbrace));
   // FIXME: Possible draft standard bug: attribute-specifier should be allowed?
-  OwningStmtResult TryBlock(ParseCompoundStatement(0));
+  StmtResult TryBlock(ParseCompoundStatement(0));
   if (TryBlock.isInvalid())
     return move(TryBlock);
 
@@ -1551,7 +1557,7 @@ Parser::OwningStmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
   if (Tok.isNot(tok::kw_catch))
     return StmtError(Diag(Tok, diag::err_expected_catch));
   while (Tok.is(tok::kw_catch)) {
-    OwningStmtResult Handler(ParseCXXCatchBlock());
+    StmtResult Handler(ParseCXXCatchBlock());
     if (!Handler.isInvalid())
       Handlers.push_back(Handler.release());
   }
@@ -1560,7 +1566,7 @@ Parser::OwningStmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
   if (Handlers.empty())
     return StmtError();
 
-  return Actions.ActOnCXXTryBlock(TryLoc, move(TryBlock), move_arg(Handlers));
+  return Actions.ActOnCXXTryBlock(TryLoc, TryBlock.take(), move_arg(Handlers));
 }
 
 /// ParseCXXCatchBlock - Parse a C++ catch block, called handler in the standard
@@ -1574,7 +1580,7 @@ Parser::OwningStmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
 ///         type-specifier-seq
 ///         '...'
 ///
-Parser::OwningStmtResult Parser::ParseCXXCatchBlock() {
+StmtResult Parser::ParseCXXCatchBlock() {
   assert(Tok.is(tok::kw_catch) && "Expected 'catch'");
 
   SourceLocation CatchLoc = ConsumeToken();
@@ -1590,7 +1596,7 @@ Parser::OwningStmtResult Parser::ParseCXXCatchBlock() {
 
   // exception-declaration is equivalent to '...' or a parameter-declaration
   // without default arguments.
-  DeclPtrTy ExceptionDecl;
+  Decl *ExceptionDecl = 0;
   if (Tok.isNot(tok::ellipsis)) {
     DeclSpec DS;
     if (ParseCXXTypeSpecifierSeq(DS))
@@ -1608,9 +1614,9 @@ Parser::OwningStmtResult Parser::ParseCXXCatchBlock() {
     return StmtError(Diag(Tok, diag::err_expected_lbrace));
 
   // FIXME: Possible draft standard bug: attribute-specifier should be allowed?
-  OwningStmtResult Block(ParseCompoundStatement(0));
+  StmtResult Block(ParseCompoundStatement(0));
   if (Block.isInvalid())
     return move(Block);
 
-  return Actions.ActOnCXXCatchBlock(CatchLoc, ExceptionDecl, move(Block));
+  return Actions.ActOnCXXCatchBlock(CatchLoc, ExceptionDecl, Block.take());
 }
diff --git a/lib/Parse/ParseTemplate.cpp b/lib/Parse/ParseTemplate.cpp
index e1aaf91bd658..dfb4785489d0 100644
--- a/lib/Parse/ParseTemplate.cpp
+++ b/lib/Parse/ParseTemplate.cpp
@@ -13,15 +13,15 @@
 
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/Scope.h"
 #include "RAIIObjectsForParser.h"
 using namespace clang;
 
 /// \brief Parse a template declaration, explicit instantiation, or
 /// explicit specialization.
-Parser::DeclPtrTy
+Decl *
 Parser::ParseDeclarationStartingWithTemplate(unsigned Context,
                                              SourceLocation &DeclEnd,
                                              AccessSpecifier AS) {
@@ -70,7 +70,7 @@ namespace {
 ///
 ///       explicit-specialization: [ C++ temp.expl.spec]
 ///         'template' '<' '>' declaration
-Parser::DeclPtrTy
+Decl *
 Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
                                                  SourceLocation &DeclEnd,
                                                  AccessSpecifier AS) {
@@ -80,6 +80,10 @@ Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
   // Enter template-parameter scope.
   ParseScope TemplateParmScope(this, Scope::TemplateParamScope);
 
+  // Tell the action that names should be checked in the context of
+  // the declaration to come.
+  ParsingDeclRAIIObject ParsingTemplateParams(*this);
+
   // Parse multiple levels of template headers within this template
   // parameter scope, e.g.,
   //
@@ -118,19 +122,19 @@ Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
       TemplateLoc = ConsumeToken();
     } else {
       Diag(Tok.getLocation(), diag::err_expected_template);
-      return DeclPtrTy();
+      return 0;
     }
 
     // Parse the '<' template-parameter-list '>'
     SourceLocation LAngleLoc, RAngleLoc;
-    TemplateParameterList TemplateParams;
+    llvm::SmallVector<Decl*, 4> TemplateParams;
     if (ParseTemplateParameters(Depth, TemplateParams, LAngleLoc,
                                 RAngleLoc)) {
       // Skip until the semi-colon or a }.
       SkipUntil(tok::r_brace, true, true);
       if (Tok.is(tok::semi))
         ConsumeToken();
-      return DeclPtrTy();
+      return 0;
     }
 
     ParamLists.push_back(
@@ -152,6 +156,7 @@ Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
                                              ParsedTemplateInfo(&ParamLists,
                                                              isSpecialization,
                                                          LastParamListWasEmpty),
+                                             ParsingTemplateParams,
                                              DeclEnd, AS);
 }
 
@@ -175,10 +180,11 @@ Parser::ParseTemplateDeclarationOrSpecialization(unsigned Context,
 /// declaration. Will be AS_none for namespace-scope declarations.
 ///
 /// \returns the new declaration.
-Parser::DeclPtrTy
+Decl *
 Parser::ParseSingleDeclarationAfterTemplate(
                                        unsigned Context,
                                        const ParsedTemplateInfo &TemplateInfo,
+                                       ParsingDeclRAIIObject &DiagsFromTParams,
                                        SourceLocation &DeclEnd,
                                        AccessSpecifier AS) {
   assert(TemplateInfo.Kind != ParsedTemplateInfo::NonTemplate &&
@@ -186,12 +192,13 @@ Parser::ParseSingleDeclarationAfterTemplate(
 
   if (Context == Declarator::MemberContext) {
     // We are parsing a member template.
-    ParseCXXClassMemberDeclaration(AS, TemplateInfo);
-    return DeclPtrTy::make((void*)0);
+    ParseCXXClassMemberDeclaration(AS, TemplateInfo, &DiagsFromTParams);
+    return 0;
   }
 
-  // Parse the declaration specifiers.
-  ParsingDeclSpec DS(*this);
+  // Parse the declaration specifiers, stealing the accumulated
+  // diagnostics from the template parameters.
+  ParsingDeclSpec DS(DiagsFromTParams);
 
   if (getLang().CPlusPlus0x && isCXX0XAttributeSpecifier())
     DS.AddAttributes(ParseCXX0XAttributes().AttrList);
@@ -201,7 +208,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
 
   if (Tok.is(tok::semi)) {
     DeclEnd = ConsumeToken();
-    DeclPtrTy Decl = Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS, DS);
+    Decl *Decl = Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS, DS);
     DS.complete(Decl);
     return Decl;
   }
@@ -215,14 +222,14 @@ Parser::ParseSingleDeclarationAfterTemplate(
     SkipUntil(tok::r_brace, true, true);
     if (Tok.is(tok::semi))
       ConsumeToken();
-    return DeclPtrTy();
+    return 0;
   }
 
   // If we have a declaration or declarator list, handle it.
   if (isDeclarationAfterDeclarator()) {
     // Parse this declaration.
-    DeclPtrTy ThisDecl = ParseDeclarationAfterDeclarator(DeclaratorInfo,
-                                                         TemplateInfo);
+    Decl *ThisDecl = ParseDeclarationAfterDeclarator(DeclaratorInfo,
+                                                     TemplateInfo);
 
     if (Tok.is(tok::comma)) {
       Diag(Tok, diag::err_multiple_template_declarators)
@@ -251,7 +258,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
       } else {
         SkipUntil(tok::semi);
       }
-      return DeclPtrTy();
+      return 0;
     }
     return ParseFunctionDefinition(DeclaratorInfo, TemplateInfo);
   }
@@ -261,7 +268,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
   else
     Diag(Tok, diag::err_invalid_token_after_toplevel_declarator);
   SkipUntil(tok::semi);
-  return DeclPtrTy();
+  return 0;
 }
 
 /// ParseTemplateParameters - Parses a template-parameter-list enclosed in
@@ -274,7 +281,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
 ///
 /// \returns true if an error occurred, false otherwise.
 bool Parser::ParseTemplateParameters(unsigned Depth,
-                                     TemplateParameterList &TemplateParams,
+                               llvm::SmallVectorImpl<Decl*> &TemplateParams,
                                      SourceLocation &LAngleLoc,
                                      SourceLocation &RAngleLoc) {
   // Get the template parameter list.
@@ -307,9 +314,9 @@ bool Parser::ParseTemplateParameters(unsigned Depth,
 ///         template-parameter-list ',' template-parameter
 bool
 Parser::ParseTemplateParameterList(unsigned Depth,
-                                   TemplateParameterList &TemplateParams) {
+                             llvm::SmallVectorImpl<Decl*> &TemplateParams) {
   while (1) {
-    if (DeclPtrTy TmpParam
+    if (Decl *TmpParam
           = ParseTemplateParameter(Depth, TemplateParams.size())) {
       TemplateParams.push_back(TmpParam);
     } else {
@@ -414,8 +421,7 @@ bool Parser::isStartOfTemplateTypeParameter() {
 ///         'typename' identifier[opt] '=' type-id
 ///         'template' ...[opt][C++0x] '<' template-parameter-list '>' 'class' identifier[opt]
 ///         'template' '<' template-parameter-list '>' 'class' identifier[opt] = id-expression
-Parser::DeclPtrTy
-Parser::ParseTemplateParameter(unsigned Depth, unsigned Position) {
+Decl *Parser::ParseTemplateParameter(unsigned Depth, unsigned Position) {
   if (isStartOfTemplateTypeParameter())
     return ParseTypeParameter(Depth, Position);
 
@@ -437,7 +443,7 @@ Parser::ParseTemplateParameter(unsigned Depth, unsigned Position) {
 ///         'class' identifier[opt] '=' type-id
 ///         'typename' ...[opt][C++0x] identifier[opt]
 ///         'typename' identifier[opt] '=' type-id
-Parser::DeclPtrTy Parser::ParseTypeParameter(unsigned Depth, unsigned Position){
+Decl *Parser::ParseTypeParameter(unsigned Depth, unsigned Position) {
   assert((Tok.is(tok::kw_class) || Tok.is(tok::kw_typename)) &&
          "A type-parameter starts with 'class' or 'typename'");
 
@@ -468,14 +474,14 @@ Parser::DeclPtrTy Parser::ParseTypeParameter(unsigned Depth, unsigned Position){
     // don't consume this token.
   } else {
     Diag(Tok.getLocation(), diag::err_expected_ident);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Grab a default argument (if available).
   // Per C++0x [basic.scope.pdecl]p9, we parse the default argument before
   // we introduce the type parameter into the local scope.
   SourceLocation EqualLoc;
-  TypeTy *DefaultArg = 0;
+  ParsedType DefaultArg;
   if (Tok.is(tok::equal)) {
     EqualLoc = ConsumeToken();
     DefaultArg = ParseTypeName().get();
@@ -492,19 +498,19 @@ Parser::DeclPtrTy Parser::ParseTypeParameter(unsigned Depth, unsigned Position){
 ///       type-parameter:    [C++ temp.param]
 ///         'template' '<' template-parameter-list '>' 'class' identifier[opt]
 ///         'template' '<' template-parameter-list '>' 'class' identifier[opt] = id-expression
-Parser::DeclPtrTy
+Decl *
 Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   assert(Tok.is(tok::kw_template) && "Expected 'template' keyword");
 
   // Handle the template <...> part.
   SourceLocation TemplateLoc = ConsumeToken();
-  TemplateParameterList TemplateParams;
+  llvm::SmallVector<Decl*,8> TemplateParams;
   SourceLocation LAngleLoc, RAngleLoc;
   {
     ParseScope TemplateParmScope(this, Scope::TemplateParamScope);
     if (ParseTemplateParameters(Depth + 1, TemplateParams, LAngleLoc,
                                RAngleLoc)) {
-      return DeclPtrTy();
+      return 0;
     }
   }
 
@@ -513,7 +519,7 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
   if (!Tok.is(tok::kw_class)) {
     Diag(Tok.getLocation(), diag::err_expected_class_before)
       << PP.getSpelling(Tok);
-    return DeclPtrTy();
+    return 0;
   }
   SourceLocation ClassLoc = ConsumeToken();
 
@@ -528,7 +534,7 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
     // don't consume this token.
   } else {
     Diag(Tok.getLocation(), diag::err_expected_ident);
-    return DeclPtrTy();
+    return 0;
   }
 
   TemplateParamsTy *ParamList =
@@ -568,7 +574,7 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
 ///       template-parameter:
 ///         ...
 ///         parameter-declaration
-Parser::DeclPtrTy
+Decl *
 Parser::ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position) {
   SourceLocation StartLoc = Tok.getLocation();
 
@@ -581,21 +587,21 @@ Parser::ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position) {
   // Parse this as a typename.
   Declarator ParamDecl(DS, Declarator::TemplateParamContext);
   ParseDeclarator(ParamDecl);
-  if (DS.getTypeSpecType() == DeclSpec::TST_unspecified && !DS.getTypeRep()) {
+  if (DS.getTypeSpecType() == DeclSpec::TST_unspecified) {
     // This probably shouldn't happen - and it's more of a Sema thing, but
     // basically we didn't parse the type name because we couldn't associate
     // it with an AST node. we should just skip to the comma or greater.
     // TODO: This is currently a placeholder for some kind of Sema Error.
     Diag(Tok.getLocation(), diag::err_parse_error);
     SkipUntil(tok::comma, tok::greater, true, true);
-    return DeclPtrTy();
+    return 0;
   }
 
   // If there is a default value, parse it.
   // Per C++0x [basic.scope.pdecl]p9, we parse the default argument before
   // we introduce the template parameter into the local scope.
   SourceLocation EqualLoc;
-  OwningExprResult DefaultArg(Actions);
+  ExprResult DefaultArg;
   if (Tok.is(tok::equal)) {
     EqualLoc = ConsumeToken();
 
@@ -614,7 +620,7 @@ Parser::ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position) {
   // Create the parameter.
   return Actions.ActOnNonTypeTemplateParameter(getCurScope(), ParamDecl, 
                                                Depth, Position, EqualLoc, 
-                                               move(DefaultArg));
+                                               DefaultArg.take());
 }
 
 /// \brief Parses a template-id that after the template name has
@@ -766,7 +772,7 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
 
   // Build the annotation token.
   if (TNK == TNK_Type_template && AllowTypeAnnotation) {
-    Action::TypeResult Type
+    TypeResult Type
       = Actions.ActOnTemplateIdType(Template, TemplateNameLoc,
                                     LAngleLoc, TemplateArgsPtr,
                                     RAngleLoc);
@@ -779,7 +785,7 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
     }
 
     Tok.setKind(tok::annot_typename);
-    Tok.setAnnotationValue(Type.get());
+    setTypeAnnotation(Tok, Type.get());
     if (SS && SS->isNotEmpty())
       Tok.setLocation(SS->getBeginLoc());
     else if (TemplateKWLoc.isValid())
@@ -800,7 +806,7 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
       TemplateId->Name = 0;
       TemplateId->Operator = TemplateName.OperatorFunctionId.Operator;
     }
-    TemplateId->Template = Template.getAs<void*>();
+    TemplateId->Template = Template;
     TemplateId->Kind = TNK;
     TemplateId->LAngleLoc = LAngleLoc;
     TemplateId->RAngleLoc = RAngleLoc;
@@ -844,15 +850,15 @@ void Parser::AnnotateTemplateIdTokenAsType(const CXXScopeSpec *SS) {
                                      TemplateId->getTemplateArgs(),
                                      TemplateId->NumArgs);
 
-  Action::TypeResult Type
-    = Actions.ActOnTemplateIdType(TemplateTy::make(TemplateId->Template),
+  TypeResult Type
+    = Actions.ActOnTemplateIdType(TemplateId->Template,
                                   TemplateId->TemplateNameLoc,
                                   TemplateId->LAngleLoc,
                                   TemplateArgsPtr,
                                   TemplateId->RAngleLoc);
   // Create the new "type" annotation token.
   Tok.setKind(tok::annot_typename);
-  Tok.setAnnotationValue(Type.isInvalid()? 0 : Type.get());
+  setTypeAnnotation(Tok, Type.isInvalid() ? ParsedType() : Type.get());
   if (SS && SS->isNotEmpty()) // it was a C++ qualified type name.
     Tok.setLocation(SS->getBeginLoc());
   // End location stays the same
@@ -887,7 +893,7 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
   // followed by a token that terminates a template argument, such as ',', 
   // '>', or (in some cases) '>>'.
   CXXScopeSpec SS; // nested-name-specifier, if present
-  ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, 
+  ParseOptionalCXXScopeSpecifier(SS, ParsedType(),
                                  /*EnteringContext=*/false);
   
   if (SS.isSet() && Tok.is(tok::kw_template)) {
@@ -906,8 +912,9 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
       // template argument.
       TemplateTy Template;
       if (isEndOfTemplateArgument(Tok) &&
-          Actions.ActOnDependentTemplateName(getCurScope(), TemplateLoc, SS, Name, 
-                                             /*ObjectType=*/0,
+          Actions.ActOnDependentTemplateName(getCurScope(), TemplateLoc,
+                                             SS, Name, 
+                                             /*ObjectType=*/ ParsedType(),
                                              /*EnteringContext=*/false,
                                              Template))
         return ParsedTemplateArgument(SS, Template, Name.StartLocation);
@@ -921,8 +928,10 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
     
     if (isEndOfTemplateArgument(Tok)) {
       bool MemberOfUnknownSpecialization;
-      TemplateNameKind TNK = Actions.isTemplateName(getCurScope(), SS, Name, 
-                                                    /*ObjectType=*/0, 
+      TemplateNameKind TNK = Actions.isTemplateName(getCurScope(), SS,
+                                               /*hasTemplateKeyword=*/false,
+                                                    Name,
+                                               /*ObjectType=*/ ParsedType(), 
                                                     /*EnteringContext=*/false, 
                                                     Template,
                                                 MemberOfUnknownSpecialization);
@@ -957,7 +966,8 @@ ParsedTemplateArgument Parser::ParseTemplateArgument() {
     if (TypeArg.isInvalid())
       return ParsedTemplateArgument();
     
-    return ParsedTemplateArgument(ParsedTemplateArgument::Type, TypeArg.get(), 
+    return ParsedTemplateArgument(ParsedTemplateArgument::Type,
+                                  TypeArg.get().getAsOpaquePtr(), 
                                   Loc);
   }
   
@@ -978,7 +988,7 @@ ParsedTemplateArgument Parser::ParseTemplateArgument() {
   
   // Parse a non-type template argument. 
   SourceLocation Loc = Tok.getLocation();
-  OwningExprResult ExprArg = ParseConstantExpression();
+  ExprResult ExprArg = ParseConstantExpression();
   if (ExprArg.isInvalid() || !ExprArg.get())
     return ParsedTemplateArgument();
 
@@ -1053,12 +1063,15 @@ Parser::ParseTemplateArgumentList(TemplateArgList &TemplateArgs) {
 ///         'extern' [opt] 'template' declaration
 ///
 /// Note that the 'extern' is a GNU extension and C++0x feature.
-Parser::DeclPtrTy
-Parser::ParseExplicitInstantiation(SourceLocation ExternLoc,
-                                   SourceLocation TemplateLoc,
-                                   SourceLocation &DeclEnd) {
+Decl *Parser::ParseExplicitInstantiation(SourceLocation ExternLoc,
+                                         SourceLocation TemplateLoc,
+                                         SourceLocation &DeclEnd) {
+  // This isn't really required here.
+  ParsingDeclRAIIObject ParsingTemplateParams(*this);
+
   return ParseSingleDeclarationAfterTemplate(Declarator::FileContext,
                                              ParsedTemplateInfo(ExternLoc,
                                                                 TemplateLoc),
+                                             ParsingTemplateParams,
                                              DeclEnd, AS_none);
 }
diff --git a/lib/Parse/ParseTentative.cpp b/lib/Parse/ParseTentative.cpp
index 5e64e6162b73..c22d99fa9d38 100644
--- a/lib/Parse/ParseTentative.cpp
+++ b/lib/Parse/ParseTentative.cpp
@@ -14,7 +14,7 @@
 
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/ParsedTemplate.h"
 using namespace clang;
 
 /// isCXXDeclarationStatement - C++-specialized function that disambiguates
@@ -172,14 +172,6 @@ Parser::TPResult Parser::TryParseSimpleDeclaration() {
 ///   '{' '}'
 ///
 Parser::TPResult Parser::TryParseInitDeclaratorList() {
-  // GCC only examines the first declarator for disambiguation:
-  // i.e:
-  // int(x), ++x; // GCC regards it as ill-formed declaration.
-  //
-  // Comeau and MSVC will regard the above statement as correct expression.
-  // Clang examines all of the declarators and also regards the above statement
-  // as correct expression.
-
   while (1) {
     // declarator
     TPResult TPR = TryParseDeclarator(false/*mayBeAbstract*/);
@@ -196,15 +188,22 @@ Parser::TPResult Parser::TryParseInitDeclaratorList() {
       ConsumeParen();
       if (!SkipUntil(tok::r_paren))
         return TPResult::Error();
-    } else if (Tok.is(tok::equal)) {
-      // MSVC won't examine the rest of declarators if '=' is encountered, it
-      // will conclude that it is a declaration.
-      // Comeau and Clang will examine the rest of declarators.
-      // Note that "int(x) = {0}, ++x;" will be interpreted as ill-formed
-      // expression.
+    } else if (Tok.is(tok::equal) || isTokIdentifier_in()) {
+      // MSVC and g++ won't examine the rest of declarators if '=' is 
+      // encountered; they just conclude that we have a declaration.
+      // EDG parses the initializer completely, which is the proper behavior
+      // for this case.
       //
-      // Parse through the initializer-clause.
-      SkipUntil(tok::comma, true/*StopAtSemi*/, true/*DontConsume*/);
+      // At present, Clang follows MSVC and g++, since the parser does not have
+      // the ability to parse an expression fully without recording the
+      // results of that parse.
+      // Also allow 'in' after on objective-c declaration as in: 
+      // for (int (^b)(void) in array). Ideally this should be done in the 
+      // context of parsing for-init-statement of a foreach statement only. But,
+      // in any other context 'in' is invalid after a declaration and parser
+      // issues the error regardless of outcome of this decision.
+      // FIXME. Change if above assumption does not hold.
+      return TPResult::True();
     }
 
     if (Tok.isNot(tok::comma))
@@ -758,6 +757,10 @@ Parser::TPResult Parser::isCXXDeclarationSpecifier() {
   case tok::kw___ptr64:
   case tok::kw___forceinline:
     return TPResult::True();
+
+    // Borland
+  case tok::kw___pascal:
+    return TPResult::True();
   
     // AltiVec
   case tok::kw___vector:
diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp
index ac78f114a967..44bd0fbc0c03 100644
--- a/lib/Parse/Parser.cpp
+++ b/lib/Parse/Parser.cpp
@@ -13,15 +13,15 @@
 
 #include "clang/Parse/Parser.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "llvm/Support/raw_ostream.h"
 #include "RAIIObjectsForParser.h"
 #include "ParsePragma.h"
 using namespace clang;
 
-Parser::Parser(Preprocessor &pp, Action &actions)
+Parser::Parser(Preprocessor &pp, Sema &actions)
   : CrashInfo(*this), PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
     GreaterThanIsOperator(true), ColonIsSacred(false),
     TemplateParameterDepth(0) {
@@ -29,10 +29,16 @@ Parser::Parser(Preprocessor &pp, Action &actions)
   Actions.CurScope = 0;
   NumCachedScopes = 0;
   ParenCount = BracketCount = BraceCount = 0;
-  ObjCImpDecl = DeclPtrTy();
+  ObjCImpDecl = 0;
 
   // Add #pragma handlers. These are removed and destroyed in the
   // destructor.
+  AlignHandler.reset(new PragmaAlignHandler(actions));
+  PP.AddPragmaHandler(AlignHandler.get());
+
+  GCCVisibilityHandler.reset(new PragmaGCCVisibilityHandler(actions));
+  PP.AddPragmaHandler("GCC", GCCVisibilityHandler.get());
+
   OptionsHandler.reset(new PragmaOptionsHandler(actions));
   PP.AddPragmaHandler(OptionsHandler.get());
 
@@ -44,6 +50,8 @@ Parser::Parser(Preprocessor &pp, Action &actions)
 
   WeakHandler.reset(new PragmaWeakHandler(actions));
   PP.AddPragmaHandler(WeakHandler.get());
+      
+  PP.setCodeCompletionHandler(*this);
 }
 
 /// If a crash happens while the parser is active, print out a line indicating
@@ -298,6 +306,10 @@ Parser::~Parser() {
     delete ScopeCache[i];
 
   // Remove the pragma handlers we installed.
+  PP.RemovePragmaHandler(AlignHandler.get());
+  AlignHandler.reset();
+  PP.RemovePragmaHandler("GCC", GCCVisibilityHandler.get());
+  GCCVisibilityHandler.reset();
   PP.RemovePragmaHandler(OptionsHandler.get());
   OptionsHandler.reset();
   PP.RemovePragmaHandler(PackHandler.get());
@@ -306,18 +318,19 @@ Parser::~Parser() {
   UnusedHandler.reset();
   PP.RemovePragmaHandler(WeakHandler.get());
   WeakHandler.reset();
+  PP.clearCodeCompletionHandler();
 }
 
 /// Initialize - Warm up the parser.
 ///
 void Parser::Initialize() {
-  // Prime the lexer look-ahead.
-  ConsumeToken();
-
   // Create the translation unit scope.  Install it as the current scope.
   assert(getCurScope() == 0 && "A scope is already active?");
   EnterScope(Scope::DeclScope);
-  Actions.ActOnTranslationUnitScope(Tok.getLocation(), getCurScope());
+  Actions.ActOnTranslationUnitScope(getCurScope());
+
+  // Prime the lexer look-ahead.
+  ConsumeToken();
 
   if (Tok.is(tok::eof) &&
       !getLang().CPlusPlus)  // Empty source file is an extension in C
@@ -395,10 +408,11 @@ void Parser::ParseTranslationUnit() {
 ///           ';'
 ///
 /// [C++0x/GNU] 'extern' 'template' declaration
-Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr) {
+Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr,
+                                                        ParsingDeclSpec *DS) {
   ParenBraceBracketBalancer BalancerRAIIObj(*this);
   
-  DeclPtrTy SingleDecl;
+  Decl *SingleDecl = 0;
   switch (Tok.getKind()) {
   case tok::semi:
     if (!getLang().CPlusPlus0x)
@@ -426,14 +440,14 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr)
       Diag(Attr.Range.getBegin(), diag::err_attributes_not_allowed)
         << Attr.Range;
 
-    OwningExprResult Result(ParseSimpleAsm());
+    ExprResult Result(ParseSimpleAsm());
 
     ExpectAndConsume(tok::semi, diag::err_expected_semi_after,
                      "top-level asm block");
 
     if (Result.isInvalid())
       return DeclGroupPtrTy();
-    SingleDecl = Actions.ActOnFileScopeAsmDecl(Tok.getLocation(), move(Result));
+    SingleDecl = Actions.ActOnFileScopeAsmDecl(Tok.getLocation(), Result.get());
     break;
   }
   case tok::at:
@@ -453,8 +467,8 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr)
     break;
   case tok::code_completion:
       Actions.CodeCompleteOrdinaryName(getCurScope(), 
-                                   ObjCImpDecl? Action::CCC_ObjCImplementation
-                                              : Action::CCC_Namespace);
+                                   ObjCImpDecl? Sema::PCC_ObjCImplementation
+                                              : Sema::PCC_Namespace);
     ConsumeCodeCompletionToken();
     return ParseExternalDeclaration(Attr);
   case tok::kw_using:
@@ -468,6 +482,15 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr)
       SourceLocation DeclEnd;
       return ParseDeclaration(Declarator::FileContext, DeclEnd, Attr);
     }
+
+  case tok::kw_inline:
+    if (getLang().CPlusPlus && NextToken().is(tok::kw_namespace)) {
+      // Inline namespaces. Allowed as an extension even in C++03.
+      SourceLocation DeclEnd;
+      return ParseDeclaration(Declarator::FileContext, DeclEnd, Attr);
+    }
+    goto dont_know;
+
   case tok::kw_extern:
     if (getLang().CPlusPlus && NextToken().is(tok::kw_template)) {
       // Extern templates
@@ -477,14 +500,16 @@ Parser::DeclGroupPtrTy Parser::ParseExternalDeclaration(CXX0XAttributeList Attr)
       return Actions.ConvertDeclToDeclGroup(
                   ParseExplicitInstantiation(ExternLoc, TemplateLoc, DeclEnd));
     }
-
     // FIXME: Detect C++ linkage specifications here?
-
-    // Fall through to handle other declarations or function definitions.
+    goto dont_know;
 
   default:
+  dont_know:
     // We can't tell whether this is a function-definition or declaration yet.
-    return ParseDeclarationOrFunctionDefinition(Attr.AttrList);
+    if (DS)
+      return ParseDeclarationOrFunctionDefinition(*DS, Attr.AttrList);
+    else
+      return ParseDeclarationOrFunctionDefinition(Attr.AttrList);
   }
 
   // This routine returns a DeclGroup, if the thing we parsed only contains a
@@ -551,7 +576,7 @@ Parser::ParseDeclarationOrFunctionDefinition(ParsingDeclSpec &DS,
   // declaration-specifiers init-declarator-list[opt] ';'
   if (Tok.is(tok::semi)) {
     ConsumeToken();
-    DeclPtrTy TheDecl = Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS, DS);
+    Decl *TheDecl = Actions.ParsedFreeStandingDeclSpec(getCurScope(), AS, DS);
     DS.complete(TheDecl);
     return Actions.ConvertDeclToDeclGroup(TheDecl);
   }
@@ -575,7 +600,7 @@ Parser::ParseDeclarationOrFunctionDefinition(ParsingDeclSpec &DS,
     if (DS.SetTypeSpecType(DeclSpec::TST_unspecified, AtLoc, PrevSpec, DiagID))
       Diag(AtLoc, DiagID) << PrevSpec;
 
-    DeclPtrTy TheDecl;
+    Decl *TheDecl = 0;
     if (Tok.isObjCAtKeyword(tok::objc_protocol))
       TheDecl = ParseObjCAtProtocolDeclaration(AtLoc, DS.getAttributes());
     else
@@ -589,7 +614,7 @@ Parser::ParseDeclarationOrFunctionDefinition(ParsingDeclSpec &DS,
   if (Tok.is(tok::string_literal) && getLang().CPlusPlus &&
       DS.getStorageClassSpec() == DeclSpec::SCS_extern &&
       DS.getParsedSpecifiers() == DeclSpec::PQ_StorageClassSpecifier) {
-    DeclPtrTy TheDecl = ParseLinkage(DS, Declarator::FileContext);
+    Decl *TheDecl = ParseLinkage(DS, Declarator::FileContext);
     return Actions.ConvertDeclToDeclGroup(TheDecl);
   }
 
@@ -617,7 +642,7 @@ Parser::ParseDeclarationOrFunctionDefinition(AttributeList *Attr,
 /// [C++] function-definition: [C++ 8.4]
 ///         decl-specifier-seq[opt] declarator function-try-block
 ///
-Parser::DeclPtrTy Parser::ParseFunctionDefinition(ParsingDeclarator &D,
+Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
                                      const ParsedTemplateInfo &TemplateInfo) {
   const DeclaratorChunk &FnTypeInfo = D.getTypeObject(0);
   assert(FnTypeInfo.Kind == DeclaratorChunk::Function &&
@@ -653,7 +678,7 @@ Parser::DeclPtrTy Parser::ParseFunctionDefinition(ParsingDeclarator &D,
 
     // If we didn't find the '{', bail out.
     if (Tok.isNot(tok::l_brace))
-      return DeclPtrTy();
+      return 0;
   }
 
   // Enter a scope for the function body.
@@ -661,9 +686,9 @@ Parser::DeclPtrTy Parser::ParseFunctionDefinition(ParsingDeclarator &D,
 
   // Tell the actions module that we have entered a function definition with the
   // specified Declarator for the function.
-  DeclPtrTy Res = TemplateInfo.TemplateParams?
+  Decl *Res = TemplateInfo.TemplateParams?
       Actions.ActOnStartOfFunctionTemplateDef(getCurScope(),
-                              Action::MultiTemplateParamsArg(Actions,
+                              MultiTemplateParamsArg(Actions,
                                           TemplateInfo.TemplateParams->data(),
                                          TemplateInfo.TemplateParams->size()),
                                               D)
@@ -686,7 +711,7 @@ Parser::DeclPtrTy Parser::ParseFunctionDefinition(ParsingDeclarator &D,
 
     // Recover from error.
     if (!Tok.is(tok::l_brace)) {
-      Actions.ActOnFinishFunctionBody(Res, Action::StmtArg(Actions));
+      Actions.ActOnFinishFunctionBody(Res, 0);
       return Res;
     }
   } else
@@ -752,7 +777,7 @@ void Parser::ParseKNRParamDeclarations(Declarator &D) {
       }
 
       // Ask the actions module to compute the type for this declarator.
-      Action::DeclPtrTy Param =
+      Decl *Param =
         Actions.ActOnParamDeclarator(getCurScope(), ParmDeclarator);
 
       if (Param &&
@@ -819,13 +844,13 @@ void Parser::ParseKNRParamDeclarations(Declarator &D) {
 /// [GNU] asm-string-literal:
 ///         string-literal
 ///
-Parser::OwningExprResult Parser::ParseAsmStringLiteral() {
+Parser::ExprResult Parser::ParseAsmStringLiteral() {
   if (!isTokenStringLiteral()) {
     Diag(Tok, diag::err_expected_string_literal);
     return ExprError();
   }
 
-  OwningExprResult Res(ParseStringLiteralExpression());
+  ExprResult Res(ParseStringLiteralExpression());
   if (Res.isInvalid()) return move(Res);
 
   // TODO: Diagnose: wide string literal in 'asm'
@@ -838,7 +863,7 @@ Parser::OwningExprResult Parser::ParseAsmStringLiteral() {
 /// [GNU] simple-asm-expr:
 ///         'asm' '(' asm-string-literal ')'
 ///
-Parser::OwningExprResult Parser::ParseSimpleAsm(SourceLocation *EndLoc) {
+Parser::ExprResult Parser::ParseSimpleAsm(SourceLocation *EndLoc) {
   assert(Tok.is(tok::kw_asm) && "Not an asm!");
   SourceLocation Loc = ConsumeToken();
 
@@ -859,7 +884,7 @@ Parser::OwningExprResult Parser::ParseSimpleAsm(SourceLocation *EndLoc) {
 
   Loc = ConsumeParen();
 
-  OwningExprResult Result(ParseAsmStringLiteral());
+  ExprResult Result(ParseAsmStringLiteral());
 
   if (Result.isInvalid()) {
     SkipUntil(tok::r_paren, true, true);
@@ -911,7 +936,7 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
     //            simple-template-id
     SourceLocation TypenameLoc = ConsumeToken();
     CXXScopeSpec SS;
-    if (ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, false))
+    if (ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/ParsedType(), false))
       return true;
     if (!SS.isSet()) {
       Diag(Tok.getLocation(), diag::err_expected_qualified_after_typename);
@@ -939,7 +964,7 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
       if (Tok.getAnnotationValue())
         Ty = Actions.ActOnTypenameType(getCurScope(), TypenameLoc, SS, 
                                        SourceLocation(),
-                                       Tok.getAnnotationValue());
+                                       getTypeAnnotation(Tok));
       else
         Ty = true;
     } else {
@@ -950,7 +975,7 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
 
     SourceLocation EndLoc = Tok.getLastLoc();
     Tok.setKind(tok::annot_typename);
-    Tok.setAnnotationValue(Ty.isInvalid()? 0 : Ty.get());
+    setTypeAnnotation(Tok, Ty.isInvalid() ? ParsedType() : Ty.get());
     Tok.setAnnotationEndLoc(EndLoc);
     Tok.setLocation(TypenameLoc);
     PP.AnnotateCachedTokens(Tok);
@@ -962,17 +987,18 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
 
   CXXScopeSpec SS;
   if (getLang().CPlusPlus)
-    if (ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, EnteringContext))
+    if (ParseOptionalCXXScopeSpecifier(SS, ParsedType(), EnteringContext))
       return true;
 
   if (Tok.is(tok::identifier)) {
     // Determine whether the identifier is a type name.
-    if (TypeTy *Ty = Actions.getTypeName(*Tok.getIdentifierInfo(),
-                                         Tok.getLocation(), getCurScope(), &SS)) {
+    if (ParsedType Ty = Actions.getTypeName(*Tok.getIdentifierInfo(),
+                                            Tok.getLocation(), getCurScope(),
+                                            &SS)) {
       // This is a typename. Replace the current token in-place with an
       // annotation type token.
       Tok.setKind(tok::annot_typename);
-      Tok.setAnnotationValue(Ty);
+      setTypeAnnotation(Tok, Ty);
       Tok.setAnnotationEndLoc(Tok.getLocation());
       if (SS.isNotEmpty()) // it was a C++ qualified type name.
         Tok.setLocation(SS.getBeginLoc());
@@ -997,9 +1023,11 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext) {
       TemplateName.setIdentifier(Tok.getIdentifierInfo(), Tok.getLocation());
       bool MemberOfUnknownSpecialization;
       if (TemplateNameKind TNK
-            = Actions.isTemplateName(getCurScope(), SS, TemplateName, 
-                                     /*ObjectType=*/0, EnteringContext,
-                                     Template, MemberOfUnknownSpecialization)) {
+          = Actions.isTemplateName(getCurScope(), SS,
+                                   /*hasTemplateKeyword=*/false, TemplateName,
+                                   /*ObjectType=*/ ParsedType(),
+                                   EnteringContext,
+                                   Template, MemberOfUnknownSpecialization)) {
         // Consume the identifier.
         ConsumeToken();
         if (AnnotateTemplateIdToken(Template, TNK, &SS, TemplateName)) {
@@ -1066,7 +1094,7 @@ bool Parser::TryAnnotateCXXScopeToken(bool EnteringContext) {
          "Cannot be a type or scope token!");
 
   CXXScopeSpec SS;
-  if (ParseOptionalCXXScopeSpecifier(SS, /*ObjectType=*/0, EnteringContext))
+  if (ParseOptionalCXXScopeSpecifier(SS, ParsedType(), EnteringContext))
     return true;
   if (SS.isEmpty())
     return false;
@@ -1090,17 +1118,17 @@ bool Parser::TryAnnotateCXXScopeToken(bool EnteringContext) {
 void Parser::CodeCompletionRecovery() {
   for (Scope *S = getCurScope(); S; S = S->getParent()) {
     if (S->getFlags() & Scope::FnScope) {
-      Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_RecoveryInFunction);
+      Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_RecoveryInFunction);
       return;
     }
     
     if (S->getFlags() & Scope::ClassScope) {
-      Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_Class);
+      Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class);
       return;
     }
   }
   
-  Actions.CodeCompleteOrdinaryName(getCurScope(), Action::CCC_Namespace);
+  Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace);
 }
 
 // Anchor the Parser::FieldCallback vtable to this translation unit.
@@ -1109,3 +1137,32 @@ void Parser::CodeCompletionRecovery() {
 // performance-sensitive.
 void Parser::FieldCallback::_anchor() {
 }
+
+// Code-completion pass-through functions
+
+void Parser::CodeCompleteDirective(bool InConditional) {
+  Actions.CodeCompletePreprocessorDirective(InConditional);
+}
+
+void Parser::CodeCompleteInConditionalExclusion() {
+  Actions.CodeCompleteInPreprocessorConditionalExclusion(getCurScope());
+}
+
+void Parser::CodeCompleteMacroName(bool IsDefinition) {
+  Actions.CodeCompletePreprocessorMacroName(IsDefinition);
+}
+
+void Parser::CodeCompletePreprocessorExpression() { 
+  Actions.CodeCompletePreprocessorExpression();
+}
+
+void Parser::CodeCompleteMacroArgument(IdentifierInfo *Macro,
+                                       MacroInfo *MacroInfo,
+                                       unsigned ArgumentIndex) {
+  Actions.CodeCompletePreprocessorMacroArgument(getCurScope(), Macro, MacroInfo, 
+                                                ArgumentIndex);
+}
+
+void Parser::CodeCompleteNaturalLanguage() {
+  Actions.CodeCompleteNaturalLanguage();
+}
diff --git a/lib/Rewrite/CMakeLists.txt b/lib/Rewrite/CMakeLists.txt
index ce728afea850..ffeb3e66ebcb 100644
--- a/lib/Rewrite/CMakeLists.txt
+++ b/lib/Rewrite/CMakeLists.txt
@@ -13,3 +13,9 @@ add_clang_library(clangRewrite
   Rewriter.cpp
   TokenRewriter.cpp
   )
+
+add_dependencies(clangBasic
+  ClangAttrClasses
+  ClangAttrList
+  ClangDeclNodes
+  ClangStmtNodes)
diff --git a/lib/Rewrite/DeltaTree.cpp b/lib/Rewrite/DeltaTree.cpp
index 35e888bcf595..085dfd89ef15 100644
--- a/lib/Rewrite/DeltaTree.cpp
+++ b/lib/Rewrite/DeltaTree.cpp
@@ -116,7 +116,7 @@ namespace {
 
     void Destroy();
 
-    static inline bool classof(const DeltaTreeNode *) { return true; }
+    //static inline bool classof(const DeltaTreeNode *) { return true; }
   };
 } // end anonymous namespace
 
@@ -133,12 +133,6 @@ namespace {
   public:
     DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
 
-    DeltaTreeInteriorNode(DeltaTreeNode *FirstChild)
-    : DeltaTreeNode(false /*nonleaf*/) {
-      FullDelta = FirstChild->FullDelta;
-      Children[0] = FirstChild;
-    }
-
     DeltaTreeInteriorNode(const InsertResult &IR)
       : DeltaTreeNode(false /*nonleaf*/) {
       Children[0] = IR.LHS;
@@ -157,7 +151,7 @@ namespace {
       return Children[i];
     }
 
-    static inline bool classof(const DeltaTreeInteriorNode *) { return true; }
+  //static inline bool classof(const DeltaTreeInteriorNode *) { return true; }
     static inline bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
   };
 }
diff --git a/lib/Rewrite/FixItRewriter.cpp b/lib/Rewrite/FixItRewriter.cpp
index 29ac7e380bfe..582096978d7b 100644
--- a/lib/Rewrite/FixItRewriter.cpp
+++ b/lib/Rewrite/FixItRewriter.cpp
@@ -27,16 +27,17 @@ using namespace clang;
 
 FixItRewriter::FixItRewriter(Diagnostic &Diags, SourceManager &SourceMgr,
                              const LangOptions &LangOpts,
-                             FixItPathRewriter *PathRewriter)
+                             FixItOptions *FixItOpts)
   : Diags(Diags),
     Rewrite(SourceMgr, LangOpts),
-    PathRewriter(PathRewriter),
+    FixItOpts(FixItOpts),
     NumFailures(0) {
-  Client = Diags.getClient();
+  Client = Diags.takeClient();
   Diags.setClient(this);
 }
 
 FixItRewriter::~FixItRewriter() {
+  Diags.takeClient();
   Diags.setClient(Client);
 }
 
@@ -49,16 +50,14 @@ bool FixItRewriter::WriteFixedFile(FileID ID, llvm::raw_ostream &OS) {
 }
 
 bool FixItRewriter::WriteFixedFiles() {
-  if (NumFailures > 0) {
+  if (NumFailures > 0 && !FixItOpts->FixWhatYouCan) {
     Diag(FullSourceLoc(), diag::warn_fixit_no_changes);
     return true;
   }
 
   for (iterator I = buffer_begin(), E = buffer_end(); I != E; ++I) {
     const FileEntry *Entry = Rewrite.getSourceMgr().getFileEntryForID(I->first);
-    std::string Filename = Entry->getName();
-    if (PathRewriter)
-      Filename = PathRewriter->RewriteFilename(Filename);
+    std::string Filename = FixItOpts->RewriteFilename(Entry->getName());
     std::string Err;
     llvm::raw_fd_ostream OS(Filename.c_str(), Err,
                             llvm::raw_fd_ostream::F_Binary);
@@ -98,12 +97,6 @@ void FixItRewriter::HandleDiagnostic(Diagnostic::Level DiagLevel,
       CanRewrite = false;
       break;
     }
-
-    if (Hint.InsertionLoc.isValid() &&
-        !Rewrite.isRewritable(Hint.InsertionLoc)) {
-      CanRewrite = false;
-      break;
-    }
   }
 
   if (!CanRewrite) {
@@ -122,12 +115,6 @@ void FixItRewriter::HandleDiagnostic(Diagnostic::Level DiagLevel,
   for (unsigned Idx = 0, Last = Info.getNumFixItHints();
        Idx < Last; ++Idx) {
     const FixItHint &Hint = Info.getFixItHint(Idx);
-    if (!Hint.RemoveRange.isValid()) {
-      // We're adding code.
-      if (Rewrite.InsertTextBefore(Hint.InsertionLoc, Hint.CodeToInsert))
-        Failed = true;
-      continue;
-    }
 
     if (Hint.CodeToInsert.empty()) {
       // We're removing code.
@@ -158,10 +145,12 @@ void FixItRewriter::Diag(FullSourceLoc Loc, unsigned DiagID) {
   // When producing this diagnostic, we temporarily bypass ourselves,
   // clear out any current diagnostic, and let the downstream client
   // format the diagnostic.
+  Diags.takeClient();
   Diags.setClient(Client);
   Diags.Clear();
   Diags.Report(Loc, DiagID);
+  Diags.takeClient();
   Diags.setClient(this);
 }
 
-FixItPathRewriter::~FixItPathRewriter() {}
+FixItOptions::~FixItOptions() {}
diff --git a/lib/Rewrite/FrontendActions.cpp b/lib/Rewrite/FrontendActions.cpp
index 6da3b4bf519c..977e0cfba6d0 100644
--- a/lib/Rewrite/FrontendActions.cpp
+++ b/lib/Rewrite/FrontendActions.cpp
@@ -42,12 +42,19 @@ ASTConsumer *FixItAction::CreateASTConsumer(CompilerInstance &CI,
   return new ASTConsumer();
 }
 
-class FixItActionSuffixInserter : public FixItPathRewriter {
+class FixItRewriteInPlace : public FixItOptions {
+public:
+  std::string RewriteFilename(const std::string &Filename) { return Filename; }
+};
+
+class FixItActionSuffixInserter : public FixItOptions {
   std::string NewSuffix;
 
 public:
-  explicit FixItActionSuffixInserter(std::string NewSuffix)
-    : NewSuffix(NewSuffix) {}
+  FixItActionSuffixInserter(std::string NewSuffix, bool FixWhatYouCan)
+    : NewSuffix(NewSuffix) {
+      this->FixWhatYouCan = FixWhatYouCan;
+  }
 
   std::string RewriteFilename(const std::string &Filename) {
     llvm::sys::Path Path(Filename);
@@ -62,12 +69,14 @@ bool FixItAction::BeginSourceFileAction(CompilerInstance &CI,
                                         llvm::StringRef Filename) {
   const FrontendOptions &FEOpts = getCompilerInstance().getFrontendOpts();
   if (!FEOpts.FixItSuffix.empty()) {
-    PathRewriter.reset(new FixItActionSuffixInserter(FEOpts.FixItSuffix));
+    FixItOpts.reset(new FixItActionSuffixInserter(FEOpts.FixItSuffix,
+                                                  FEOpts.FixWhatYouCan));
   } else {
-    PathRewriter.reset();
+    FixItOpts.reset(new FixItRewriteInPlace);
+    FixItOpts->FixWhatYouCan = FEOpts.FixWhatYouCan;
   }
   Rewriter.reset(new FixItRewriter(CI.getDiagnostics(), CI.getSourceManager(),
-                                   CI.getLangOpts(), PathRewriter.get()));
+                                   CI.getLangOpts(), FixItOpts.get()));
   return true;
 }
 
diff --git a/lib/Rewrite/HTMLRewrite.cpp b/lib/Rewrite/HTMLRewrite.cpp
index 5fe064990e96..b461df462e2e 100644
--- a/lib/Rewrite/HTMLRewrite.cpp
+++ b/lib/Rewrite/HTMLRewrite.cpp
@@ -486,8 +486,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
 
   // Temporarily change the diagnostics object so that we ignore any generated
   // diagnostics from this pass.
-  IgnoringDiagClient TmpDC;
-  Diagnostic TmpDiags(&TmpDC);
+  Diagnostic TmpDiags(new IgnoringDiagClient);
 
   // FIXME: This is a huge hack; we reuse the input preprocessor because we want
   // its state, but we aren't actually changing it (we hope). This should really
diff --git a/lib/Rewrite/Makefile b/lib/Rewrite/Makefile
index 1c5b8a8117c2..5fef9b2c0d38 100644
--- a/lib/Rewrite/Makefile
+++ b/lib/Rewrite/Makefile
@@ -13,7 +13,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangRewrite
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Rewrite/RewriteObjC.cpp b/lib/Rewrite/RewriteObjC.cpp
index 489fec9be0e5..578a063614a1 100644
--- a/lib/Rewrite/RewriteObjC.cpp
+++ b/lib/Rewrite/RewriteObjC.cpp
@@ -229,14 +229,6 @@ namespace {
       Diags.Report(Context->getFullLoc(Loc), RewriteFailedDiag);
     }
 
-    void RemoveText(SourceLocation Loc, unsigned StrLen) {
-      // If removal succeeded or warning disabled return with no warning.
-      if (!Rewrite.RemoveText(Loc, StrLen) || SilenceRewriteMacroWarning)
-        return;
-
-      Diags.Report(Context->getFullLoc(Loc), RewriteFailedDiag);
-    }
-
     void ReplaceText(SourceLocation Start, unsigned OrigLength,
                      llvm::StringRef Str) {
       // If removal succeeded or warning disabled return with no warning.
@@ -248,9 +240,7 @@ namespace {
     }
 
     // Syntactic Rewriting.
-    void RewritePrologue(SourceLocation Loc);
     void RewriteInclude();
-    void RewriteTabs();
     void RewriteForwardClassDecl(ObjCClassDecl *Dcl);
     void RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
                                  ObjCImplementationDecl *IMD,
@@ -275,7 +265,6 @@ namespace {
     void RewriteTypeOfDecl(VarDecl *VD);
     void RewriteObjCQualifiedInterfaceTypes(Expr *E);
     bool needToScanForQualifiers(QualType T);
-    bool isSuperReceiver(Expr *recExpr);
     QualType getSuperStructType();
     QualType getConstantStringStructType();
     QualType convertFunctionTypeOfBlocks(const FunctionType *FT);
@@ -305,8 +294,6 @@ namespace {
     void RewriteSyncReturnStmts(Stmt *S, std::string buf);
     Stmt *RewriteObjCTryStmt(ObjCAtTryStmt *S);
     Stmt *RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S);
-    Stmt *RewriteObjCCatchStmt(ObjCAtCatchStmt *S);
-    Stmt *RewriteObjCFinallyStmt(ObjCAtFinallyStmt *S);
     Stmt *RewriteObjCThrowStmt(ObjCAtThrowStmt *S);
     Stmt *RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
                                        SourceLocation OrigEnd);
@@ -343,17 +330,17 @@ namespace {
     void RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
                                     MethodIterator MethodEnd,
                                     bool IsInstanceMethod,
-                                    const char *prefix,
-                                    const char *ClassName,
+                                    llvm::StringRef prefix,
+                                    llvm::StringRef ClassName,
                                     std::string &Result);
 
     void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
-                                     const char *prefix,
-                                     const char *ClassName,
+                                     llvm::StringRef prefix,
+                                     llvm::StringRef ClassName,
                                      std::string &Result);
     void RewriteObjCProtocolListMetaData(const ObjCList<ObjCProtocolDecl> &Prots,
-                                         const char *prefix,
-                                         const char *ClassName,
+                                         llvm::StringRef prefix,
+                                         llvm::StringRef ClassName,
                                          std::string &Result);
     void SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
                                       std::string &Result);
@@ -371,7 +358,6 @@ namespace {
     void InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD);
 
     // Block specific rewrite rules.
-    void RewriteBlockCall(CallExpr *Exp);
     void RewriteBlockPointerDecl(NamedDecl *VD);
     void RewriteByRefVar(VarDecl *VD);
     std::string SynthesizeByrefCopyDestroyHelper(VarDecl *VD, int flag);
@@ -380,18 +366,18 @@ namespace {
     void RewriteBlockPointerFunctionArgs(FunctionDecl *FD);
 
     std::string SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                      const char *funcName, std::string Tag);
+                                      llvm::StringRef funcName, std::string Tag);
     std::string SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                      const char *funcName, std::string Tag);
+                                      llvm::StringRef funcName, std::string Tag);
     std::string SynthesizeBlockImpl(BlockExpr *CE, 
                                     std::string Tag, std::string Desc);
     std::string SynthesizeBlockDescriptor(std::string DescTag, 
                                           std::string ImplTag,
-                                          int i, const char *funcName,
+                                          int i, llvm::StringRef funcName,
                                           unsigned hasCopy);
     Stmt *SynthesizeBlockCall(CallExpr *Exp, const Expr* BlockExp);
     void SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                 const char *FunName);
+                                 llvm::StringRef FunName);
     void RewriteRecordBody(RecordDecl *RD);
 
     void CollectBlockDeclRefInfo(BlockExpr *Exp);
@@ -441,7 +427,7 @@ namespace {
                             const char *&RParen);
     void RewriteCastExpr(CStyleCastExpr *CE);
 
-    FunctionDecl *SynthBlockInitFunctionDecl(const char *name);
+    FunctionDecl *SynthBlockInitFunctionDecl(llvm::StringRef name);
     Stmt *SynthBlockInitExpr(BlockExpr *Exp,
             const llvm::SmallVector<BlockDeclRefExpr *, 8> &InnerBlockDeclRefs);
 
@@ -457,10 +443,10 @@ namespace {
 
   // Helper function: create a CStyleCastExpr with trivial type source info.
   CStyleCastExpr* NoTypeInfoCStyleCastExpr(ASTContext *Ctx, QualType Ty,
-                                           CastExpr::CastKind Kind, Expr *E) {
+                                           CastKind Kind, Expr *E) {
     TypeSourceInfo *TInfo = Ctx->getTrivialTypeSourceInfo(Ty, SourceLocation());
-    return new (Ctx) CStyleCastExpr(Ty, Kind, E, CXXBaseSpecifierArray(), TInfo,
-                                    SourceLocation(), SourceLocation());
+    return CStyleCastExpr::Create(*Ctx, Ty, Kind, E, 0, TInfo,
+                                  SourceLocation(), SourceLocation());
   }
 }
 
@@ -692,7 +678,7 @@ void RewriteObjC::HandleTopLevelSingleDecl(Decl *D) {
     RewriteFunctionDecl(FD);
   } else if (VarDecl *FVD = dyn_cast<VarDecl>(D)) {
     // declared in <Foundation/NSString.h>
-    if (strcmp(FVD->getNameAsCString(), "_NSConstantStringClassReference") == 0) {
+    if (FVD->getName() == "_NSConstantStringClassReference") {
       ConstantStringClassReference = FVD;
       return;
     }
@@ -747,36 +733,6 @@ void RewriteObjC::RewriteInclude() {
   }
 }
 
-void RewriteObjC::RewriteTabs() {
-  llvm::StringRef MainBuf = SM->getBufferData(MainFileID);
-  const char *MainBufStart = MainBuf.begin();
-  const char *MainBufEnd = MainBuf.end();
-
-  // Loop over the whole file, looking for tabs.
-  for (const char *BufPtr = MainBufStart; BufPtr != MainBufEnd; ++BufPtr) {
-    if (*BufPtr != '\t')
-      continue;
-
-    // Okay, we found a tab.  This tab will turn into at least one character,
-    // but it depends on which 'virtual column' it is in.  Compute that now.
-    unsigned VCol = 0;
-    while (BufPtr-VCol != MainBufStart && BufPtr[-VCol-1] != '\t' &&
-           BufPtr[-VCol-1] != '\n' && BufPtr[-VCol-1] != '\r')
-      ++VCol;
-
-    // Okay, now that we know the virtual column, we know how many spaces to
-    // insert.  We assume 8-character tab-stops.
-    unsigned Spaces = 8-(VCol & 7);
-
-    // Get the location of the tab.
-    SourceLocation TabLoc = SM->getLocForStartOfFile(MainFileID);
-    TabLoc = TabLoc.getFileLocWithOffset(BufPtr-MainBufStart);
-
-    // Rewrite the single tab character into a sequence of spaces.
-    ReplaceText(TabLoc, 1, llvm::StringRef("        ", Spaces));
-  }
-}
-
 static std::string getIvarAccessString(ObjCInterfaceDecl *ClassDecl,
                                        ObjCIvarDecl *OID) {
   std::string S;
@@ -885,7 +841,7 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
     Setr += "objc_setProperty (self, _cmd, ";
     SynthesizeIvarOffsetComputation(ClassDecl, OID, Setr);
     Setr += ", (id)";
-    Setr += PD->getNameAsCString();
+    Setr += PD->getName();
     Setr += ", ";
     if (Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic)
       Setr += "0, ";
@@ -898,7 +854,7 @@ void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
   }
   else {
     Setr += getIvarAccessString(ClassDecl, OID) + " = ";
-    Setr += PD->getNameAsCString();
+    Setr += PD->getName();
   }
   Setr += "; }";
   InsertText(onePastSemiLoc, Setr);
@@ -1374,7 +1330,7 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
       assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
       QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
       CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
-                                                    CastExpr::CK_Unknown,
+                                                    CK_Unknown,
                                                     IV->getBase());
       // Don't forget the parens to enforce the proper binding.
       ParenExpr *PE = new (Context) ParenExpr(IV->getBase()->getLocStart(),
@@ -1419,7 +1375,7 @@ Stmt *RewriteObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV,
       assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
       QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
       CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
-                                                    CastExpr::CK_Unknown,
+                                                    CK_Unknown,
                                                     IV->getBase());
       // Don't forget the parens to enforce the proper binding.
       ParenExpr *PE = new (Context) ParenExpr(IV->getBase()->getLocStart(),
@@ -1553,7 +1509,7 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
 
   SourceLocation startLoc = S->getLocStart();
   const char *startBuf = SM->getCharacterData(startLoc);
-  const char *elementName;
+  llvm::StringRef elementName;
   std::string elementTypeAsString;
   std::string buf;
   buf = "\n{\n\t";
@@ -1569,13 +1525,13 @@ Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
       elementTypeAsString = ElementType.getAsString(Context->PrintingPolicy);
     buf += elementTypeAsString;
     buf += " ";
-    elementName = D->getNameAsCString();
+    elementName = D->getName();
     buf += elementName;
     buf += ";\n\t";
   }
   else {
     DeclRefExpr *DR = cast<DeclRefExpr>(S->getElement());
-    elementName = DR->getDecl()->getNameAsCString();
+    elementName = DR->getDecl()->getName();
     ValueDecl *VD = cast<ValueDecl>(DR->getDecl());
     if (VD->getType()->isObjCQualifiedIdType() ||
         VD->getType()->isObjCQualifiedInterfaceType())
@@ -1755,7 +1711,7 @@ Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   std::string syncBuf;
   syncBuf += " objc_sync_exit(";
   Expr *syncExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CastExpr::CK_Unknown,
+                                            CK_Unknown,
                                             S->getSynchExpr());
   std::string syncExprBufS;
   llvm::raw_string_ostream syncExprBuf(syncExprBufS);
@@ -2024,14 +1980,6 @@ Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
   return 0;
 }
 
-Stmt *RewriteObjC::RewriteObjCCatchStmt(ObjCAtCatchStmt *S) {
-  return 0;
-}
-
-Stmt *RewriteObjC::RewriteObjCFinallyStmt(ObjCAtFinallyStmt *S) {
-  return 0;
-}
-
 // This can't be done with ReplaceStmt(S, ThrowExpr), since
 // the throw expression is typically a message expression that's already
 // been rewritten! (which implies the SourceLocation's are invalid).
@@ -2106,9 +2054,8 @@ CallExpr *RewriteObjC::SynthesizeCallToFunctionDecl(
   // Now, we cast the reference to a pointer to the objc_msgSend type.
   QualType pToFunc = Context->getPointerType(msgSendType);
   ImplicitCastExpr *ICE = 
-    new (Context) ImplicitCastExpr(pToFunc, CastExpr::CK_Unknown,
-                                   DRE, CXXBaseSpecifierArray(),
-                                   /*isLvalue=*/false);
+    ImplicitCastExpr::Create(*Context, pToFunc, CK_Unknown,
+                             DRE, 0, VK_RValue);
 
   const FunctionType *FT = msgSendType->getAs<FunctionType>();
 
@@ -2318,14 +2265,14 @@ void RewriteObjC::SynthSelGetUidFunctionDecl() {
   SelGetUidFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                            SourceLocation(),
                                            SelGetUidIdent, getFuncType, 0,
-                                           FunctionDecl::Extern,
-                                           FunctionDecl::None, false);
+                                           SC_Extern,
+                                           SC_None, false);
 }
 
 void RewriteObjC::RewriteFunctionDecl(FunctionDecl *FD) {
   // declared in <objc/objc.h>
   if (FD->getIdentifier() &&
-      strcmp(FD->getNameAsCString(), "sel_registerName") == 0) {
+      FD->getName() == "sel_registerName") {
     SelGetUidFunctionDecl = FD;
     return;
   }
@@ -2385,7 +2332,7 @@ void RewriteObjC::RewriteBlockLiteralFunctionDecl(FunctionDecl *FD) {
   QualType Type = proto->getResultType();
   std::string FdStr = Type.getAsString(Context->PrintingPolicy);
   FdStr += " ";
-  FdStr += FD->getNameAsCString();
+  FdStr += FD->getName();
   FdStr +=  "(";
   unsigned numArgs = proto->getNumArgs();
   for (unsigned i = 0; i < numArgs; i++) {
@@ -2417,8 +2364,8 @@ void RewriteObjC::SynthSuperContructorFunctionDecl() {
   SuperContructorFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                          SourceLocation(),
                                          msgSendIdent, msgSendType, 0,
-                                         FunctionDecl::Extern,
-                                         FunctionDecl::None, false);
+                                         SC_Extern,
+                                         SC_None, false);
 }
 
 // SynthMsgSendFunctionDecl - id objc_msgSend(id self, SEL op, ...);
@@ -2439,8 +2386,8 @@ void RewriteObjC::SynthMsgSendFunctionDecl() {
   MsgSendFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                          SourceLocation(),
                                          msgSendIdent, msgSendType, 0,
-                                         FunctionDecl::Extern,
-                                         FunctionDecl::None, false);
+                                         SC_Extern,
+                                         SC_None, false);
 }
 
 // SynthMsgSendSuperFunctionDecl - id objc_msgSendSuper(struct objc_super *, SEL op, ...);
@@ -2464,8 +2411,8 @@ void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
   MsgSendSuperFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                               SourceLocation(),
                                               msgSendIdent, msgSendType, 0,
-                                              FunctionDecl::Extern,
-                                              FunctionDecl::None, false);
+                                              SC_Extern,
+                                              SC_None, false);
 }
 
 // SynthMsgSendStretFunctionDecl - id objc_msgSend_stret(id self, SEL op, ...);
@@ -2486,8 +2433,8 @@ void RewriteObjC::SynthMsgSendStretFunctionDecl() {
   MsgSendStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                          SourceLocation(),
                                          msgSendIdent, msgSendType, 0,
-                                         FunctionDecl::Extern,
-                                         FunctionDecl::None, false);
+                                         SC_Extern,
+                                         SC_None, false);
 }
 
 // SynthMsgSendSuperStretFunctionDecl -
@@ -2513,8 +2460,8 @@ void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
   MsgSendSuperStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                                        SourceLocation(),
                                               msgSendIdent, msgSendType, 0,
-                                              FunctionDecl::Extern,
-                                              FunctionDecl::None, false);
+                                              SC_Extern,
+                                              SC_None, false);
 }
 
 // SynthMsgSendFpretFunctionDecl - double objc_msgSend_fpret(id self, SEL op, ...);
@@ -2535,8 +2482,8 @@ void RewriteObjC::SynthMsgSendFpretFunctionDecl() {
   MsgSendFpretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                               SourceLocation(),
                                               msgSendIdent, msgSendType, 0,
-                                              FunctionDecl::Extern,
-                                              FunctionDecl::None, false);
+                                              SC_Extern,
+                                              SC_None, false);
 }
 
 // SynthGetClassFunctionDecl - id objc_getClass(const char *name);
@@ -2552,8 +2499,8 @@ void RewriteObjC::SynthGetClassFunctionDecl() {
   GetClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                           SourceLocation(),
                                           getClassIdent, getClassType, 0,
-                                          FunctionDecl::Extern,
-                                          FunctionDecl::None, false);
+                                          SC_Extern,
+                                          SC_None, false);
 }
 
 // SynthGetSuperClassFunctionDecl - Class class_getSuperclass(Class cls);
@@ -2571,8 +2518,8 @@ void RewriteObjC::SynthGetSuperClassFunctionDecl() {
                                                    SourceLocation(),
                                                    getSuperClassIdent,
                                                    getClassType, 0,
-                                                   FunctionDecl::Extern,
-                                                   FunctionDecl::None,
+                                                   SC_Extern,
+                                                   SC_None,
                                                    false);
 }
 
@@ -2589,8 +2536,8 @@ void RewriteObjC::SynthGetMetaClassFunctionDecl() {
   GetMetaClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
                                               SourceLocation(),
                                               getClassIdent, getClassType, 0,
-                                              FunctionDecl::Extern,
-                                              FunctionDecl::None, false);
+                                              SC_Extern,
+                                              SC_None, false);
 }
 
 Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
@@ -2624,25 +2571,19 @@ Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
 
   VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
                                     &Context->Idents.get(S), strType, 0,
-                                    VarDecl::Static, VarDecl::None);
+                                    SC_Static, SC_None);
   DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, strType, SourceLocation());
-  Expr *Unop = new (Context) UnaryOperator(DRE, UnaryOperator::AddrOf,
+  Expr *Unop = new (Context) UnaryOperator(DRE, UO_AddrOf,
                                  Context->getPointerType(DRE->getType()),
                                  SourceLocation());
   // cast to NSConstantString *
   CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
-                                            CastExpr::CK_Unknown, Unop);
+                                            CK_Unknown, Unop);
   ReplaceStmt(Exp, cast);
   // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
   return cast;
 }
 
-bool RewriteObjC::isSuperReceiver(Expr *recExpr) {
-  // check if we are sending a message to 'super'
-  if (!CurMethodDef || !CurMethodDef->isInstanceMethod()) return false;
-  return isa<ObjCSuperExpr>(recExpr);
-}
-
 // struct objc_super { struct objc_object *receiver; struct objc_class *super; };
 QualType RewriteObjC::getSuperStructType() {
   if (!SuperStructDecl) {
@@ -2751,7 +2692,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // set the receiver to self, the first argument to all methods.
     InitExprs.push_back(
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CastExpr::CK_Unknown,
+                               CK_Unknown,
                    new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
                                    Context->getObjCIdType(),
                                    SourceLocation()))
@@ -2772,7 +2713,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // (Class)objc_getClass("CurrentClass")
     CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
                                              Context->getObjCClassType(),
-                                             CastExpr::CK_Unknown, Cls);
+                                             CK_Unknown, Cls);
     ClsExprs.clear();
     ClsExprs.push_back(ArgExpr);
     Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
@@ -2784,7 +2725,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     InitExprs.push_back( // set 'super class', using class_getSuperclass().
                         NoTypeInfoCStyleCastExpr(Context,
                                                  Context->getObjCIdType(),
-                                                 CastExpr::CK_Unknown, Cls));
+                                                 CK_Unknown, Cls));
     // struct objc_super
     QualType superType = getSuperStructType();
     Expr *SuperRep;
@@ -2803,12 +2744,12 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       // we need the cast below. For example:
       // (struct objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
       //
-      SuperRep = new (Context) UnaryOperator(SuperRep, UnaryOperator::AddrOf,
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
                                Context->getPointerType(SuperRep->getType()),
                                SourceLocation());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                           Context->getPointerType(superType),
-                                          CastExpr::CK_Unknown, SuperRep);
+                                          CK_Unknown, SuperRep);
     } else {
       // (struct objc_super) { <exprs from above> }
       InitListExpr *ILE =
@@ -2820,7 +2761,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
                                                    superType, ILE, false);
       // struct objc_super *
-      SuperRep = new (Context) UnaryOperator(SuperRep, UnaryOperator::AddrOf,
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
                                Context->getPointerType(SuperRep->getType()),
                                SourceLocation());
     }
@@ -2857,7 +2798,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
 
     InitExprs.push_back(
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CastExpr::CK_Unknown,
+                               CK_Unknown,
                    new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
                                    Context->getObjCIdType(),
                                    SourceLocation()))
@@ -2877,7 +2818,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // (Class)objc_getClass("CurrentClass")
     CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
                                                  Context->getObjCClassType(),
-                                                 CastExpr::CK_Unknown, Cls);
+                                                 CK_Unknown, Cls);
     ClsExprs.clear();
     ClsExprs.push_back(ArgExpr);
     Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
@@ -2889,7 +2830,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     InitExprs.push_back(
       // set 'super class', using class_getSuperclass().
       NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CastExpr::CK_Unknown, Cls));
+                               CK_Unknown, Cls));
     // struct objc_super
     QualType superType = getSuperStructType();
     Expr *SuperRep;
@@ -2908,12 +2849,12 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
       // we need the cast below. For example:
       // (struct objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
       //
-      SuperRep = new (Context) UnaryOperator(SuperRep, UnaryOperator::AddrOf,
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
                                Context->getPointerType(SuperRep->getType()),
                                SourceLocation());
       SuperRep = NoTypeInfoCStyleCastExpr(Context,
                                Context->getPointerType(superType),
-                               CastExpr::CK_Unknown, SuperRep);
+                               CK_Unknown, SuperRep);
     } else {
       // (struct objc_super) { <exprs from above> }
       InitListExpr *ILE =
@@ -2936,7 +2877,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
       recExpr = CE->getSubExpr();
     recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                       CastExpr::CK_Unknown, recExpr);
+                                       CK_Unknown, recExpr);
     MsgExprs.push_back(recExpr);
     break;
   }
@@ -2966,7 +2907,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                                 : ICE->getType();
       // Make sure we convert "type (^)(...)" to "type (*)(...)".
       (void)convertBlockPointerToFunctionPointer(type);
-      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CastExpr::CK_Unknown,
+      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK_Unknown,
                                           userExpr);
     }
     // Make id<P...> cast into an 'id' cast.
@@ -2975,7 +2916,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
         while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
           userExpr = CE->getSubExpr();
         userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CastExpr::CK_Unknown, userExpr);
+                                            CK_Unknown, userExpr);
       }
     }
     MsgExprs.push_back(userExpr);
@@ -3025,7 +2966,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
   // xx.m:13: note: if this code is reached, the program will abort
   cast = NoTypeInfoCStyleCastExpr(Context,
                                   Context->getPointerType(Context->VoidTy),
-                                  CastExpr::CK_Unknown, DRE);
+                                  CK_Unknown, DRE);
 
   // Now do the "normal" pointer to function cast.
   QualType castType = Context->getFunctionType(returnType,
@@ -3035,7 +2976,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                                                false, false, 0, 0,
                                                FunctionType::ExtInfo());
   castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CastExpr::CK_Unknown,
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_Unknown,
                                   cast);
 
   // Don't forget the parens to enforce the proper binding.
@@ -3058,7 +2999,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // Need to cast objc_msgSend_stret to "void *" (see above comment).
     cast = NoTypeInfoCStyleCastExpr(Context,
                                     Context->getPointerType(Context->VoidTy),
-                                    CastExpr::CK_Unknown, STDRE);
+                                    CK_Unknown, STDRE);
     // Now do the "normal" pointer to function cast.
     castType = Context->getFunctionType(returnType,
       &ArgTypes[0], ArgTypes.size(),
@@ -3066,7 +3007,7 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
                                         false, false, 0, 0,
                                         FunctionType::ExtInfo());
     castType = Context->getPointerType(castType);
-    cast = NoTypeInfoCStyleCastExpr(Context, castType, CastExpr::CK_Unknown,
+    cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_Unknown,
                                     cast);
 
     // Don't forget the parens to enforce the proper binding.
@@ -3088,19 +3029,22 @@ Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
     // is needed to decide what to do.
     unsigned IntSize =
       static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-    IntegerLiteral *limit = new (Context) IntegerLiteral(llvm::APInt(IntSize, 8),
-                                               Context->IntTy,
-                                               SourceLocation());
+    IntegerLiteral *limit = IntegerLiteral::Create(*Context,
+                                                   llvm::APInt(IntSize, 8),
+                                                   Context->IntTy,
+                                                   SourceLocation());
     BinaryOperator *lessThanExpr = new (Context) BinaryOperator(sizeofExpr, limit,
-                                                      BinaryOperator::LE,
+                                                      BO_LE,
                                                       Context->IntTy,
                                                       SourceLocation());
     // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
     ConditionalOperator *CondExpr =
       new (Context) ConditionalOperator(lessThanExpr,
                                         SourceLocation(), CE,
-                                        SourceLocation(), STCE, returnType);
-    ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), CondExpr);
+                                        SourceLocation(), STCE, (Expr*)0,
+                                        returnType);
+    ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
+                                            CondExpr);
   }
   // delete Exp; leak for now, see RewritePropertySetter() usage for more info.
   return ReplacingStmt;
@@ -3139,13 +3083,13 @@ Stmt *RewriteObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
   IdentifierInfo *ID = &Context->Idents.get(Name);
   VarDecl *VD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
                                 ID, getProtocolType(), 0,
-                                VarDecl::Extern, VarDecl::None);
+                                SC_Extern, SC_None);
   DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, getProtocolType(), SourceLocation());
-  Expr *DerefExpr = new (Context) UnaryOperator(DRE, UnaryOperator::AddrOf,
+  Expr *DerefExpr = new (Context) UnaryOperator(DRE, UO_AddrOf,
                              Context->getPointerType(DRE->getType()),
                              SourceLocation());
   CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
-                                                CastExpr::CK_Unknown,
+                                                CK_Unknown,
                                                 DerefExpr);
   ReplaceStmt(Exp, castExpr);
   ProtocolExprDecls.insert(Exp->getProtocol());
@@ -3185,7 +3129,7 @@ bool RewriteObjC::BufferContainsPPDirectives(const char *startBuf,
 void RewriteObjC::SynthesizeObjCInternalStruct(ObjCInterfaceDecl *CDecl,
                                                std::string &Result) {
   assert(CDecl && "Class missing in SynthesizeObjCInternalStruct");
-  assert(CDecl->getNameAsCString() &&
+  assert(CDecl->getName() != "" &&
          "Name missing in SynthesizeObjCInternalStruct");
   // Do not synthesize more than once.
   if (ObjCSynthesizedStructs.count(CDecl))
@@ -3318,8 +3262,8 @@ template<typename MethodIterator>
 void RewriteObjC::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
                                              MethodIterator MethodEnd,
                                              bool IsInstanceMethod,
-                                             const char *prefix,
-                                             const char *ClassName,
+                                             llvm::StringRef prefix,
+                                             llvm::StringRef ClassName,
                                              std::string &Result) {
   if (MethodBegin == MethodEnd) return;
 
@@ -3388,8 +3332,8 @@ void RewriteObjC::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
 
 /// RewriteObjCProtocolMetaData - Rewrite protocols meta-data.
 void RewriteObjC::
-RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, const char *prefix,
-                            const char *ClassName, std::string &Result) {
+RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, llvm::StringRef prefix,
+                            llvm::StringRef ClassName, std::string &Result) {
   static bool objc_protocol_methods = false;
 
   // Output struct protocol_methods holder of method selector and type.
@@ -3435,7 +3379,7 @@ RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, const char *prefix,
         Result += "\t  ,{{(struct objc_selector *)\"";
       else
         Result += "\t  ,{(struct objc_selector *)\"";
-      Result += (*I)->getSelector().getAsString().c_str();
+      Result += (*I)->getSelector().getAsString();
       std::string MethodTypeString;
       Context->getObjCEncodingForMethodDecl((*I), MethodTypeString);
       Result += "\", \"";
@@ -3473,7 +3417,7 @@ RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, const char *prefix,
         Result += "\t  ,{{(struct objc_selector *)\"";
       else
         Result += "\t  ,{(struct objc_selector *)\"";
-      Result += (*I)->getSelector().getAsString().c_str();
+      Result += (*I)->getSelector().getAsString();
       std::string MethodTypeString;
       Context->getObjCEncodingForMethodDecl((*I), MethodTypeString);
       Result += "\", \"";
@@ -3536,7 +3480,7 @@ RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, const char *prefix,
 
 void RewriteObjC::
 RewriteObjCProtocolListMetaData(const ObjCList<ObjCProtocolDecl> &Protocols,
-                                const char *prefix, const char *ClassName,
+                                llvm::StringRef prefix, llvm::StringRef ClassName,
                                 std::string &Result) {
   if (Protocols.empty()) return;
 
@@ -3629,7 +3573,7 @@ void RewriteObjC::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
   // Null CDecl is case of a category implementation with no category interface
   if (CDecl)
     RewriteObjCProtocolListMetaData(CDecl->getReferencedProtocols(), "CATEGORY",
-                                    FullCategoryName.c_str(), Result);
+                                    FullCategoryName, Result);
   /* struct _objc_category {
    char *category_name;
    char *class_name;
@@ -3827,15 +3771,15 @@ void RewriteObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
       InstanceMethods.push_back(Setter);
   }
   RewriteObjCMethodsMetaData(InstanceMethods.begin(), InstanceMethods.end(),
-                             true, "", IDecl->getNameAsCString(), Result);
+                             true, "", IDecl->getName(), Result);
 
   // Build _objc_method_list for class's class methods if needed
   RewriteObjCMethodsMetaData(IDecl->classmeth_begin(), IDecl->classmeth_end(),
-                             false, "", IDecl->getNameAsCString(), Result);
+                             false, "", IDecl->getName(), Result);
 
   // Protocols referenced in class declaration?
   RewriteObjCProtocolListMetaData(CDecl->getReferencedProtocols(),
-                                  "CLASS", CDecl->getNameAsCString(), Result);
+                                  "CLASS", CDecl->getName(), Result);
 
   // Declaration of class/meta-class metadata
   /* struct _objc_class {
@@ -4101,13 +4045,13 @@ static bool HasLocalVariableExternalStorage(ValueDecl *VD) {
 }
 
 std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                                   const char *funcName,
+                                                   llvm::StringRef funcName,
                                                    std::string Tag) {
   const FunctionType *AFT = CE->getFunctionType();
   QualType RT = AFT->getResultType();
   std::string StructRef = "struct " + Tag;
   std::string S = "static " + RT.getAsString(Context->PrintingPolicy) + " __" +
-                  funcName + "_" + "block_func_" + utostr(i);
+                  funcName.str() + "_" + "block_func_" + utostr(i);
 
   BlockDecl *BD = CE->getBlockDecl();
 
@@ -4195,7 +4139,7 @@ std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
 }
 
 std::string RewriteObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                                   const char *funcName,
+                                                   llvm::StringRef funcName,
                                                    std::string Tag) {
   std::string StructRef = "struct " + Tag;
   std::string S = "static void __";
@@ -4309,37 +4253,48 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
       S += FieldName + "; // by ref\n";
     }
     // Finish writing the constructor.
-    Constructor += ", int flags=0) {\n";
-    if (GlobalVarDecl)
-      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
-    else
-      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
-    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
-
-    Constructor += "    Desc = desc;\n";
-
+    Constructor += ", int flags=0)";
     // Initialize all "by copy" arguments.
+    bool firsTime = true;
     for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
          E = BlockByCopyDecls.end(); I != E; ++I) {
       std::string Name = (*I)->getNameAsString();
-      Constructor += "    ";
-      if (isTopLevelBlockPointerType((*I)->getType()))
-        Constructor += Name + " = (struct __block_impl *)_";
-      else
-        Constructor += Name + " = _";
-      Constructor += Name + ";\n";
+        if (firsTime) {
+          Constructor += " : ";
+          firsTime = false;
+        }
+        else
+          Constructor += ", ";
+        if (isTopLevelBlockPointerType((*I)->getType()))
+          Constructor += Name + "((struct __block_impl *)_" + Name + ")";
+        else
+          Constructor += Name + "(_" + Name + ")";
     }
     // Initialize all "by ref" arguments.
     for (llvm::SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
          E = BlockByRefDecls.end(); I != E; ++I) {
       std::string Name = (*I)->getNameAsString();
-      Constructor += "    ";
+      if (firsTime) {
+        Constructor += " : ";
+        firsTime = false;
+      }
+      else
+        Constructor += ", ";
       if (isTopLevelBlockPointerType((*I)->getType()))
-        Constructor += Name + " = (struct __block_impl *)_";
+        Constructor += Name + "((struct __block_impl *)_" 
+                        + Name + "->__forwarding)";
       else
-        Constructor += Name + " = _";
-      Constructor += Name + "->__forwarding;\n";
+        Constructor += Name + "(_" + Name + "->__forwarding)";
     }
+    
+    Constructor += " {\n";
+    if (GlobalVarDecl)
+      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
+    else
+      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
+    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
+
+    Constructor += "    Desc = desc;\n";
   } else {
     // Finish writing the constructor.
     Constructor += ", int flags=0) {\n";
@@ -4359,7 +4314,7 @@ std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag,
 
 std::string RewriteObjC::SynthesizeBlockDescriptor(std::string DescTag, 
                                                    std::string ImplTag, int i,
-                                                   const char *FunName,
+                                                   llvm::StringRef FunName,
                                                    unsigned hasCopy) {
   std::string S = "\nstatic struct " + DescTag;
   
@@ -4378,21 +4333,21 @@ std::string RewriteObjC::SynthesizeBlockDescriptor(std::string DescTag,
   S += DescTag + "_DATA = { 0, sizeof(struct ";
   S += ImplTag + ")";
   if (hasCopy) {
-    S += ", __" + std::string(FunName) + "_block_copy_" + utostr(i);
-    S += ", __" + std::string(FunName) + "_block_dispose_" + utostr(i);
+    S += ", __" + FunName.str() + "_block_copy_" + utostr(i);
+    S += ", __" + FunName.str() + "_block_dispose_" + utostr(i);
   }
   S += "};\n";
   return S;
 }
 
 void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                          const char *FunName) {
+                                          llvm::StringRef FunName) {
   // Insert declaration for the function in which block literal is used.
   if (CurFunctionDeclToDeclareForBlock && !Blocks.empty())
     RewriteBlockLiteralFunctionDecl(CurFunctionDeclToDeclareForBlock);
   bool RewriteSC = (GlobalVarDecl &&
                     !Blocks.empty() &&
-                    GlobalVarDecl->getStorageClass() == VarDecl::Static &&
+                    GlobalVarDecl->getStorageClass() == SC_Static &&
                     GlobalVarDecl->getType().getCVRQualifiers());
   if (RewriteSC) {
     std::string SC(" void __");
@@ -4420,8 +4375,8 @@ void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
       }
     }
 
-    std::string ImplTag = "__" + std::string(FunName) + "_block_impl_" + utostr(i);
-    std::string DescTag = "__" + std::string(FunName) + "_block_desc_" + utostr(i);
+    std::string ImplTag = "__" + FunName.str() + "_block_impl_" + utostr(i);
+    std::string DescTag = "__" + FunName.str() + "_block_desc_" + utostr(i);
 
     std::string CI = SynthesizeBlockImpl(Blocks[i], ImplTag, DescTag);
 
@@ -4450,7 +4405,7 @@ void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
     // Must insert any 'const/volatile/static here. Since it has been
     // removed as result of rewriting of block literals.
     std::string SC;
-    if (GlobalVarDecl->getStorageClass() == VarDecl::Static)
+    if (GlobalVarDecl->getStorageClass() == SC_Static)
       SC = "static ";
     if (GlobalVarDecl->getType().isConstQualified())
       SC += "const ";
@@ -4469,7 +4424,7 @@ void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
 
 void RewriteObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
   SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
-  const char *FuncName = FD->getNameAsCString();
+  llvm::StringRef FuncName = FD->getName();
 
   SynthesizeBlockLiterals(FunLocStart, FuncName);
 }
@@ -4477,7 +4432,7 @@ void RewriteObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
 static void BuildUniqueMethodName(std::string &Name,
                                   ObjCMethodDecl *MD) {
   ObjCInterfaceDecl *IFace = MD->getClassInterface();
-  Name = IFace->getNameAsCString();
+  Name = IFace->getName();
   Name += "__" + MD->getSelector().getAsString();
   // Convert colons to underscores.
   std::string::size_type loc = 0;
@@ -4491,7 +4446,7 @@ void RewriteObjC::InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD) {
   SourceLocation FunLocStart = MD->getLocStart();
   std::string FuncName;
   BuildUniqueMethodName(FuncName, MD);
-  SynthesizeBlockLiterals(FunLocStart, FuncName.c_str());
+  SynthesizeBlockLiterals(FunLocStart, FuncName);
 }
 
 void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
@@ -4613,7 +4568,8 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
     ConditionalOperator *CondExpr =
       new (Context) ConditionalOperator(CONDExp,
                                       SourceLocation(), cast<Expr>(LHSStmt),
-                                      SourceLocation(), cast<Expr>(RHSStmt), 
+                                      SourceLocation(), cast<Expr>(RHSStmt),
+                                      (Expr*)0,
                                       Exp->getType());
     return CondExpr;
   } else if (const ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(BlockExp)) {
@@ -4655,7 +4611,7 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
   PtrToFuncCastType = Context->getPointerType(PtrToFuncCastType);
 
   CastExpr *BlkCast = NoTypeInfoCStyleCastExpr(Context, PtrBlock,
-                                               CastExpr::CK_Unknown,
+                                               CK_Unknown,
                                                const_cast<Expr*>(BlockExp));
   // Don't forget the parens to enforce the proper binding.
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
@@ -4669,7 +4625,7 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
                                             FD->getType());
 
   CastExpr *FunkCast = NoTypeInfoCStyleCastExpr(Context, PtrToFuncCastType,
-                                                CastExpr::CK_Unknown, ME);
+                                                CK_Unknown, ME);
   PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
 
   llvm::SmallVector<Expr*, 8> BlkExprs;
@@ -4686,11 +4642,6 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
   return CE;
 }
 
-void RewriteObjC::RewriteBlockCall(CallExpr *Exp) {
-  Stmt *BlockCall = SynthesizeBlockCall(Exp, Exp->getCallee());
-  ReplaceStmt(Exp, BlockCall);
-}
-
 // We need to return the rewritten expression to handle cases where the
 // BlockDeclRefExpr is embedded in another expression being rewritten.
 // For example:
@@ -4724,7 +4675,7 @@ Stmt *RewriteObjC::RewriteBlockDeclRefExpr(Expr *DeclRefExp) {
                                             FD, SourceLocation(),
                                             FD->getType());
 
-  const char *Name = VD->getNameAsCString();
+  llvm::StringRef Name = VD->getName();
   FD = FieldDecl::Create(*Context, 0, SourceLocation(),
                          &Context->Idents.get(Name), 
                          Context->VoidPtrTy, 0,
@@ -4749,7 +4700,7 @@ Stmt *RewriteObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
   if (VarDecl *Var = dyn_cast<VarDecl>(VD))
     if (!ImportedLocalExternalDecls.count(Var))
       return DRE;
-  Expr *Exp = new (Context) UnaryOperator(DRE, UnaryOperator::Deref,
+  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref,
                                     DRE->getType(), DRE->getLocation());
   // Need parens to enforce precedence.
   ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
@@ -5098,7 +5049,6 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
       startLoc = E->getLocStart();
     startLoc = SM->getInstantiationLoc(startLoc);
     endBuf = SM->getCharacterData(startLoc);
-   
     ByrefType += " " + Name;
     ByrefType += " = {(void*)";
     ByrefType += utostr(isa);
@@ -5125,11 +5075,11 @@ void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
     // 
     // double __block BYREFVAR = 1.34, BYREFVAR2 = 1.37;
     //
-    const char *startBuf = SM->getCharacterData(startLoc);
-    const char *semiBuf = strchr(startBuf, ';');
+    const char *startInitializerBuf = SM->getCharacterData(startLoc);
+    const char *semiBuf = strchr(startInitializerBuf, ';');
     assert((*semiBuf == ';') && "RewriteByRefVar: can't find ';'");
     SourceLocation semiLoc =
-      startLoc.getFileLocWithOffset(semiBuf-startBuf);
+      startLoc.getFileLocWithOffset(semiBuf-startInitializerBuf);
 
     InsertText(semiLoc, "}");
   }
@@ -5165,12 +5115,12 @@ void RewriteObjC::CollectBlockDeclRefInfo(BlockExpr *Exp) {
   }
 }
 
-FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(const char *name) {
+FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(llvm::StringRef name) {
   IdentifierInfo *ID = &Context->Idents.get(name);
   QualType FType = Context->getFunctionNoProtoType(Context->VoidPtrTy);
   return FunctionDecl::Create(*Context, TUDecl,SourceLocation(),
-                              ID, FType, 0, FunctionDecl::Extern,
-                              FunctionDecl::None, false, false);
+                              ID, FType, 0, SC_Extern,
+                              SC_None, false, false);
 }
 
 Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
@@ -5232,17 +5182,17 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
   Expr *NewRep;
 
   // Simulate a contructor call...
-  FD = SynthBlockInitFunctionDecl(Tag.c_str());
+  FD = SynthBlockInitFunctionDecl(Tag);
   DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, FType, SourceLocation());
 
   llvm::SmallVector<Expr*, 4> InitExprs;
 
   // Initialize the block function.
-  FD = SynthBlockInitFunctionDecl(Func.c_str());
+  FD = SynthBlockInitFunctionDecl(Func);
   DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, FD->getType(),
                                                SourceLocation());
   CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                                CastExpr::CK_Unknown, Arg);
+                                                CK_Unknown, Arg);
   InitExprs.push_back(castExpr);
 
   // Initialize the block descriptor.
@@ -5251,11 +5201,11 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
   VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(), 
                                     &Context->Idents.get(DescData.c_str()), 
                                     Context->VoidPtrTy, 0,
-                                    VarDecl::Static, VarDecl::None);
+                                    SC_Static, SC_None);
   UnaryOperator *DescRefExpr = new (Context) UnaryOperator(
                                   new (Context) DeclRefExpr(NewVD, 
                                     Context->VoidPtrTy, SourceLocation()), 
-                                  UnaryOperator::AddrOf,
+                                  UO_AddrOf,
                                   Context->getPointerType(Context->VoidPtrTy), 
                                   SourceLocation());
   InitExprs.push_back(DescRefExpr); 
@@ -5268,26 +5218,26 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
          E = BlockByCopyDecls.end(); I != E; ++I) {
       if (isObjCType((*I)->getType())) {
         // FIXME: Conform to ABI ([[obj retain] autorelease]).
-        FD = SynthBlockInitFunctionDecl((*I)->getNameAsCString());
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
         Exp = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UnaryOperator::AddrOf, QT,
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT,
                                             SourceLocation());
         }
       } else if (isTopLevelBlockPointerType((*I)->getType())) {
-        FD = SynthBlockInitFunctionDecl((*I)->getNameAsCString());
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
         Arg = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
         Exp = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                       CastExpr::CK_Unknown, Arg);
+                                       CK_Unknown, Arg);
       } else {
-        FD = SynthBlockInitFunctionDecl((*I)->getNameAsCString());
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
         Exp = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
         if (HasLocalVariableExternalStorage(*I)) {
           QualType QT = (*I)->getType();
           QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UnaryOperator::AddrOf, QT, 
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, 
                                             SourceLocation());
         }
         
@@ -5308,12 +5258,12 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
       assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
       QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
       
-      FD = SynthBlockInitFunctionDecl((*I)->getNameAsCString());
+      FD = SynthBlockInitFunctionDecl((*I)->getName());
       Exp = new (Context) DeclRefExpr(FD, FD->getType(), SourceLocation());
-      Exp = new (Context) UnaryOperator(Exp, UnaryOperator::AddrOf,
+      Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
                               Context->getPointerType(Exp->getType()),
                               SourceLocation());
-      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CastExpr::CK_Unknown, Exp);
+      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_Unknown, Exp);
       InitExprs.push_back(Exp);
     }
   }
@@ -5322,16 +5272,16 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
     int flag = (BLOCK_HAS_COPY_DISPOSE | BLOCK_HAS_DESCRIPTOR);
     unsigned IntSize = 
       static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-    Expr *FlagExp = new (Context) IntegerLiteral(llvm::APInt(IntSize, flag), 
-                                             Context->IntTy, SourceLocation());
+    Expr *FlagExp = IntegerLiteral::Create(*Context, llvm::APInt(IntSize, flag), 
+                                           Context->IntTy, SourceLocation());
     InitExprs.push_back(FlagExp);
   }
   NewRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0], InitExprs.size(),
                                   FType, SourceLocation());
-  NewRep = new (Context) UnaryOperator(NewRep, UnaryOperator::AddrOf,
+  NewRep = new (Context) UnaryOperator(NewRep, UO_AddrOf,
                              Context->getPointerType(NewRep->getType()),
                              SourceLocation());
-  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CastExpr::CK_Unknown,
+  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_Unknown,
                                     NewRep);
   BlockDeclRefs.clear();
   BlockByRefDecls.clear();
@@ -5609,7 +5559,9 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
   }
 #if 0
   if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
-    CastExpr *Replacement = new (Context) CastExpr(ICE->getType(), ICE->getSubExpr(), SourceLocation());
+    CastExpr *Replacement = new (Context) CastExpr(ICE->getType(),
+                                                   ICE->getSubExpr(),
+                                                   SourceLocation());
     // Get the new text.
     std::string SStr;
     llvm::raw_string_ostream Buf(SStr);
@@ -5722,7 +5674,7 @@ void RewriteObjC::HandleDeclInMainFile(Decl *D) {
         PropParentMap = 0;
       }
       SynthesizeBlockLiterals(VD->getTypeSpecStartLoc(),
-                              VD->getNameAsCString());
+                              VD->getName());
       GlobalVarDecl = 0;
 
       // This is needed for blocks.
diff --git a/lib/Sema/AnalysisBasedWarnings.cpp b/lib/Sema/AnalysisBasedWarnings.cpp
index 448d16116a39..cfebed6a05b2 100644
--- a/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/lib/Sema/AnalysisBasedWarnings.cpp
@@ -13,9 +13,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "AnalysisBasedWarnings.h"
+#include "clang/Sema/AnalysisBasedWarnings.h"
+#include "clang/Sema/SemaInternal.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/StmtObjC.h"
@@ -197,6 +199,8 @@ static ControlFlowKind CheckFallThrough(AnalysisContext &AC) {
   return AlwaysFallThrough;
 }
 
+namespace {
+
 struct CheckFallThroughDiagnostics {
   unsigned diag_MaybeFallThrough_HasNoReturn;
   unsigned diag_MaybeFallThrough_ReturnsNonVoid;
@@ -266,6 +270,8 @@ struct CheckFallThroughDiagnostics {
   }
 };
 
+}
+
 /// CheckFallThroughForFunctionDef - Check that we don't fall off the end of a
 /// function that should return a value.  Check that we don't fall off the end
 /// of a noreturn function.  We assume that functions and blocks not marked
@@ -375,19 +381,16 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
       S.SourceMgr.isInSystemHeader(D->getLocation()))
     return;
 
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
-    // For function templates, class templates and member function templates
-    // we'll do the analysis at instantiation time.
-    if (FD->isDependentContext())
-      return;
-  }
+  // For code in dependent contexts, we'll do this at instantiation time.
+  if (cast<DeclContext>(D)->isDependentContext())
+    return;
 
   const Stmt *Body = D->getBody();
   assert(Body);
 
   // Don't generate EH edges for CallExprs as we'd like to avoid the n^2
   // explosion for destrutors that can result and the compile time hit.
-  AnalysisContext AC(D, false);
+  AnalysisContext AC(D, 0, false);
 
   // Warning: check missing 'return'
   if (P.enableCheckFallThrough) {
@@ -401,3 +404,21 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   if (P.enableCheckUnreachable)
     CheckUnreachable(S, AC);
 }
+
+void clang::sema::
+AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
+                                     const BlockExpr *E) {
+  return IssueWarnings(P, E->getBlockDecl(), E->getType());
+}
+
+void clang::sema::
+AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
+                                     const ObjCMethodDecl *D) {
+  return IssueWarnings(P, D, QualType());
+}
+
+void clang::sema::
+AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
+                                     const FunctionDecl *D) {
+  return IssueWarnings(P, D, QualType());
+}
diff --git a/lib/Sema/AnalysisBasedWarnings.h b/lib/Sema/AnalysisBasedWarnings.h
deleted file mode 100644
index dea19ba28ccf..000000000000
--- a/lib/Sema/AnalysisBasedWarnings.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//=- AnalysisBasedWarnings.h - Sema warnings based on libAnalysis -*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines AnalysisBasedWarnings, a worker object used by Sema
-// that issues warnings based on dataflow-analysis.
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_SEMA_ANALYSIS_WARNINGS_H
-#define LLVM_CLANG_SEMA_ANALYSIS_WARNINGS_H
-
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/DenseMap.h"
-
-namespace clang {
-
-class Sema;
-
-namespace sema {
-
-class AnalysisBasedWarnings {
-public:
-  class Policy {
-    friend class AnalysisBasedWarnings;
-    // The warnings to run.
-    unsigned enableCheckFallThrough : 1;
-    unsigned enableCheckUnreachable : 1;
-  public:
-    Policy();
-    void disableCheckFallThrough() { enableCheckFallThrough = 0; }
-  };
-
-private:
-  Sema &S;
-  Policy DefaultPolicy;
-
-  enum VisitFlag { NotVisited = 0, Visited = 1, Pending = 2 };
-  llvm::DenseMap<const FunctionDecl*, VisitFlag> VisitedFD;
-
-public:
-  AnalysisBasedWarnings(Sema &s);
-
-  Policy getDefaultPolicy() { return DefaultPolicy; }
-
-  void IssueWarnings(Policy P, const Decl *D, QualType BlockTy = QualType());
-};
-
-}} // end namespace clang::sema
-
-#endif
diff --git a/lib/Parse/AttributeList.cpp b/lib/Sema/AttributeList.cpp
index 98d5d07e151e..8ccb2ca586eb 100644
--- a/lib/Parse/AttributeList.cpp
+++ b/lib/Sema/AttributeList.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Parse/AttributeList.h"
+#include "clang/Sema/AttributeList.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/StringSwitch.h"
 using namespace clang;
@@ -19,7 +19,7 @@ using namespace clang;
 AttributeList::AttributeList(IdentifierInfo *aName, SourceLocation aLoc,
                              IdentifierInfo *sName, SourceLocation sLoc,
                              IdentifierInfo *pName, SourceLocation pLoc,
-                             ActionBase::ExprTy **ExprList, unsigned numArgs,
+                             Expr **ExprList, unsigned numArgs,
                              AttributeList *n, bool declspec, bool cxx0x)
   : AttrName(aName), AttrLoc(aLoc), ScopeName(sName), ScopeLoc(sLoc),
     ParmName(pName), ParmLoc(pLoc), NumArgs(numArgs), Next(n),
@@ -28,7 +28,7 @@ AttributeList::AttributeList(IdentifierInfo *aName, SourceLocation aLoc,
   if (numArgs == 0)
     Args = 0;
   else {
-    Args = new ActionBase::ExprTy*[numArgs];
+    Args = new Expr*[numArgs];
     memcpy(Args, ExprList, numArgs*sizeof(Args[0]));
   }
 }
@@ -100,6 +100,7 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("format_arg", AT_format_arg)
     .Case("gnu_inline", AT_gnu_inline)
     .Case("weak_import", AT_weak_import)
+    .Case("vecreturn", AT_vecreturn)
     .Case("vector_size", AT_vector_size)
     .Case("constructor", AT_constructor)
     .Case("unavailable", AT_unavailable)
@@ -118,13 +119,18 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo *Name) {
     .Case("ns_returns_retained", AT_ns_returns_retained)
     .Case("cf_returns_not_retained", AT_cf_returns_not_retained)
     .Case("cf_returns_retained", AT_cf_returns_retained)
+    .Case("ownership_returns", AT_ownership_returns)
+    .Case("ownership_holds", AT_ownership_holds)
+    .Case("ownership_takes", AT_ownership_takes)
     .Case("reqd_work_group_size", AT_reqd_wg_size)
     .Case("init_priority", AT_init_priority)
     .Case("no_instrument_function", AT_no_instrument_function)
     .Case("thiscall", AT_thiscall)
+    .Case("pascal", AT_pascal)
     .Case("__cdecl", AT_cdecl)
     .Case("__stdcall", AT_stdcall)
     .Case("__fastcall", AT_fastcall)
     .Case("__thiscall", AT_thiscall)
+    .Case("__pascal", AT_pascal)
     .Default(UnknownAttribute);
 }
diff --git a/lib/Sema/CMakeLists.txt b/lib/Sema/CMakeLists.txt
index 70b4792f3ba5..e65bb227f159 100644
--- a/lib/Sema/CMakeLists.txt
+++ b/lib/Sema/CMakeLists.txt
@@ -2,10 +2,11 @@ set(LLVM_NO_RTTI 1)
 
 add_clang_library(clangSema
   AnalysisBasedWarnings.cpp
+  AttributeList.cpp
   CodeCompleteConsumer.cpp
+  DeclSpec.cpp
   IdentifierResolver.cpp
   JumpDiagnostics.cpp
-  ParseAST.cpp
   Sema.cpp
   SemaAccess.cpp
   SemaAttr.cpp
diff --git a/lib/Sema/CXXFieldCollector.h b/lib/Sema/CXXFieldCollector.h
deleted file mode 100644
index 63c6ee3f74ba..000000000000
--- a/lib/Sema/CXXFieldCollector.h
+++ /dev/null
@@ -1,79 +0,0 @@
-//===- CXXFieldCollector.h - Utility class for C++ class semantic analysis ===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file provides CXXFieldCollector that is used during parsing & semantic
-//  analysis of C++ classes.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_SEMA_CXXFIELDCOLLECTOR_H
-#define LLVM_CLANG_SEMA_CXXFIELDCOLLECTOR_H
-
-#include "llvm/ADT/SmallVector.h"
-
-namespace clang {
-  class FieldDecl;
-
-/// CXXFieldCollector - Used to keep track of CXXFieldDecls during parsing of
-/// C++ classes.
-class CXXFieldCollector {
-  /// Fields - Contains all FieldDecls collected during parsing of a C++
-  /// class. When a nested class is entered, its fields are appended to the
-  /// fields of its parent class, when it is exited its fields are removed.
-  llvm::SmallVector<FieldDecl*, 32> Fields;
-
-  /// FieldCount - Each entry represents the number of fields collected during
-  /// the parsing of a C++ class. When a nested class is entered, a new field
-  /// count is pushed, when it is exited, the field count is popped.
-  llvm::SmallVector<size_t, 4> FieldCount;
-
-  // Example:
-  //
-  // class C {
-  //   int x,y;
-  //   class NC {
-  //     int q;
-  //     // At this point, Fields contains [x,y,q] decls and FieldCount contains
-  //     // [2,1].
-  //   };
-  //   int z;
-  //   // At this point, Fields contains [x,y,z] decls and FieldCount contains
-  //   // [3].
-  // };
-
-public:
-  /// StartClass - Called by Sema::ActOnStartCXXClassDef.
-  void StartClass() { FieldCount.push_back(0); }
-
-  /// Add - Called by Sema::ActOnCXXMemberDeclarator.
-  void Add(FieldDecl *D) {
-    Fields.push_back(D);
-    ++FieldCount.back();
-  }
-
-  /// getCurNumField - The number of fields added to the currently parsed class.
-  size_t getCurNumFields() const {
-    assert(!FieldCount.empty() && "no currently-parsed class");
-    return FieldCount.back();
-  }
-
-  /// getCurFields - Pointer to array of fields added to the currently parsed
-  /// class.
-  FieldDecl **getCurFields() { return &*(Fields.end() - getCurNumFields()); }
-
-  /// FinishClass - Called by Sema::ActOnFinishCXXClassDef.
-  void FinishClass() {
-    Fields.resize(Fields.size() - getCurNumFields());
-    FieldCount.pop_back();
-  }
-};
-
-} // end namespace clang
-
-#endif
diff --git a/lib/Sema/CodeCompleteConsumer.cpp b/lib/Sema/CodeCompleteConsumer.cpp
index 6cefc61f6640..58a1627b478a 100644
--- a/lib/Sema/CodeCompleteConsumer.cpp
+++ b/lib/Sema/CodeCompleteConsumer.cpp
@@ -11,11 +11,13 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/Sema/CodeCompleteConsumer.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/Sema.h"
 #include "clang/AST/DeclCXX.h"
-#include "clang/Parse/Scope.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang-c/Index.h"
-#include "Sema.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -234,8 +236,7 @@ std::string CodeCompletionString::getAsString() const {
     default: OS << C->Text; break;
     }
   }
-  OS.flush();
-  return Result;
+  return OS.str();
 }
 
 const char *CodeCompletionString::getTypedText() const {
@@ -246,8 +247,10 @@ const char *CodeCompletionString::getTypedText() const {
   return 0;
 }
 
-CodeCompletionString *CodeCompletionString::Clone() const {
-  CodeCompletionString *Result = new CodeCompletionString;
+CodeCompletionString *
+CodeCompletionString::Clone(CodeCompletionString *Result) const {
+  if (!Result)
+    Result = new CodeCompletionString;
   for (iterator C = begin(), CEnd = end(); C != CEnd; ++C)
     Result->AddChunk(C->Clone());
   return Result;
@@ -373,19 +376,19 @@ bool CodeCompletionString::Deserialize(const char *&Str, const char *StrEnd) {
   return true;
 }
 
-void CodeCompleteConsumer::Result::Destroy() {
+void CodeCompletionResult::Destroy() {
   if (Kind == RK_Pattern) {
     delete Pattern;
     Pattern = 0;
   }
 }
 
-unsigned CodeCompleteConsumer::Result::getPriorityFromDecl(NamedDecl *ND) {
+unsigned CodeCompletionResult::getPriorityFromDecl(NamedDecl *ND) {
   if (!ND)
     return CCP_Unlikely;
   
   // Context-based decisions.
-  DeclContext *DC = ND->getDeclContext()->getLookupContext();
+  DeclContext *DC = ND->getDeclContext()->getRedeclContext();
   if (DC->isFunctionOrMethod() || isa<BlockDecl>(DC))
     return CCP_LocalDeclaration;
   if (DC->isRecord() || isa<ObjCContainerDecl>(DC))
@@ -437,13 +440,16 @@ CodeCompleteConsumer::~CodeCompleteConsumer() { }
 
 void 
 PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
-                                                         Result *Results, 
+                                                 CodeCompletionContext Context,
+                                                 CodeCompletionResult *Results,
                                                          unsigned NumResults) {
+  std::stable_sort(Results, Results + NumResults);
+  
   // Print the results.
   for (unsigned I = 0; I != NumResults; ++I) {
     OS << "COMPLETION: ";
     switch (Results[I].Kind) {
-    case Result::RK_Declaration:
+    case CodeCompletionResult::RK_Declaration:
       OS << Results[I].Declaration;
       if (Results[I].Hidden)
         OS << " (Hidden)";
@@ -456,11 +462,11 @@ PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
       OS << '\n';
       break;
       
-    case Result::RK_Keyword:
+    case CodeCompletionResult::RK_Keyword:
       OS << Results[I].Keyword << '\n';
       break;
         
-    case Result::RK_Macro: {
+    case CodeCompletionResult::RK_Macro: {
       OS << Results[I].Macro->getName();
       if (CodeCompletionString *CCS 
             = Results[I].CreateCodeCompletionString(SemaRef)) {
@@ -471,7 +477,7 @@ PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
       break;
     }
         
-    case Result::RK_Pattern: {
+    case CodeCompletionResult::RK_Pattern: {
       OS << "Pattern : " 
          << Results[I].Pattern->getAsString() << '\n';
       break;
@@ -494,116 +500,104 @@ PrintingCodeCompleteConsumer::ProcessOverloadCandidates(Sema &SemaRef,
   }
 }
 
+void CodeCompletionResult::computeCursorKindAndAvailability() {
+  switch (Kind) {
+  case RK_Declaration:
+    // Set the availability based on attributes.
+    Availability = CXAvailability_Available;      
+    if (Declaration->getAttr<UnavailableAttr>())
+      Availability = CXAvailability_NotAvailable;
+    else if (Declaration->getAttr<DeprecatedAttr>())
+      Availability = CXAvailability_Deprecated;
+      
+    if (FunctionDecl *Function = dyn_cast<FunctionDecl>(Declaration))
+      if (Function->isDeleted())
+        Availability = CXAvailability_NotAvailable;
+      
+    CursorKind = getCursorKindForDecl(Declaration);
+    if (CursorKind == CXCursor_UnexposedDecl)
+      CursorKind = CXCursor_NotImplemented;
+    break;
+
+  case RK_Macro:
+    Availability = CXAvailability_Available;      
+    CursorKind = CXCursor_MacroDefinition;
+    break;
+      
+  case RK_Keyword:
+    Availability = CXAvailability_Available;      
+    CursorKind = CXCursor_NotImplemented;
+    break;
+      
+  case RK_Pattern:
+    // Do nothing: Patterns can come with cursor kinds!
+    break;
+  }
+}
+
+/// \brief Retrieve the name that should be used to order a result.
+///
+/// If the name needs to be constructed as a string, that string will be
+/// saved into Saved and the returned StringRef will refer to it.
+static llvm::StringRef getOrderedName(const CodeCompletionResult &R,
+                                    std::string &Saved) {
+  switch (R.Kind) {
+    case CodeCompletionResult::RK_Keyword:
+      return R.Keyword;
+      
+    case CodeCompletionResult::RK_Pattern:
+      return R.Pattern->getTypedText();
+      
+    case CodeCompletionResult::RK_Macro:
+      return R.Macro->getName();
+      
+    case CodeCompletionResult::RK_Declaration:
+      // Handle declarations below.
+      break;
+  }
+  
+  DeclarationName Name = R.Declaration->getDeclName();
+  
+  // If the name is a simple identifier (by far the common case), or a
+  // zero-argument selector, just return a reference to that identifier.
+  if (IdentifierInfo *Id = Name.getAsIdentifierInfo())
+    return Id->getName();
+  if (Name.isObjCZeroArgSelector())
+    if (IdentifierInfo *Id
+        = Name.getObjCSelector().getIdentifierInfoForSlot(0))
+      return Id->getName();
+  
+  Saved = Name.getAsString();
+  return Saved;
+}
+    
+bool clang::operator<(const CodeCompletionResult &X, 
+                      const CodeCompletionResult &Y) {
+  std::string XSaved, YSaved;
+  llvm::StringRef XStr = getOrderedName(X, XSaved);
+  llvm::StringRef YStr = getOrderedName(Y, YSaved);
+  int cmp = XStr.compare_lower(YStr);
+  if (cmp)
+    return cmp < 0;
+  
+  // If case-insensitive comparison fails, try case-sensitive comparison.
+  cmp = XStr.compare(YStr);
+  if (cmp)
+    return cmp < 0;
+  
+  return false;
+}
+
 void 
 CIndexCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &SemaRef,
-                                                       Result *Results, 
+                                                 CodeCompletionContext Context,
+                                                 CodeCompletionResult *Results,
                                                        unsigned NumResults) {
   // Print the results.
   for (unsigned I = 0; I != NumResults; ++I) {
-    CXCursorKind Kind = CXCursor_NotImplemented;
-
-    switch (Results[I].Kind) {
-    case Result::RK_Declaration:
-      switch (Results[I].Declaration->getKind()) {
-      case Decl::Record:
-      case Decl::CXXRecord:
-      case Decl::ClassTemplateSpecialization: {
-        RecordDecl *Record = cast<RecordDecl>(Results[I].Declaration);
-        if (Record->isStruct())
-          Kind = CXCursor_StructDecl;
-        else if (Record->isUnion())
-          Kind = CXCursor_UnionDecl;
-        else
-          Kind = CXCursor_ClassDecl;
-        break;
-      }
-        
-      case Decl::ObjCMethod: {
-        ObjCMethodDecl *Method = cast<ObjCMethodDecl>(Results[I].Declaration);
-        if (Method->isInstanceMethod())
-            Kind = CXCursor_ObjCInstanceMethodDecl;
-        else
-          Kind = CXCursor_ObjCClassMethodDecl;
-        break;
-      }
-        
-      case Decl::Typedef:
-        Kind = CXCursor_TypedefDecl;
-        break;
-        
-      case Decl::Enum:
-        Kind = CXCursor_EnumDecl;
-        break;
-        
-      case Decl::Field:
-        Kind = CXCursor_FieldDecl;
-        break;
-        
-      case Decl::EnumConstant:
-        Kind = CXCursor_EnumConstantDecl;
-        break;
-        
-      case Decl::Function:
-      case Decl::CXXMethod:
-      case Decl::CXXConstructor:
-      case Decl::CXXDestructor:
-      case Decl::CXXConversion:
-        Kind = CXCursor_FunctionDecl;
-        break;
-        
-      case Decl::Var:
-        Kind = CXCursor_VarDecl;
-        break;
-        
-      case Decl::ParmVar:
-        Kind = CXCursor_ParmDecl;
-        break;
-        
-      case Decl::ObjCInterface:
-        Kind = CXCursor_ObjCInterfaceDecl;
-        break;
-        
-      case Decl::ObjCCategory:
-        Kind = CXCursor_ObjCCategoryDecl;
-        break;
-        
-      case Decl::ObjCProtocol:
-        Kind = CXCursor_ObjCProtocolDecl;
-        break;
-        
-      case Decl::ObjCProperty:
-        Kind = CXCursor_ObjCPropertyDecl;
-        break;
-        
-      case Decl::ObjCIvar:
-        Kind = CXCursor_ObjCIvarDecl;
-        break;
-        
-      case Decl::ObjCImplementation:
-        Kind = CXCursor_ObjCImplementationDecl;
-        break;
-        
-      case Decl::ObjCCategoryImpl:
-        Kind = CXCursor_ObjCCategoryImplDecl;
-        break;
-        
-      default:
-        break;
-      }
-      break;
-
-    case Result::RK_Macro:
-      Kind = CXCursor_MacroDefinition;
-      break;
-
-    case Result::RK_Keyword:
-    case Result::RK_Pattern:
-      Kind = CXCursor_NotImplemented;
-      break;
-    }
-
-    WriteUnsigned(OS, Kind);
+    WriteUnsigned(OS, Results[I].CursorKind);
     WriteUnsigned(OS, Results[I].Priority);
+    WriteUnsigned(OS, Results[I].Availability);
     CodeCompletionString *CCS = Results[I].CreateCodeCompletionString(SemaRef);
     assert(CCS && "No code-completion string?");
     CCS->Serialize(OS);
@@ -618,7 +612,8 @@ CIndexCodeCompleteConsumer::ProcessOverloadCandidates(Sema &SemaRef,
                                                        unsigned NumCandidates) {
   for (unsigned I = 0; I != NumCandidates; ++I) {
     WriteUnsigned(OS, CXCursor_NotImplemented);
-    WriteUnsigned(OS, /*Priority=*/0);
+    WriteUnsigned(OS, /*Priority=*/I);
+    WriteUnsigned(OS, /*Availability=*/CXAvailability_Available);
     CodeCompletionString *CCS
       = Candidates[I].CreateSignatureString(CurrentArg, SemaRef);
     assert(CCS && "No code-completion string?");
diff --git a/lib/Parse/DeclSpec.cpp b/lib/Sema/DeclSpec.cpp
index d2cd74418af0..b46e8af9db86 100644
--- a/lib/Parse/DeclSpec.cpp
+++ b/lib/Sema/DeclSpec.cpp
@@ -11,9 +11,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/Template.h"
+#include "clang/Parse/ParseDiagnostic.h" // FIXME: remove this back-dependency!
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/STLExtras.h"
@@ -54,7 +54,7 @@ DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto, bool isVariadic,
                                              bool hasExceptionSpec,
                                              SourceLocation ThrowLoc,
                                              bool hasAnyExceptionSpec,
-                                             ActionBase::TypeTy **Exceptions,
+                                             ParsedType *Exceptions,
                                              SourceRange *ExceptionRanges,
                                              unsigned NumExceptions,
                                              SourceLocation LPLoc,
@@ -219,8 +219,15 @@ const char *DeclSpec::getSpecifierName(TQ T) {
 bool DeclSpec::SetStorageClassSpec(SCS S, SourceLocation Loc,
                                    const char *&PrevSpec,
                                    unsigned &DiagID) {
-  if (StorageClassSpec != SCS_unspecified)
-    return BadSpecifier(S, (SCS)StorageClassSpec, PrevSpec, DiagID);
+  if (StorageClassSpec != SCS_unspecified) {
+    // Changing storage class is allowed only if the previous one
+    // was the 'extern' that is part of a linkage specification and
+    // the new storage class is 'typedef'.
+    if (!(SCS_extern_in_linkage_spec &&
+          StorageClassSpec == SCS_extern &&
+          S == SCS_typedef))
+      return BadSpecifier(S, (SCS)StorageClassSpec, PrevSpec, DiagID);
+  }
   StorageClassSpec = S;
   StorageClassSpecLoc = Loc;
   assert((unsigned)S == StorageClassSpec && "SCS constants overflow bitfield");
@@ -240,7 +247,6 @@ bool DeclSpec::SetStorageClassSpecThread(SourceLocation Loc,
   return false;
 }
 
-
 /// These methods set the specified attribute of the DeclSpec, but return true
 /// and ignore the request if invalid (e.g. "extern" then "auto" is
 /// specified).
@@ -285,7 +291,63 @@ bool DeclSpec::SetTypeSpecSign(TSS S, SourceLocation Loc,
 bool DeclSpec::SetTypeSpecType(TST T, SourceLocation Loc,
                                const char *&PrevSpec,
                                unsigned &DiagID,
-                               void *Rep, bool Owned) {
+                               ParsedType Rep) {
+  assert(isTypeRep(T) && "T does not store a type");
+  assert(Rep && "no type provided!");
+  if (TypeSpecType != TST_unspecified) {
+    PrevSpec = DeclSpec::getSpecifierName((TST) TypeSpecType);
+    DiagID = diag::err_invalid_decl_spec_combination;
+    return true;
+  }
+  TypeSpecType = T;
+  TypeRep = Rep;
+  TSTLoc = Loc;
+  TypeSpecOwned = false;
+  return false;
+}
+
+bool DeclSpec::SetTypeSpecType(TST T, SourceLocation Loc,
+                               const char *&PrevSpec,
+                               unsigned &DiagID,
+                               Expr *Rep) {
+  assert(isExprRep(T) && "T does not store an expr");
+  assert(Rep && "no expression provided!");
+  if (TypeSpecType != TST_unspecified) {
+    PrevSpec = DeclSpec::getSpecifierName((TST) TypeSpecType);
+    DiagID = diag::err_invalid_decl_spec_combination;
+    return true;
+  }
+  TypeSpecType = T;
+  ExprRep = Rep;
+  TSTLoc = Loc;
+  TypeSpecOwned = false;
+  return false;
+}
+
+bool DeclSpec::SetTypeSpecType(TST T, SourceLocation Loc,
+                               const char *&PrevSpec,
+                               unsigned &DiagID,
+                               Decl *Rep, bool Owned) {
+  assert(isDeclRep(T) && "T does not store a decl");
+  // Unlike the other cases, we don't assert that we actually get a decl.
+
+  if (TypeSpecType != TST_unspecified) {
+    PrevSpec = DeclSpec::getSpecifierName((TST) TypeSpecType);
+    DiagID = diag::err_invalid_decl_spec_combination;
+    return true;
+  }
+  TypeSpecType = T;
+  DeclRep = Rep;
+  TSTLoc = Loc;
+  TypeSpecOwned = Owned;
+  return false;
+}
+
+bool DeclSpec::SetTypeSpecType(TST T, SourceLocation Loc,
+                               const char *&PrevSpec,
+                               unsigned &DiagID) {
+  assert(!isDeclRep(T) && !isTypeRep(T) && !isExprRep(T) &&
+         "rep required for these type-spec kinds!");
   if (TypeSpecType != TST_unspecified) {
     PrevSpec = DeclSpec::getSpecifierName((TST) TypeSpecType);
     DiagID = diag::err_invalid_decl_spec_combination;
@@ -297,9 +359,8 @@ bool DeclSpec::SetTypeSpecType(TST T, SourceLocation Loc,
     return false;
   }
   TypeSpecType = T;
-  TypeRep = Rep;
   TSTLoc = Loc;
-  TypeSpecOwned = Owned;
+  TypeSpecOwned = false;
   if (TypeAltiVecVector && !TypeAltiVecBool && (TypeSpecType == TST_double)) {
     PrevSpec = DeclSpec::getSpecifierName((TST) TypeSpecType);
     DiagID = diag::err_invalid_vector_decl_spec;
@@ -335,7 +396,7 @@ bool DeclSpec::SetTypeAltiVecPixel(bool isAltiVecPixel, SourceLocation Loc,
 
 bool DeclSpec::SetTypeSpecError() {
   TypeSpecType = TST_error;
-  TypeRep = 0;
+  TypeSpecOwned = false;
   TSTLoc = SourceLocation();
   return false;
 }
@@ -401,14 +462,14 @@ bool DeclSpec::SetConstexprSpec(SourceLocation Loc, const char *&PrevSpec,
   return false;
 }
 
-void DeclSpec::setProtocolQualifiers(const ActionBase::DeclPtrTy *Protos,
+void DeclSpec::setProtocolQualifiers(Decl * const *Protos,
                                      unsigned NP,
                                      SourceLocation *ProtoLocs,
                                      SourceLocation LAngleLoc) {
   if (NP == 0) return;
-  ProtocolQualifiers = new ActionBase::DeclPtrTy[NP];
+  ProtocolQualifiers = new Decl*[NP];
   ProtocolLocs = new SourceLocation[NP];
-  memcpy((void*)ProtocolQualifiers, Protos, sizeof(ActionBase::DeclPtrTy)*NP);
+  memcpy((void*)ProtocolQualifiers, Protos, sizeof(Decl*)*NP);
   memcpy(ProtocolLocs, ProtoLocs, sizeof(SourceLocation)*NP);
   NumProtocolQualifiers = NP;
   ProtocolLAngleLoc = LAngleLoc;
@@ -430,6 +491,15 @@ void DeclSpec::SaveWrittenBuiltinSpecs() {
   }
 }
 
+void DeclSpec::SaveStorageSpecifierAsWritten() {
+  if (SCS_extern_in_linkage_spec && StorageClassSpec == SCS_extern)
+    // If 'extern' is part of a linkage specification,
+    // then it is not a storage class "as written".
+    StorageClassSpecAsWritten = SCS_unspecified;
+  else
+    StorageClassSpecAsWritten = StorageClassSpec;
+}
+
 /// Finish - This does final analysis of the declspec, rejecting things like
 /// "_Imaginary" (lacking an FP type).  This returns a diagnostic to issue or
 /// diag::NUM_DIAGNOSTICS if there is no error.  After calling this method,
@@ -475,6 +545,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
       TypeSpecType = TST_int;
       TypeSpecSign = TSS_unsigned;
       TypeSpecWidth = TSW_short;
+      TypeSpecOwned = false;
     }
   }
 
@@ -504,6 +575,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
                                       : diag::err_invalid_longlong_spec)
         <<  getSpecifierName((TST)TypeSpecType);
       TypeSpecType = TST_int;
+      TypeSpecOwned = false;
     }
     break;
   case TSW_long:  // long double, long int
@@ -513,6 +585,7 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
       Diag(D, TSWLoc, SrcMgr, diag::err_invalid_long_spec)
         << getSpecifierName((TST)TypeSpecType);
       TypeSpecType = TST_int;
+      TypeSpecOwned = false;
     }
     break;
   }
@@ -553,6 +626,8 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
     ClearStorageClassSpecs();
   }
 
+  assert(!TypeSpecOwned || isDeclRep((TST) TypeSpecType));
+
   // Okay, now we can infer the real type.
 
   // TODO: return "auto function" and other bad things based on the real type.
@@ -562,11 +637,8 @@ void DeclSpec::Finish(Diagnostic &D, Preprocessor &PP) {
 
 bool DeclSpec::isMissingDeclaratorOk() {
   TST tst = getTypeSpecType();
-  return (tst == TST_union
-       || tst == TST_struct
-       || tst == TST_class
-       || tst == TST_enum
-          ) && getTypeRep() != 0 && StorageClassSpec != DeclSpec::SCS_typedef;
+  return isDeclRep(tst) && getRepAsDecl() != 0 &&
+    StorageClassSpec != DeclSpec::SCS_typedef;
 }
 
 void UnqualifiedId::clear() {
diff --git a/lib/Sema/IdentifierResolver.cpp b/lib/Sema/IdentifierResolver.cpp
index b09526e097b6..3f16ed772352 100644
--- a/lib/Sema/IdentifierResolver.cpp
+++ b/lib/Sema/IdentifierResolver.cpp
@@ -12,7 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "IdentifierResolver.h"
+#include "clang/Sema/IdentifierResolver.h"
+#include "clang/Sema/Scope.h"
+#include "clang/AST/Decl.h"
 #include "clang/Basic/LangOptions.h"
 
 using namespace clang;
@@ -103,7 +105,7 @@ IdentifierResolver::~IdentifierResolver() {
 /// true if 'D' belongs to the given declaration context.
 bool IdentifierResolver::isDeclInScope(Decl *D, DeclContext *Ctx,
                                        ASTContext &Context, Scope *S) const {
-  Ctx = Ctx->getLookupContext();
+  Ctx = Ctx->getRedeclContext();
 
   if (Ctx->isFunctionOrMethod()) {
     // Ignore the scopes associated within transparent declaration contexts.
@@ -111,7 +113,7 @@ bool IdentifierResolver::isDeclInScope(Decl *D, DeclContext *Ctx,
            ((DeclContext *)S->getEntity())->isTransparentContext())
       S = S->getParent();
 
-    if (S->isDeclScope(Action::DeclPtrTy::make(D)))
+    if (S->isDeclScope(D))
       return true;
     if (LangOpt.CPlusPlus) {
       // C++ 3.3.2p3:
@@ -128,17 +130,20 @@ bool IdentifierResolver::isDeclInScope(Decl *D, DeclContext *Ctx,
       //
       assert(S->getParent() && "No TUScope?");
       if (S->getParent()->getFlags() & Scope::ControlScope)
-        return S->getParent()->isDeclScope(Action::DeclPtrTy::make(D));
+        return S->getParent()->isDeclScope(D);
     }
     return false;
   }
 
-  return D->getDeclContext()->getLookupContext()->Equals(Ctx);
+  return D->getDeclContext()->getRedeclContext()->Equals(Ctx);
 }
 
 /// AddDecl - Link the decl to its shadowed decl chain.
 void IdentifierResolver::AddDecl(NamedDecl *D) {
   DeclarationName Name = D->getDeclName();
+  if (IdentifierInfo *II = Name.getAsIdentifierInfo())
+    II->setIsFromAST(false);
+
   void *Ptr = Name.getFETokenInfo<void>();
 
   if (!Ptr) {
@@ -164,6 +169,9 @@ void IdentifierResolver::AddDecl(NamedDecl *D) {
 void IdentifierResolver::RemoveDecl(NamedDecl *D) {
   assert(D && "null param passed");
   DeclarationName Name = D->getDeclName();
+  if (IdentifierInfo *II = Name.getAsIdentifierInfo())
+    II->setIsFromAST(false);
+
   void *Ptr = Name.getFETokenInfo<void>();
 
   assert(Ptr && "Didn't find this decl on its identifier's chain!");
@@ -182,6 +190,9 @@ bool IdentifierResolver::ReplaceDecl(NamedDecl *Old, NamedDecl *New) {
          "Cannot replace a decl with another decl of a different name");
 
   DeclarationName Name = Old->getDeclName();
+  if (IdentifierInfo *II = Name.getAsIdentifierInfo())
+    II->setIsFromAST(false);
+
   void *Ptr = Name.getFETokenInfo<void>();
 
   if (!Ptr)
@@ -218,6 +229,7 @@ IdentifierResolver::begin(DeclarationName Name) {
 
 void IdentifierResolver::AddDeclToIdentifierChain(IdentifierInfo *II,
                                                   NamedDecl *D) {
+  II->setIsFromAST(false);
   void *Ptr = II->getFETokenInfo<void>();
 
   if (!Ptr) {
@@ -261,3 +273,16 @@ IdentifierResolver::IdDeclInfoMap::operator[](DeclarationName Name) {
   ++CurIndex;
   return *IDI;
 }
+
+void IdentifierResolver::iterator::incrementSlowCase() {
+  NamedDecl *D = **this;
+  void *InfoPtr = D->getDeclName().getFETokenInfo<void>();
+  assert(!isDeclPtr(InfoPtr) && "Decl with wrong id ?");
+  IdDeclInfo *Info = toIdDeclInfo(InfoPtr);
+
+  BaseIter I = getIterator();
+  if (I != Info->decls_begin())
+    *this = iterator(I-1);
+  else // No more decls.
+    *this = iterator();
+}
diff --git a/lib/Sema/IdentifierResolver.h b/lib/Sema/IdentifierResolver.h
deleted file mode 100644
index 59bd834073f3..000000000000
--- a/lib/Sema/IdentifierResolver.h
+++ /dev/null
@@ -1,203 +0,0 @@
-//===- IdentifierResolver.h - Lexical Scope Name lookup ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the IdentifierResolver class, which is used for lexical
-// scoped lookup, based on declaration names.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_AST_SEMA_IDENTIFIERRESOLVER_H
-#define LLVM_CLANG_AST_SEMA_IDENTIFIERRESOLVER_H
-
-#include "clang/Basic/IdentifierTable.h"
-#include "clang/Parse/Scope.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclarationName.h"
-#include "clang/AST/DeclCXX.h"
-
-namespace clang {
-
-/// IdentifierResolver - Keeps track of shadowed decls on enclosing
-/// scopes.  It manages the shadowing chains of declaration names and
-/// implements efficent decl lookup based on a declaration name.
-class IdentifierResolver {
-
-  /// IdDeclInfo - Keeps track of information about decls associated
-  /// to a particular declaration name. IdDeclInfos are lazily
-  /// constructed and assigned to a declaration name the first time a
-  /// decl with that declaration name is shadowed in some scope.
-  class IdDeclInfo {
-  public:
-    typedef llvm::SmallVector<NamedDecl*, 2> DeclsTy;
-
-    inline DeclsTy::iterator decls_begin() { return Decls.begin(); }
-    inline DeclsTy::iterator decls_end() { return Decls.end(); }
-
-    void AddDecl(NamedDecl *D) { Decls.push_back(D); }
-
-    /// RemoveDecl - Remove the decl from the scope chain.
-    /// The decl must already be part of the decl chain.
-    void RemoveDecl(NamedDecl *D);
-
-    /// Replaces the Old declaration with the New declaration. If the
-    /// replacement is successful, returns true. If the old
-    /// declaration was not found, returns false.
-    bool ReplaceDecl(NamedDecl *Old, NamedDecl *New);
-
-  private:
-    DeclsTy Decls;
-  };
-
-public:
-
-  /// iterator - Iterate over the decls of a specified declaration name.
-  /// It will walk or not the parent declaration contexts depending on how
-  /// it was instantiated.
-  class iterator {
-  public:
-    typedef NamedDecl *             value_type;
-    typedef NamedDecl *             reference;
-    typedef NamedDecl *             pointer;
-    typedef std::input_iterator_tag iterator_category;
-    typedef std::ptrdiff_t          difference_type;
-
-    /// Ptr - There are 3 forms that 'Ptr' represents:
-    /// 1) A single NamedDecl. (Ptr & 0x1 == 0)
-    /// 2) A IdDeclInfo::DeclsTy::iterator that traverses only the decls of the
-    ///    same declaration context. (Ptr & 0x3 == 0x1)
-    /// 3) A IdDeclInfo::DeclsTy::iterator that traverses the decls of parent
-    ///    declaration contexts too. (Ptr & 0x3 == 0x3)
-    uintptr_t Ptr;
-    typedef IdDeclInfo::DeclsTy::iterator BaseIter;
-
-    /// A single NamedDecl. (Ptr & 0x1 == 0)
-    iterator(NamedDecl *D) {
-      Ptr = reinterpret_cast<uintptr_t>(D);
-      assert((Ptr & 0x1) == 0 && "Invalid Ptr!");
-    }
-    /// A IdDeclInfo::DeclsTy::iterator that walks or not the parent declaration
-    /// contexts depending on 'LookInParentCtx'.
-    iterator(BaseIter I) {
-      Ptr = reinterpret_cast<uintptr_t>(I) | 0x1;
-    }
-
-    bool isIterator() const { return (Ptr & 0x1); }
-
-    BaseIter getIterator() const {
-      assert(isIterator() && "Ptr not an iterator!");
-      return reinterpret_cast<BaseIter>(Ptr & ~0x3);
-    }
-
-    friend class IdentifierResolver;
-  public:
-    iterator() : Ptr(0) {}
-
-    NamedDecl *operator*() const {
-      if (isIterator())
-        return *getIterator();
-      else
-        return reinterpret_cast<NamedDecl*>(Ptr);
-    }
-
-    bool operator==(const iterator &RHS) const {
-      return Ptr == RHS.Ptr;
-    }
-    bool operator!=(const iterator &RHS) const {
-      return Ptr != RHS.Ptr;
-    }
-
-    // Preincrement.
-    iterator& operator++() {
-      if (!isIterator()) // common case.
-        Ptr = 0;
-      else {
-        NamedDecl *D = **this;
-        void *InfoPtr = D->getDeclName().getFETokenInfo<void>();
-        assert(!isDeclPtr(InfoPtr) && "Decl with wrong id ?");
-        IdDeclInfo *Info = toIdDeclInfo(InfoPtr);
-
-        BaseIter I = getIterator();
-        if (I != Info->decls_begin())
-          *this = iterator(I-1);
-        else // No more decls.
-          *this = iterator();
-      }
-      return *this;
-    }
-
-    uintptr_t getAsOpaqueValue() const { return Ptr; }
-
-    static iterator getFromOpaqueValue(uintptr_t P) {
-      iterator Result;
-      Result.Ptr = P;
-      return Result;
-    }
-  };
-
-  /// begin - Returns an iterator for decls with the name 'Name'.
-  static iterator begin(DeclarationName Name);
-
-  /// end - Returns an iterator that has 'finished'.
-  static iterator end() {
-    return iterator();
-  }
-
-  /// isDeclInScope - If 'Ctx' is a function/method, isDeclInScope returns true
-  /// if 'D' is in Scope 'S', otherwise 'S' is ignored and isDeclInScope returns
-  /// true if 'D' belongs to the given declaration context.
-  bool isDeclInScope(Decl *D, DeclContext *Ctx, ASTContext &Context,
-                     Scope *S = 0) const;
-
-  /// AddDecl - Link the decl to its shadowed decl chain.
-  void AddDecl(NamedDecl *D);
-
-  /// RemoveDecl - Unlink the decl from its shadowed decl chain.
-  /// The decl must already be part of the decl chain.
-  void RemoveDecl(NamedDecl *D);
-
-  /// Replace the decl Old with the new declaration New on its
-  /// identifier chain. Returns true if the old declaration was found
-  /// (and, therefore, replaced).
-  bool ReplaceDecl(NamedDecl *Old, NamedDecl *New);
-
-  /// \brief Link the declaration into the chain of declarations for
-  /// the given identifier.
-  ///
-  /// This is a lower-level routine used by the PCH reader to link a
-  /// declaration into a specific IdentifierInfo before the
-  /// declaration actually has a name.
-  void AddDeclToIdentifierChain(IdentifierInfo *II, NamedDecl *D);
-
-  explicit IdentifierResolver(const LangOptions &LangOpt);
-  ~IdentifierResolver();
-
-private:
-  const LangOptions &LangOpt;
-
-  class IdDeclInfoMap;
-  IdDeclInfoMap *IdDeclInfos;
-
-  /// FETokenInfo contains a Decl pointer if lower bit == 0.
-  static inline bool isDeclPtr(void *Ptr) {
-    return (reinterpret_cast<uintptr_t>(Ptr) & 0x1) == 0;
-  }
-
-  /// FETokenInfo contains a IdDeclInfo pointer if lower bit == 1.
-  static inline IdDeclInfo *toIdDeclInfo(void *Ptr) {
-    assert((reinterpret_cast<uintptr_t>(Ptr) & 0x1) == 1
-          && "Ptr not a IdDeclInfo* !");
-    return reinterpret_cast<IdDeclInfo*>(
-                    reinterpret_cast<uintptr_t>(Ptr) & ~0x1
-                                                            );
-  }
-};
-
-} // end namespace clang
-
-#endif
diff --git a/lib/Sema/JumpDiagnostics.cpp b/lib/Sema/JumpDiagnostics.cpp
index 3431ac611886..b23f615af7b7 100644
--- a/lib/Sema/JumpDiagnostics.cpp
+++ b/lib/Sema/JumpDiagnostics.cpp
@@ -12,11 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ADT/BitVector.h"
-#include "Sema.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/StmtObjC.h"
 #include "clang/AST/StmtCXX.h"
+#include "llvm/ADT/BitVector.h"
 using namespace clang;
 
 namespace {
@@ -180,12 +181,6 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned ParentScope) {
   
   // If we found a label, remember that it is in ParentScope scope.
   switch (S->getStmtClass()) {
-  case Stmt::LabelStmtClass:
-  case Stmt::DefaultStmtClass:
-  case Stmt::CaseStmtClass:
-    LabelAndGotoScopes[S] = ParentScope;
-    break;
-
   case Stmt::AddrLabelExprClass:
     IndirectJumpTargets.push_back(cast<AddrLabelExpr>(S)->getLabel());
     break;
@@ -225,6 +220,24 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned ParentScope) {
     Stmt *SubStmt = *CI;
     if (SubStmt == 0) continue;
 
+    // Cases, labels, and defaults aren't "scope parents".  It's also
+    // important to handle these iteratively instead of recursively in
+    // order to avoid blowing out the stack.
+    while (true) {
+      Stmt *Next;
+      if (isa<CaseStmt>(SubStmt))
+        Next = cast<CaseStmt>(SubStmt)->getSubStmt();
+      else if (isa<DefaultStmt>(SubStmt))
+        Next = cast<DefaultStmt>(SubStmt)->getSubStmt();
+      else if (isa<LabelStmt>(SubStmt))
+        Next = cast<LabelStmt>(SubStmt)->getSubStmt();
+      else
+        break;
+
+      LabelAndGotoScopes[SubStmt] = ParentScope;
+      SubStmt = Next;
+    }
+
     // If this is a declstmt with a VLA definition, it defines a scope from here
     // to the end of the containing context.
     if (DeclStmt *DS = dyn_cast<DeclStmt>(SubStmt)) {
diff --git a/lib/Sema/Lookup.h b/lib/Sema/Lookup.h
deleted file mode 100644
index 271bb5bcd4a7..000000000000
--- a/lib/Sema/Lookup.h
+++ /dev/null
@@ -1,654 +0,0 @@
-//===--- Lookup.h - Classes for name lookup ---------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the LookupResult class, which is integral to
-// Sema's name-lookup subsystem.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_SEMA_LOOKUP_H
-#define LLVM_CLANG_SEMA_LOOKUP_H
-
-#include "Sema.h"
-
-namespace clang {
-
-/// @brief Represents the results of name lookup.
-///
-/// An instance of the LookupResult class captures the results of a
-/// single name lookup, which can return no result (nothing found),
-/// a single declaration, a set of overloaded functions, or an
-/// ambiguity. Use the getKind() method to determine which of these
-/// results occurred for a given lookup.
-class LookupResult {
-public:
-  enum LookupResultKind {
-    /// @brief No entity found met the criteria.
-    NotFound = 0,
-
-    /// @brief No entity found met the criteria within the current 
-    /// instantiation,, but there were dependent base classes of the 
-    /// current instantiation that could not be searched.
-    NotFoundInCurrentInstantiation,
-    
-    /// @brief Name lookup found a single declaration that met the
-    /// criteria.  getFoundDecl() will return this declaration.
-    Found,
-
-    /// @brief Name lookup found a set of overloaded functions that
-    /// met the criteria.
-    FoundOverloaded,
-
-    /// @brief Name lookup found an unresolvable value declaration
-    /// and cannot yet complete.  This only happens in C++ dependent
-    /// contexts with dependent using declarations.
-    FoundUnresolvedValue,
-
-    /// @brief Name lookup results in an ambiguity; use
-    /// getAmbiguityKind to figure out what kind of ambiguity
-    /// we have.
-    Ambiguous
-  };
-
-  enum AmbiguityKind {
-    /// Name lookup results in an ambiguity because multiple
-    /// entities that meet the lookup criteria were found in
-    /// subobjects of different types. For example:
-    /// @code
-    /// struct A { void f(int); }
-    /// struct B { void f(double); }
-    /// struct C : A, B { };
-    /// void test(C c) {
-    ///   c.f(0); // error: A::f and B::f come from subobjects of different
-    ///           // types. overload resolution is not performed.
-    /// }
-    /// @endcode
-    AmbiguousBaseSubobjectTypes,
-
-    /// Name lookup results in an ambiguity because multiple
-    /// nonstatic entities that meet the lookup criteria were found
-    /// in different subobjects of the same type. For example:
-    /// @code
-    /// struct A { int x; };
-    /// struct B : A { };
-    /// struct C : A { };
-    /// struct D : B, C { };
-    /// int test(D d) {
-    ///   return d.x; // error: 'x' is found in two A subobjects (of B and C)
-    /// }
-    /// @endcode
-    AmbiguousBaseSubobjects,
-
-    /// Name lookup results in an ambiguity because multiple definitions
-    /// of entity that meet the lookup criteria were found in different
-    /// declaration contexts.
-    /// @code
-    /// namespace A {
-    ///   int i;
-    ///   namespace B { int i; }
-    ///   int test() {
-    ///     using namespace B;
-    ///     return i; // error 'i' is found in namespace A and A::B
-    ///    }
-    /// }
-    /// @endcode
-    AmbiguousReference,
-
-    /// Name lookup results in an ambiguity because an entity with a
-    /// tag name was hidden by an entity with an ordinary name from
-    /// a different context.
-    /// @code
-    /// namespace A { struct Foo {}; }
-    /// namespace B { void Foo(); }
-    /// namespace C {
-    ///   using namespace A;
-    ///   using namespace B;
-    /// }
-    /// void test() {
-    ///   C::Foo(); // error: tag 'A::Foo' is hidden by an object in a
-    ///             // different namespace
-    /// }
-    /// @endcode
-    AmbiguousTagHiding
-  };
-
-  /// A little identifier for flagging temporary lookup results.
-  enum TemporaryToken {
-    Temporary
-  };
-
-  typedef UnresolvedSetImpl::iterator iterator;
-
-  LookupResult(Sema &SemaRef, DeclarationName Name, SourceLocation NameLoc,
-               Sema::LookupNameKind LookupKind,
-               Sema::RedeclarationKind Redecl = Sema::NotForRedeclaration)
-    : ResultKind(NotFound),
-      Paths(0),
-      NamingClass(0),
-      SemaRef(SemaRef),
-      Name(Name),
-      NameLoc(NameLoc),
-      LookupKind(LookupKind),
-      IDNS(0),
-      Redecl(Redecl != Sema::NotForRedeclaration),
-      HideTags(true),
-      Diagnose(Redecl == Sema::NotForRedeclaration)
-  {
-    configure();
-  }
-
-  /// Creates a temporary lookup result, initializing its core data
-  /// using the information from another result.  Diagnostics are always
-  /// disabled.
-  LookupResult(TemporaryToken _, const LookupResult &Other)
-    : ResultKind(NotFound),
-      Paths(0),
-      NamingClass(0),
-      SemaRef(Other.SemaRef),
-      Name(Other.Name),
-      NameLoc(Other.NameLoc),
-      LookupKind(Other.LookupKind),
-      IDNS(Other.IDNS),
-      Redecl(Other.Redecl),
-      HideTags(Other.HideTags),
-      Diagnose(false)
-  {}
-
-  ~LookupResult() {
-    if (Diagnose) diagnose();
-    if (Paths) deletePaths(Paths);
-  }
-
-  /// Gets the name to look up.
-  DeclarationName getLookupName() const {
-    return Name;
-  }
-
-  /// \brief Sets the name to look up.
-  void setLookupName(DeclarationName Name) {
-    this->Name = Name;
-  }
-
-  /// Gets the kind of lookup to perform.
-  Sema::LookupNameKind getLookupKind() const {
-    return LookupKind;
-  }
-
-  /// True if this lookup is just looking for an existing declaration.
-  bool isForRedeclaration() const {
-    return Redecl;
-  }
-
-  /// Sets whether tag declarations should be hidden by non-tag
-  /// declarations during resolution.  The default is true.
-  void setHideTags(bool Hide) {
-    HideTags = Hide;
-  }
-
-  bool isAmbiguous() const {
-    return getResultKind() == Ambiguous;
-  }
-
-  /// Determines if this names a single result which is not an
-  /// unresolved value using decl.  If so, it is safe to call
-  /// getFoundDecl().
-  bool isSingleResult() const {
-    return getResultKind() == Found;
-  }
-
-  /// Determines if the results are overloaded.
-  bool isOverloadedResult() const {
-    return getResultKind() == FoundOverloaded;
-  }
-
-  bool isUnresolvableResult() const {
-    return getResultKind() == FoundUnresolvedValue;
-  }
-
-  LookupResultKind getResultKind() const {
-    sanity();
-    return ResultKind;
-  }
-
-  AmbiguityKind getAmbiguityKind() const {
-    assert(isAmbiguous());
-    return Ambiguity;
-  }
-
-  const UnresolvedSetImpl &asUnresolvedSet() const {
-    return Decls;
-  }
-
-  iterator begin() const { return iterator(Decls.begin()); }
-  iterator end() const { return iterator(Decls.end()); }
-
-  /// \brief Return true if no decls were found
-  bool empty() const { return Decls.empty(); }
-
-  /// \brief Return the base paths structure that's associated with
-  /// these results, or null if none is.
-  CXXBasePaths *getBasePaths() const {
-    return Paths;
-  }
-
-  /// \brief Tests whether the given declaration is acceptable.
-  bool isAcceptableDecl(NamedDecl *D) const {
-    return D->isInIdentifierNamespace(IDNS);
-  }
-
-  /// \brief Returns the identifier namespace mask for this lookup.
-  unsigned getIdentifierNamespace() const {
-    return IDNS;
-  }
-
-  /// \brief Returns whether these results arose from performing a
-  /// lookup into a class.
-  bool isClassLookup() const {
-    return NamingClass != 0;
-  }
-
-  /// \brief Returns the 'naming class' for this lookup, i.e. the
-  /// class which was looked into to find these results.
-  ///
-  /// C++0x [class.access.base]p5:
-  ///   The access to a member is affected by the class in which the
-  ///   member is named. This naming class is the class in which the
-  ///   member name was looked up and found. [Note: this class can be
-  ///   explicit, e.g., when a qualified-id is used, or implicit,
-  ///   e.g., when a class member access operator (5.2.5) is used
-  ///   (including cases where an implicit "this->" is added). If both
-  ///   a class member access operator and a qualified-id are used to
-  ///   name the member (as in p->T::m), the class naming the member
-  ///   is the class named by the nested-name-specifier of the
-  ///   qualified-id (that is, T). -- end note ]
-  ///
-  /// This is set by the lookup routines when they find results in a class.
-  CXXRecordDecl *getNamingClass() const {
-    return NamingClass;
-  }
-
-  /// \brief Sets the 'naming class' for this lookup.
-  void setNamingClass(CXXRecordDecl *Record) {
-    NamingClass = Record;
-  }
-
-  /// \brief Returns the base object type associated with this lookup;
-  /// important for [class.protected].  Most lookups do not have an
-  /// associated base object.
-  QualType getBaseObjectType() const {
-    return BaseObjectType;
-  }
-
-  /// \brief Sets the base object type for this lookup.
-  void setBaseObjectType(QualType T) {
-    BaseObjectType = T;
-  }
-
-  /// \brief Add a declaration to these results with its natural access.
-  /// Does not test the acceptance criteria.
-  void addDecl(NamedDecl *D) {
-    addDecl(D, D->getAccess());
-  }
-
-  /// \brief Add a declaration to these results with the given access.
-  /// Does not test the acceptance criteria.
-  void addDecl(NamedDecl *D, AccessSpecifier AS) {
-    Decls.addDecl(D, AS);
-    ResultKind = Found;
-  }
-
-  /// \brief Add all the declarations from another set of lookup
-  /// results.
-  void addAllDecls(const LookupResult &Other) {
-    Decls.append(Other.Decls.begin(), Other.Decls.end());
-    ResultKind = Found;
-  }
-
-  /// \brief Determine whether no result was found because we could not
-  /// search into dependent base classes of the current instantiation.
-  bool wasNotFoundInCurrentInstantiation() const {
-    return ResultKind == NotFoundInCurrentInstantiation;
-  }
-  
-  /// \brief Note that while no result was found in the current instantiation,
-  /// there were dependent base classes that could not be searched.
-  void setNotFoundInCurrentInstantiation() {
-    assert(ResultKind == NotFound && Decls.empty());
-    ResultKind = NotFoundInCurrentInstantiation;
-  }
-  
-  /// \brief Resolves the result kind of the lookup, possibly hiding
-  /// decls.
-  ///
-  /// This should be called in any environment where lookup might
-  /// generate multiple lookup results.
-  void resolveKind();
-
-  /// \brief Re-resolves the result kind of the lookup after a set of
-  /// removals has been performed.
-  void resolveKindAfterFilter() {
-    if (Decls.empty()) {
-      if (ResultKind != NotFoundInCurrentInstantiation)
-        ResultKind = NotFound;
-    } else {
-      ResultKind = Found;
-      resolveKind();
-      
-      if (Paths && (ResultKind != Ambiguous)) {
-        deletePaths(Paths);
-        Paths = 0;
-      }
-    }
-  }
-
-  template <class DeclClass>
-  DeclClass *getAsSingle() const {
-    if (getResultKind() != Found) return 0;
-    return dyn_cast<DeclClass>(getFoundDecl());
-  }
-
-  /// \brief Fetch the unique decl found by this lookup.  Asserts
-  /// that one was found.
-  ///
-  /// This is intended for users who have examined the result kind
-  /// and are certain that there is only one result.
-  NamedDecl *getFoundDecl() const {
-    assert(getResultKind() == Found
-           && "getFoundDecl called on non-unique result");
-    return (*begin())->getUnderlyingDecl();
-  }
-
-  /// Fetches a representative decl.  Useful for lazy diagnostics.
-  NamedDecl *getRepresentativeDecl() const {
-    assert(!Decls.empty() && "cannot get representative of empty set");
-    return *begin();
-  }
-
-  /// \brief Asks if the result is a single tag decl.
-  bool isSingleTagDecl() const {
-    return getResultKind() == Found && isa<TagDecl>(getFoundDecl());
-  }
-
-  /// \brief Make these results show that the name was found in
-  /// base classes of different types.
-  ///
-  /// The given paths object is copied and invalidated.
-  void setAmbiguousBaseSubobjectTypes(CXXBasePaths &P);
-
-  /// \brief Make these results show that the name was found in
-  /// distinct base classes of the same type.
-  ///
-  /// The given paths object is copied and invalidated.
-  void setAmbiguousBaseSubobjects(CXXBasePaths &P);
-
-  /// \brief Make these results show that the name was found in
-  /// different contexts and a tag decl was hidden by an ordinary
-  /// decl in a different context.
-  void setAmbiguousQualifiedTagHiding() {
-    setAmbiguous(AmbiguousTagHiding);
-  }
-
-  /// \brief Clears out any current state.
-  void clear() {
-    ResultKind = NotFound;
-    Decls.clear();
-    if (Paths) deletePaths(Paths);
-    Paths = NULL;
-  }
-
-  /// \brief Clears out any current state and re-initializes for a
-  /// different kind of lookup.
-  void clear(Sema::LookupNameKind Kind) {
-    clear();
-    LookupKind = Kind;
-    configure();
-  }
-
-  /// \brief Change this lookup's redeclaration kind.
-  void setRedeclarationKind(Sema::RedeclarationKind RK) {
-    Redecl = RK;
-    configure();
-  }
-
-  void print(llvm::raw_ostream &);
-
-  /// Suppress the diagnostics that would normally fire because of this
-  /// lookup.  This happens during (e.g.) redeclaration lookups.
-  void suppressDiagnostics() {
-    Diagnose = false;
-  }
-
-  /// Determines whether this lookup is suppressing diagnostics.
-  bool isSuppressingDiagnostics() const {
-    return Diagnose;
-  }
-
-  /// Sets a 'context' source range.
-  void setContextRange(SourceRange SR) {
-    NameContextRange = SR;
-  }
-
-  /// Gets the source range of the context of this name; for C++
-  /// qualified lookups, this is the source range of the scope
-  /// specifier.
-  SourceRange getContextRange() const {
-    return NameContextRange;
-  }
-
-  /// Gets the location of the identifier.  This isn't always defined:
-  /// sometimes we're doing lookups on synthesized names.
-  SourceLocation getNameLoc() const {
-    return NameLoc;
-  }
-
-  /// \brief Get the Sema object that this lookup result is searching
-  /// with.
-  Sema &getSema() const { return SemaRef; }
-
-  /// A class for iterating through a result set and possibly
-  /// filtering out results.  The results returned are possibly
-  /// sugared.
-  class Filter {
-    LookupResult &Results;
-    LookupResult::iterator I;
-    bool Changed;
-#ifndef NDEBUG
-    bool CalledDone;
-#endif
-    
-    friend class LookupResult;
-    Filter(LookupResult &Results)
-      : Results(Results), I(Results.begin()), Changed(false)
-#ifndef NDEBUG
-      , CalledDone(false)
-#endif
-    {}
-
-  public:
-#ifndef NDEBUG
-    ~Filter() {
-      assert(CalledDone &&
-             "LookupResult::Filter destroyed without done() call");
-    }
-#endif
-
-    bool hasNext() const {
-      return I != Results.end();
-    }
-
-    NamedDecl *next() {
-      assert(I != Results.end() && "next() called on empty filter");
-      return *I++;
-    }
-
-    /// Erase the last element returned from this iterator.
-    void erase() {
-      Results.Decls.erase(--I);
-      Changed = true;
-    }
-
-    /// Replaces the current entry with the given one, preserving the
-    /// access bits.
-    void replace(NamedDecl *D) {
-      Results.Decls.replace(I-1, D);
-      Changed = true;
-    }
-
-    /// Replaces the current entry with the given one.
-    void replace(NamedDecl *D, AccessSpecifier AS) {
-      Results.Decls.replace(I-1, D, AS);
-      Changed = true;
-    }
-
-    void done() {
-#ifndef NDEBUG
-      assert(!CalledDone && "done() called twice");
-      CalledDone = true;
-#endif
-
-      if (Changed)
-        Results.resolveKindAfterFilter();
-    }
-  };
-
-  /// Create a filter for this result set.
-  Filter makeFilter() {
-    return Filter(*this);
-  }
-
-private:
-  void diagnose() {
-    if (isAmbiguous())
-      SemaRef.DiagnoseAmbiguousLookup(*this);
-    else if (isClassLookup() && SemaRef.getLangOptions().AccessControl)
-      SemaRef.CheckLookupAccess(*this);
-  }
-
-  void setAmbiguous(AmbiguityKind AK) {
-    ResultKind = Ambiguous;
-    Ambiguity = AK;
-  }
-
-  void addDeclsFromBasePaths(const CXXBasePaths &P);
-  void configure();
-
-  // Sanity checks.
-  void sanity() const {
-    assert(ResultKind != NotFound || Decls.size() == 0);
-    assert(ResultKind != Found || Decls.size() == 1);
-    assert(ResultKind != FoundOverloaded || Decls.size() > 1 ||
-           (Decls.size() == 1 &&
-            isa<FunctionTemplateDecl>((*begin())->getUnderlyingDecl())));
-    assert(ResultKind != FoundUnresolvedValue || sanityCheckUnresolved());
-    assert(ResultKind != Ambiguous || Decls.size() > 1 ||
-           (Decls.size() == 1 && Ambiguity == AmbiguousBaseSubobjects));
-    assert((Paths != NULL) == (ResultKind == Ambiguous &&
-                               (Ambiguity == AmbiguousBaseSubobjectTypes ||
-                                Ambiguity == AmbiguousBaseSubobjects)));
-  }
-
-  bool sanityCheckUnresolved() const {
-    for (iterator I = begin(), E = end(); I != E; ++I)
-      if (isa<UnresolvedUsingValueDecl>(*I))
-        return true;
-    return false;
-  }
-
-  static void deletePaths(CXXBasePaths *);
-
-  // Results.
-  LookupResultKind ResultKind;
-  AmbiguityKind Ambiguity; // ill-defined unless ambiguous
-  UnresolvedSet<8> Decls;
-  CXXBasePaths *Paths;
-  CXXRecordDecl *NamingClass;
-  QualType BaseObjectType;
-
-  // Parameters.
-  Sema &SemaRef;
-  DeclarationName Name;
-  SourceLocation NameLoc;
-  SourceRange NameContextRange;
-  Sema::LookupNameKind LookupKind;
-  unsigned IDNS; // set by configure()
-
-  bool Redecl;
-
-  /// \brief True if tag declarations should be hidden if non-tags
-  ///   are present
-  bool HideTags;
-
-  bool Diagnose;
-};
-
-  /// \brief Consumes visible declarations found when searching for
-  /// all visible names within a given scope or context.
-  ///
-  /// This abstract class is meant to be subclassed by clients of \c
-  /// Sema::LookupVisibleDecls(), each of which should override the \c
-  /// FoundDecl() function to process declarations as they are found.
-  class VisibleDeclConsumer {
-  public:
-    /// \brief Destroys the visible declaration consumer.
-    virtual ~VisibleDeclConsumer();
-
-    /// \brief Invoked each time \p Sema::LookupVisibleDecls() finds a
-    /// declaration visible from the current scope or context.
-    ///
-    /// \param ND the declaration found.
-    ///
-    /// \param Hiding a declaration that hides the declaration \p ND,
-    /// or NULL if no such declaration exists.
-    ///
-    /// \param InBaseClass whether this declaration was found in base
-    /// class of the context we searched.
-    virtual void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, 
-                           bool InBaseClass) = 0;
-  };
-
-/// \brief A class for storing results from argument-dependent lookup.
-class ADLResult {
-private:
-  /// A map from canonical decls to the 'most recent' decl.
-  llvm::DenseMap<NamedDecl*, NamedDecl*> Decls;
-
-public:
-  /// Adds a new ADL candidate to this map.
-  void insert(NamedDecl *D);
-
-  /// Removes any data associated with a given decl.
-  void erase(NamedDecl *D) {
-    Decls.erase(cast<NamedDecl>(D->getCanonicalDecl()));
-  }
-
-  class iterator {
-    typedef llvm::DenseMap<NamedDecl*,NamedDecl*>::iterator inner_iterator;
-    inner_iterator iter;
-
-    friend class ADLResult;
-    iterator(const inner_iterator &iter) : iter(iter) {}
-  public:
-    iterator() {}
-
-    iterator &operator++() { ++iter; return *this; }
-    iterator operator++(int) { return iterator(iter++); }
-
-    NamedDecl *operator*() const { return iter->second; }
-
-    bool operator==(const iterator &other) const { return iter == other.iter; }
-    bool operator!=(const iterator &other) const { return iter != other.iter; }
-  };
-
-  iterator begin() { return iterator(Decls.begin()); }
-  iterator end() { return iterator(Decls.end()); }
-};
-
-}
-
-#endif
diff --git a/lib/Sema/Makefile b/lib/Sema/Makefile
index 90f2dffa0f1b..2c02739d268f 100644
--- a/lib/Sema/Makefile
+++ b/lib/Sema/Makefile
@@ -14,7 +14,6 @@
 
 CLANG_LEVEL := ../..
 LIBRARYNAME := clangSema
-BUILD_ARCHIVE = 1
 
 include $(CLANG_LEVEL)/Makefile
 
diff --git a/lib/Sema/Sema.cpp b/lib/Sema/Sema.cpp
index cddc84eeedff..17817d41691b 100644
--- a/lib/Sema/Sema.cpp
+++ b/lib/Sema/Sema.cpp
@@ -12,26 +12,36 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/DelayedDiagnostic.h"
 #include "TargetAttributesSema.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/APFloat.h"
+#include "clang/Sema/CXXFieldCollector.h"
 #include "clang/Sema/ExternalSemaSource.h"
-#include "clang/AST/ASTConsumer.h"
+#include "clang/Sema/PrettyDeclStackTrace.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
 using namespace clang;
+using namespace sema;
 
 FunctionScopeInfo::~FunctionScopeInfo() { }
 
 void FunctionScopeInfo::Clear(unsigned NumErrors) {
-  NeedsScopeChecking = false;
+  HasBranchProtectedScope = false;
+  HasBranchIntoScope = false;
+  HasIndirectGoto = false;
+  
   LabelMap.clear();
   SwitchStack.clear();
   Returns.clear();
@@ -40,13 +50,13 @@ void FunctionScopeInfo::Clear(unsigned NumErrors) {
 
 BlockScopeInfo::~BlockScopeInfo() { }
 
-void Sema::ActOnTranslationUnitScope(SourceLocation Loc, Scope *S) {
+void Sema::ActOnTranslationUnitScope(Scope *S) {
   TUScope = S;
   PushDeclContext(S, Context.getTranslationUnitDecl());
 
   VAListTagName = PP.getIdentifierInfo("__va_list_tag");
 
-  if (!Context.isInt128Installed() && // May be set by PCHReader.
+  if (!Context.isInt128Installed() && // May be set by ASTReader.
       PP.getTargetInfo().getPointerWidth(0) >= 64) {
     TypeSourceInfo *TInfo;
 
@@ -68,7 +78,7 @@ void Sema::ActOnTranslationUnitScope(SourceLocation Loc, Scope *S) {
 
   if (!PP.getLangOptions().ObjC1) return;
 
-  // Built-in ObjC types may already be set by PCHReader (hence isNull checks).
+  // Built-in ObjC types may already be set by ASTReader (hence isNull checks).
   if (Context.getObjCSelType().isNull()) {
     // Create the built-in typedef for 'SEL'.
     QualType SelT = Context.getPointerType(Context.ObjCBuiltinSelTy);
@@ -113,7 +123,7 @@ void Sema::ActOnTranslationUnitScope(SourceLocation Loc, Scope *S) {
     PushOnScopeChains(ClassTypedef, TUScope);
     Context.setObjCClassType(Context.getTypeDeclType(ClassTypedef));
     Context.ObjCClassRedefinitionType = Context.getObjCClassType();
-  }
+  }  
 }
 
 Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
@@ -123,9 +133,8 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
     LangOpts(pp.getLangOptions()), PP(pp), Context(ctxt), Consumer(consumer),
     Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()),
     ExternalSource(0), CodeCompleter(CodeCompleter), CurContext(0), 
-    PackContext(0), TopFunctionScope(0), ParsingDeclDepth(0),
-    IdResolver(pp.getLangOptions()), StdNamespace(0), StdBadAlloc(0),
-    GlobalNewDeleteDeclared(false), 
+    PackContext(0), VisContext(0), ParsingDeclDepth(0),
+    IdResolver(pp.getLangOptions()), GlobalNewDeleteDeclared(false), 
     CompleteTranslationUnit(CompleteTranslationUnit),
     NumSFINAEErrors(0), SuppressAccessChecking(false),
     NonInstantiationEntries(0), CurrentInstantiationScope(0), TyposCorrected(0),
@@ -140,22 +149,52 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
                                        &Context);
 
   ExprEvalContexts.push_back(
-                  ExpressionEvaluationContextRecord(PotentiallyEvaluated, 0));
+                  ExpressionEvaluationContextRecord(PotentiallyEvaluated, 0));  
+
+  FunctionScopes.push_back(new FunctionScopeInfo(Diags.getNumErrors()));
+}
+
+void Sema::Initialize() {
+  // Tell the AST consumer about this Sema object.
+  Consumer.Initialize(Context);
+  
+  // FIXME: Isn't this redundant with the initialization above?
+  if (SemaConsumer *SC = dyn_cast<SemaConsumer>(&Consumer))
+    SC->InitializeSema(*this);
+  
+  // Tell the external Sema source about this Sema object.
+  if (ExternalSemaSource *ExternalSema
+      = dyn_cast_or_null<ExternalSemaSource>(Context.getExternalSource()))
+    ExternalSema->InitializeSema(*this);
 }
 
 Sema::~Sema() {
   if (PackContext) FreePackedContext();
+  if (VisContext) FreeVisContext();
   delete TheTargetAttributesSema;
-  while (!FunctionScopes.empty())
-    PopFunctionOrBlockScope();
+
+  // Kill all the active scopes.
+  for (unsigned I = 1, E = FunctionScopes.size(); I != E; ++I)
+    delete FunctionScopes[I];
+  if (FunctionScopes.size() == 1)
+    delete FunctionScopes[0];
+  
+  // Tell the SemaConsumer to forget about us; we're going out of scope.
+  if (SemaConsumer *SC = dyn_cast<SemaConsumer>(&Consumer))
+    SC->ForgetSema();
+
+  // Detach from the external Sema source.
+  if (ExternalSemaSource *ExternalSema
+        = dyn_cast_or_null<ExternalSemaSource>(Context.getExternalSource()))
+    ExternalSema->ForgetSema();
 }
 
 /// ImpCastExprToType - If Expr is not of type 'Type', insert an implicit cast.
 /// If there is already an implicit cast, merge into the existing one.
-/// If isLvalue, the result of the cast is an lvalue.
+/// The result is of the given category.
 void Sema::ImpCastExprToType(Expr *&Expr, QualType Ty,
-                             CastExpr::CastKind Kind, 
-                             bool isLvalue, CXXBaseSpecifierArray BasePath) {
+                             CastKind Kind, ExprValueKind VK,
+                             const CXXCastPath *BasePath) {
   QualType ExprTy = Context.getCanonicalType(Expr->getType());
   QualType TypeTy = Context.getCanonicalType(Ty);
 
@@ -173,8 +212,8 @@ void Sema::ImpCastExprToType(Expr *&Expr, QualType Ty,
 
   // If this is a derived-to-base cast to a through a virtual base, we
   // need a vtable.
-  if (Kind == CastExpr::CK_DerivedToBase && 
-      BasePathInvolvesVirtualBase(BasePath)) {
+  if (Kind == CK_DerivedToBase && 
+      BasePathInvolvesVirtualBase(*BasePath)) {
     QualType T = Expr->getType();
     if (const PointerType *Pointer = T->getAs<PointerType>())
       T = Pointer->getPointeeType();
@@ -184,53 +223,96 @@ void Sema::ImpCastExprToType(Expr *&Expr, QualType Ty,
   }
 
   if (ImplicitCastExpr *ImpCast = dyn_cast<ImplicitCastExpr>(Expr)) {
-    if (ImpCast->getCastKind() == Kind && BasePath.empty()) {
+    if (ImpCast->getCastKind() == Kind && (!BasePath || BasePath->empty())) {
       ImpCast->setType(Ty);
-      ImpCast->setLvalueCast(isLvalue);
+      ImpCast->setValueKind(VK);
       return;
     }
   }
 
-  Expr = new (Context) ImplicitCastExpr(Ty, Kind, Expr, BasePath, isLvalue);
+  Expr = ImplicitCastExpr::Create(Context, Ty, Kind, Expr, BasePath, VK);
 }
 
-void Sema::DeleteExpr(ExprTy *E) {
-  if (E) static_cast<Expr*>(E)->Destroy(Context);
+ExprValueKind Sema::CastCategory(Expr *E) {
+  Expr::Classification Classification = E->Classify(Context);
+  return Classification.isRValue() ? VK_RValue :
+      (Classification.isLValue() ? VK_LValue : VK_XValue);
 }
-void Sema::DeleteStmt(StmtTy *S) {
-  if (S) static_cast<Stmt*>(S)->Destroy(Context);
+
+/// \brief Used to prune the decls of Sema's UnusedFileScopedDecls vector.
+static bool ShouldRemoveFromUnused(Sema *SemaRef, const DeclaratorDecl *D) {
+  if (D->isUsed())
+    return true;
+
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    // UnusedFileScopedDecls stores the first declaration.
+    // The declaration may have become definition so check again.
+    const FunctionDecl *DeclToCheck;
+    if (FD->hasBody(DeclToCheck))
+      return !SemaRef->ShouldWarnIfUnusedFileScopedDecl(DeclToCheck);
+
+    // Later redecls may add new information resulting in not having to warn,
+    // so check again.
+    DeclToCheck = FD->getMostRecentDeclaration();
+    if (DeclToCheck != FD)
+      return !SemaRef->ShouldWarnIfUnusedFileScopedDecl(DeclToCheck);
+  }
+
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    // UnusedFileScopedDecls stores the first declaration.
+    // The declaration may have become definition so check again.
+    const VarDecl *DeclToCheck = VD->getDefinition(); 
+    if (DeclToCheck)
+      return !SemaRef->ShouldWarnIfUnusedFileScopedDecl(DeclToCheck);
+
+    // Later redecls may add new information resulting in not having to warn,
+    // so check again.
+    DeclToCheck = VD->getMostRecentDeclaration();
+    if (DeclToCheck != VD)
+      return !SemaRef->ShouldWarnIfUnusedFileScopedDecl(DeclToCheck);
+  }
+
+  return false;
 }
 
 /// ActOnEndOfTranslationUnit - This is called at the very end of the
 /// translation unit when EOF is reached and all but the top-level scope is
 /// popped.
-void Sema::ActOnEndOfTranslationUnit() {  
-  while (1) {
-    // C++: Perform implicit template instantiations.
-    //
-    // FIXME: When we perform these implicit instantiations, we do not carefully
-    // keep track of the point of instantiation (C++ [temp.point]). This means
-    // that name lookup that occurs within the template instantiation will
-    // always happen at the end of the translation unit, so it will find
-    // some names that should not be found. Although this is common behavior
-    // for C++ compilers, it is technically wrong. In the future, we either need
-    // to be able to filter the results of name lookup or we need to perform
-    // template instantiations earlier.
-    PerformPendingImplicitInstantiations();
-    
-    /// If DefinedUsedVTables ends up marking any virtual member
-    /// functions it might lead to more pending template
-    /// instantiations, which is why we need to loop here.
-    if (!DefineUsedVTables())
-      break;
-  }
+void Sema::ActOnEndOfTranslationUnit() {
+  // At PCH writing, implicit instantiations and VTable handling info are
+  // stored and performed when the PCH is included.
+  if (CompleteTranslationUnit)
+    while (1) {
+      // C++: Perform implicit template instantiations.
+      //
+      // FIXME: When we perform these implicit instantiations, we do not
+      // carefully keep track of the point of instantiation (C++ [temp.point]).
+      // This means that name lookup that occurs within the template
+      // instantiation will always happen at the end of the translation unit,
+      // so it will find some names that should not be found. Although this is
+      // common behavior for C++ compilers, it is technically wrong. In the
+      // future, we either need to be able to filter the results of name lookup
+      // or we need to perform template instantiations earlier.
+      PerformPendingInstantiations();
+
+      /// If DefinedUsedVTables ends up marking any virtual member
+      /// functions it might lead to more pending template
+      /// instantiations, which is why we need to loop here.
+      if (!DefineUsedVTables())
+        break;
+    }
   
-  // Remove functions that turned out to be used.
-  UnusedStaticFuncs.erase(std::remove_if(UnusedStaticFuncs.begin(), 
-                                         UnusedStaticFuncs.end(), 
-                             std::bind2nd(std::mem_fun(&FunctionDecl::isUsed),
-                                          true)), 
-                          UnusedStaticFuncs.end());
+  // Remove file scoped decls that turned out to be used.
+  UnusedFileScopedDecls.erase(std::remove_if(UnusedFileScopedDecls.begin(),
+                                             UnusedFileScopedDecls.end(),
+                              std::bind1st(std::ptr_fun(ShouldRemoveFromUnused),
+                                           this)),
+                              UnusedFileScopedDecls.end());
+
+  if (!CompleteTranslationUnit) {
+    TUScope = 0;
+    return;
+  }
 
   // Check for #pragma weak identifiers that were never declared
   // FIXME: This will cause diagnostics to be emitted in a non-determinstic
@@ -244,9 +326,6 @@ void Sema::ActOnEndOfTranslationUnit() {
       << I->first;
   }
 
-  if (!CompleteTranslationUnit)
-    return;
-
   // C99 6.9.2p2:
   //   A declaration of an identifier for an object that has file
   //   scope without an initializer, and without a storage-class
@@ -293,14 +372,28 @@ void Sema::ActOnEndOfTranslationUnit() {
 
   }
   
-  // Output warning for unused functions.
-  for (std::vector<FunctionDecl*>::iterator
-       F = UnusedStaticFuncs.begin(),
-       FEnd = UnusedStaticFuncs.end();
-       F != FEnd;
-       ++F)
-    Diag((*F)->getLocation(), diag::warn_unused_function) << (*F)->getDeclName();
-  
+  // Output warning for unused file scoped decls.
+  for (llvm::SmallVectorImpl<const DeclaratorDecl*>::iterator
+         I = UnusedFileScopedDecls.begin(),
+         E = UnusedFileScopedDecls.end(); I != E; ++I) {
+    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
+      const FunctionDecl *DiagD;
+      if (!FD->hasBody(DiagD))
+        DiagD = FD;
+      Diag(DiagD->getLocation(),
+           isa<CXXMethodDecl>(DiagD) ? diag::warn_unused_member_function
+                                     : diag::warn_unused_function)
+            << DiagD->getDeclName();
+    } else {
+      const VarDecl *DiagD = cast<VarDecl>(*I)->getDefinition();
+      if (!DiagD)
+        DiagD = cast<VarDecl>(*I);
+      Diag(DiagD->getLocation(), diag::warn_unused_variable)
+            << DiagD->getDeclName();
+    }
+  }
+
+  TUScope = 0;
 }
 
 
@@ -418,11 +511,11 @@ Scope *Sema::getScopeForContext(DeclContext *Ctx) {
 
 /// \brief Enter a new function scope
 void Sema::PushFunctionScope() {
-  if (FunctionScopes.empty()) {
-    // Use the "top" function scope rather than having to allocate memory for
-    // a new scope.
-    TopFunctionScope.Clear(getDiagnostics().getNumErrors());
-    FunctionScopes.push_back(&TopFunctionScope);
+  if (FunctionScopes.size() == 1) {
+    // Use the "top" function scope rather than having to allocate
+    // memory for a new scope.
+    FunctionScopes.back()->Clear(getDiagnostics().getNumErrors());
+    FunctionScopes.push_back(FunctionScopes.back());
     return;
   }
   
@@ -436,21 +529,17 @@ void Sema::PushBlockScope(Scope *BlockScope, BlockDecl *Block) {
 }
 
 void Sema::PopFunctionOrBlockScope() {
-  if (FunctionScopes.back() != &TopFunctionScope)
-    delete FunctionScopes.back();
-  else
-    TopFunctionScope.Clear(getDiagnostics().getNumErrors());
-  
-  FunctionScopes.pop_back();
+  FunctionScopeInfo *Scope = FunctionScopes.pop_back_val();
+  assert(!FunctionScopes.empty() && "mismatched push/pop!");
+  if (FunctionScopes.back() != Scope)
+    delete Scope;
 }
 
 /// \brief Determine whether any errors occurred within this function/method/
 /// block.
 bool Sema::hasAnyErrorsInThisFunction() const {
-  unsigned NumErrors = TopFunctionScope.NumErrorsAtStartOfFunction;
-  if (!FunctionScopes.empty())
-    NumErrors = FunctionScopes.back()->NumErrorsAtStartOfFunction;
-  return NumErrors != getDiagnostics().getNumErrors();
+  return getCurFunction()->NumErrorsAtStartOfFunction
+             != getDiagnostics().getNumErrors();
 }
 
 BlockScopeInfo *Sema::getCurBlock() {
@@ -462,3 +551,21 @@ BlockScopeInfo *Sema::getCurBlock() {
 
 // Pin this vtable to this file.
 ExternalSemaSource::~ExternalSemaSource() {}
+
+void PrettyDeclStackTraceEntry::print(llvm::raw_ostream &OS) const {
+  SourceLocation Loc = this->Loc;
+  if (!Loc.isValid() && TheDecl) Loc = TheDecl->getLocation();
+  if (Loc.isValid()) {
+    Loc.print(OS, S.getSourceManager());
+    OS << ": ";
+  }
+  OS << Message;
+
+  if (TheDecl && isa<NamedDecl>(TheDecl)) {
+    std::string Name = cast<NamedDecl>(TheDecl)->getNameAsString();
+    if (!Name.empty())
+      OS << " '" << Name << '\'';
+  }
+
+  OS << '\n';
+}
diff --git a/lib/Sema/Sema.h b/lib/Sema/Sema.h
deleted file mode 100644
index 8336918c134e..000000000000
--- a/lib/Sema/Sema.h
+++ /dev/null
@@ -1,4655 +0,0 @@
-//===--- Sema.h - Semantic Analysis & AST Building --------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the Sema class, which performs semantic analysis and
-// builds ASTs.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_AST_SEMA_H
-#define LLVM_CLANG_AST_SEMA_H
-
-#include "IdentifierResolver.h"
-#include "CXXFieldCollector.h"
-#include "SemaOverload.h"
-#include "SemaTemplate.h"
-#include "AnalysisBasedWarnings.h"
-#include "clang/AST/Attr.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclObjC.h"
-#include "clang/AST/DeclTemplate.h"
-#include "clang/AST/ExprCXX.h"
-#include "clang/AST/FullExpr.h"
-#include "clang/Parse/Action.h"
-#include "clang/Sema/SemaDiagnostic.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/OwningPtr.h"
-#include <deque>
-#include <list>
-#include <map>
-#include <string>
-#include <vector>
-
-namespace llvm {
-  class APSInt;
-}
-
-namespace clang {
-  class ASTContext;
-  class ASTConsumer;
-  class CodeCompleteConsumer;
-  class Preprocessor;
-  class Decl;
-  class DeclContext;
-  class DeclSpec;
-  class ExternalSemaSource;
-  class NamedDecl;
-  class Stmt;
-  class Expr;
-  class InitListExpr;
-  class ParenListExpr;
-  class DesignatedInitExpr;
-  class CallExpr;
-  class DeclRefExpr;
-  class UnresolvedLookupExpr;
-  class UnresolvedMemberExpr;
-  class VarDecl;
-  class ParmVarDecl;
-  class TypedefDecl;
-  class FunctionDecl;
-  class QualType;
-  class LangOptions;
-  class Token;
-  class IntegerLiteral;
-  class StringLiteral;
-  class ArrayType;
-  class LabelStmt;
-  class SwitchStmt;
-  class CXXTryStmt;
-  class ExtVectorType;
-  class TypedefDecl;
-  class TemplateDecl;
-  class TemplateArgument;
-  class TemplateArgumentLoc;
-  class TemplateArgumentList;
-  class TemplateParameterList;
-  class TemplateTemplateParmDecl;
-  class ClassTemplatePartialSpecializationDecl;
-  class ClassTemplateDecl;
-  class ObjCInterfaceDecl;
-  class ObjCCompatibleAliasDecl;
-  class ObjCProtocolDecl;
-  class ObjCImplDecl;
-  class ObjCImplementationDecl;
-  class ObjCCategoryImplDecl;
-  class ObjCCategoryDecl;
-  class ObjCIvarDecl;
-  class ObjCMethodDecl;
-  class ObjCPropertyDecl;
-  class ObjCContainerDecl;
-  class PseudoDestructorTypeStorage;
-  class FunctionProtoType;
-  class CXXBasePath;
-  class CXXBasePaths;
-  class CXXTemporary;
-  class LookupResult;
-  class InitializedEntity;
-  class InitializationKind;
-  class InitializationSequence;
-  class VisibleDeclConsumer;
-  class TargetAttributesSema;
-  class ADLResult;
-
-/// \brief Retains information about a function, method, or block that is
-/// currently being parsed.
-struct FunctionScopeInfo {
-  /// \brief Whether this scope information structure defined information for
-  /// a block.
-  bool IsBlockInfo;
-
-  /// \brief Set true when a function, method contains a VLA or ObjC try block,
-  /// which introduce scopes that need to be checked for goto conditions.  If a
-  /// function does not contain this, then it need not have the jump checker run
-  /// on it.
-  bool NeedsScopeChecking;
-
-  /// \brief The number of errors that had occurred before starting this
-  /// function or block.
-  unsigned NumErrorsAtStartOfFunction;
-
-  /// LabelMap - This is a mapping from label identifiers to the LabelStmt for
-  /// it (which acts like the label decl in some ways).  Forward referenced
-  /// labels have a LabelStmt created for them with a null location & SubStmt.
-  llvm::DenseMap<IdentifierInfo*, LabelStmt*> LabelMap;
-
-  /// SwitchStack - This is the current set of active switch statements in the
-  /// block.
-  llvm::SmallVector<SwitchStmt*, 8> SwitchStack;
-
-  /// \brief The list of return statements that occur within the function or
-  /// block, if there is any chance of applying the named return value
-  /// optimization.
-  llvm::SmallVector<ReturnStmt *, 4> Returns;
-  
-  FunctionScopeInfo(unsigned NumErrors)
-    : IsBlockInfo(false), NeedsScopeChecking(false),
-      NumErrorsAtStartOfFunction(NumErrors) { }
-
-  virtual ~FunctionScopeInfo();
-
-  /// \brief Clear out the information in this function scope, making it
-  /// suitable for reuse.
-  void Clear(unsigned NumErrors);
-
-  static bool classof(const FunctionScopeInfo *FSI) { return true; }
-};
-
-
-/// \brief Retains information about a block that is currently being parsed.
-struct BlockScopeInfo : FunctionScopeInfo {
-  bool hasBlockDeclRefExprs;
-
-  BlockDecl *TheDecl;
-  
-  /// TheScope - This is the scope for the block itself, which contains
-  /// arguments etc.
-  Scope *TheScope;
-
-  /// ReturnType - The return type of the block, or null if the block
-  /// signature didn't provide an explicit return type.
-  QualType ReturnType;
-
-  /// BlockType - The function type of the block, if one was given.
-  /// Its return type may be BuiltinType::Dependent.
-  QualType FunctionType;
-
-  BlockScopeInfo(unsigned NumErrors, Scope *BlockScope, BlockDecl *Block)
-    : FunctionScopeInfo(NumErrors), hasBlockDeclRefExprs(false),
-      TheDecl(Block), TheScope(BlockScope)
-  {
-    IsBlockInfo = true;
-  }
-
-  virtual ~BlockScopeInfo();
-
-  static bool classof(const FunctionScopeInfo *FSI) { return FSI->IsBlockInfo; }
-  static bool classof(const BlockScopeInfo *BSI) { return true; }
-};
-
-/// \brief Holds a QualType and a TypeSourceInfo* that came out of a declarator
-/// parsing.
-///
-/// LocInfoType is a "transient" type, only needed for passing to/from Parser
-/// and Sema, when we want to preserve type source info for a parsed type.
-/// It will not participate in the type system semantics in any way.
-class LocInfoType : public Type {
-  enum {
-    // The last number that can fit in Type's TC.
-    // Avoids conflict with an existing Type class.
-    LocInfo = Type::TypeLast + 1
-  };
-
-  TypeSourceInfo *DeclInfo;
-
-  LocInfoType(QualType ty, TypeSourceInfo *TInfo)
-    : Type((TypeClass)LocInfo, ty, ty->isDependentType()), DeclInfo(TInfo) {
-    assert(getTypeClass() == (TypeClass)LocInfo && "LocInfo didn't fit in TC?");
-  }
-  friend class Sema;
-
-public:
-  QualType getType() const { return getCanonicalTypeInternal(); }
-  TypeSourceInfo *getTypeSourceInfo() const { return DeclInfo; }
-
-  virtual void getAsStringInternal(std::string &Str,
-                                   const PrintingPolicy &Policy) const;
-
-  static bool classof(const Type *T) {
-    return T->getTypeClass() == (TypeClass)LocInfo;
-  }
-  static bool classof(const LocInfoType *) { return true; }
-};
-
-/// Sema - This implements semantic analysis and AST building for C.
-class Sema : public Action {
-  Sema(const Sema&);           // DO NOT IMPLEMENT
-  void operator=(const Sema&); // DO NOT IMPLEMENT
-  mutable const TargetAttributesSema* TheTargetAttributesSema;
-public:
-  const LangOptions &LangOpts;
-  Preprocessor &PP;
-  ASTContext &Context;
-  ASTConsumer &Consumer;
-  Diagnostic &Diags;
-  SourceManager &SourceMgr;
-
-  /// \brief Source of additional semantic information.
-  ExternalSemaSource *ExternalSource;
-
-  /// \brief Code-completion consumer.
-  CodeCompleteConsumer *CodeCompleter;
-
-  /// CurContext - This is the current declaration context of parsing.
-  DeclContext *CurContext;
-
-  /// VAListTagName - The declaration name corresponding to __va_list_tag.
-  /// This is used as part of a hack to omit that class from ADL results.
-  DeclarationName VAListTagName;
-
-  /// A RAII object to temporarily push a declaration context.
-  class ContextRAII {
-  private:
-    Sema &S;
-    DeclContext *SavedContext;
-
-  public:
-    ContextRAII(Sema &S, DeclContext *ContextToPush)
-      : S(S), SavedContext(S.CurContext) {
-      assert(ContextToPush && "pushing null context");
-      S.CurContext = ContextToPush;
-    }
-
-    void pop() {
-      if (!SavedContext) return;
-      S.CurContext = SavedContext;
-      SavedContext = 0;
-    }
-
-    ~ContextRAII() {
-      pop();
-    }
-  };
-
-  /// PackContext - Manages the stack for #pragma pack. An alignment
-  /// of 0 indicates default alignment.
-  void *PackContext; // Really a "PragmaPackStack*"
-
-  /// \brief Stack containing information about each of the nested function,
-  /// block, and method scopes that are currently active.
-  llvm::SmallVector<FunctionScopeInfo *, 4> FunctionScopes;
-
-  /// \brief Cached function scope object used for the top function scope
-  /// and when there is no function scope (in error cases).
-  ///
-  /// This should never be accessed directly; rather, it's address will be
-  /// pushed into \c FunctionScopes when we want to re-use it.
-  FunctionScopeInfo TopFunctionScope;
-
-  /// ExprTemporaries - This is the stack of temporaries that are created by
-  /// the current full expression.
-  llvm::SmallVector<CXXTemporary*, 8> ExprTemporaries;
-
-  /// ExtVectorDecls - This is a list all the extended vector types. This allows
-  /// us to associate a raw vector type with one of the ext_vector type names.
-  /// This is only necessary for issuing pretty diagnostics.
-  llvm::SmallVector<TypedefDecl*, 24> ExtVectorDecls;
-
-  /// FieldCollector - Collects CXXFieldDecls during parsing of C++ classes.
-  llvm::OwningPtr<CXXFieldCollector> FieldCollector;
-
-  typedef llvm::SmallPtrSet<const CXXRecordDecl*, 8> RecordDeclSetTy;
-
-  /// PureVirtualClassDiagSet - a set of class declarations which we have
-  /// emitted a list of pure virtual functions. Used to prevent emitting the
-  /// same list more than once.
-  llvm::OwningPtr<RecordDeclSetTy> PureVirtualClassDiagSet;
-
-  /// \brief A mapping from external names to the most recent
-  /// locally-scoped external declaration with that name.
-  ///
-  /// This map contains external declarations introduced in local
-  /// scoped, e.g.,
-  ///
-  /// \code
-  /// void f() {
-  ///   void foo(int, int);
-  /// }
-  /// \endcode
-  ///
-  /// Here, the name "foo" will be associated with the declaration on
-  /// "foo" within f. This name is not visible outside of
-  /// "f". However, we still find it in two cases:
-  ///
-  ///   - If we are declaring another external with the name "foo", we
-  ///     can find "foo" as a previous declaration, so that the types
-  ///     of this external declaration can be checked for
-  ///     compatibility.
-  ///
-  ///   - If we would implicitly declare "foo" (e.g., due to a call to
-  ///     "foo" in C when no prototype or definition is visible), then
-  ///     we find this declaration of "foo" and complain that it is
-  ///     not visible.
-  llvm::DenseMap<DeclarationName, NamedDecl *> LocallyScopedExternalDecls;
-
-  /// \brief All the tentative definitions encountered in the TU.
-  std::vector<VarDecl *> TentativeDefinitions;
-
-  /// \brief The set of static functions seen so far that have not been used.
-  std::vector<FunctionDecl*> UnusedStaticFuncs;
-
-  class AccessedEntity {
-  public:
-    /// A member declaration found through lookup.  The target is the
-    /// member.
-    enum MemberNonce { Member };
-
-    /// A hierarchy (base-to-derived or derived-to-base) conversion.
-    /// The target is the base class.
-    enum BaseNonce { Base };
-
-    bool isMemberAccess() const { return IsMember; }
-
-    AccessedEntity(ASTContext &Context,
-                   MemberNonce _,
-                   CXXRecordDecl *NamingClass,
-                   DeclAccessPair FoundDecl,
-                   QualType BaseObjectType)
-      : Access(FoundDecl.getAccess()), IsMember(true),
-        Target(FoundDecl.getDecl()), NamingClass(NamingClass),
-        BaseObjectType(BaseObjectType), Diag(0, Context.getDiagAllocator()) {
-    }
-
-    AccessedEntity(ASTContext &Context,
-                   BaseNonce _,
-                   CXXRecordDecl *BaseClass,
-                   CXXRecordDecl *DerivedClass,
-                   AccessSpecifier Access)
-      : Access(Access), IsMember(false),
-        Target(BaseClass), NamingClass(DerivedClass),
-        Diag(0, Context.getDiagAllocator()) {
-    }
-
-    bool isQuiet() const { return Diag.getDiagID() == 0; }
-
-    AccessSpecifier getAccess() const { return AccessSpecifier(Access); }
-
-    // These apply to member decls...
-    NamedDecl *getTargetDecl() const { return Target; }
-    CXXRecordDecl *getNamingClass() const { return NamingClass; }
-
-    // ...and these apply to hierarchy conversions.
-    CXXRecordDecl *getBaseClass() const { return cast<CXXRecordDecl>(Target); }
-    CXXRecordDecl *getDerivedClass() const { return NamingClass; }
-
-    /// Retrieves the base object type, important when accessing
-    /// an instance member.
-    QualType getBaseObjectType() const { return BaseObjectType; }
-
-    /// Sets a diagnostic to be performed.  The diagnostic is given
-    /// four (additional) arguments:
-    ///   %0 - 0 if the entity was private, 1 if protected
-    ///   %1 - the DeclarationName of the entity
-    ///   %2 - the TypeDecl type of the naming class
-    ///   %3 - the TypeDecl type of the declaring class
-    void setDiag(const PartialDiagnostic &PDiag) {
-      assert(isQuiet() && "partial diagnostic already defined");
-      Diag = PDiag;
-    }
-    PartialDiagnostic &setDiag(unsigned DiagID) {
-      assert(isQuiet() && "partial diagnostic already defined");
-      assert(DiagID && "creating null diagnostic");
-      Diag.Reset(DiagID);
-      return Diag;
-    }
-    const PartialDiagnostic &getDiag() const {
-      return Diag;
-    }
-
-  private:
-    unsigned Access : 2;
-    bool IsMember;
-    NamedDecl *Target;
-    CXXRecordDecl *NamingClass;
-    QualType BaseObjectType;
-    PartialDiagnostic Diag;
-  };
-
-  struct DelayedDiagnostic {
-    enum DDKind { Deprecation, Access };
-
-    unsigned char Kind; // actually a DDKind
-    bool Triggered;
-
-    SourceLocation Loc;
-
-    union {
-      /// Deprecation.
-      struct { NamedDecl *Decl; } DeprecationData;
-
-      /// Access control.
-      char AccessData[sizeof(AccessedEntity)];
-    };
-
-    void destroy() {
-      switch (Kind) {
-      case Access: getAccessData().~AccessedEntity(); break;
-      case Deprecation: break;
-      }
-    }
-
-    static DelayedDiagnostic makeDeprecation(SourceLocation Loc,
-                                             NamedDecl *D) {
-      DelayedDiagnostic DD;
-      DD.Kind = Deprecation;
-      DD.Triggered = false;
-      DD.Loc = Loc;
-      DD.DeprecationData.Decl = D;
-      return DD;
-    }
-
-    static DelayedDiagnostic makeAccess(SourceLocation Loc,
-                                        const AccessedEntity &Entity) {
-      DelayedDiagnostic DD;
-      DD.Kind = Access;
-      DD.Triggered = false;
-      DD.Loc = Loc;
-      new (&DD.getAccessData()) AccessedEntity(Entity);
-      return DD;
-    }
-
-    AccessedEntity &getAccessData() {
-      return *reinterpret_cast<AccessedEntity*>(AccessData);
-    }
-    const AccessedEntity &getAccessData() const {
-      return *reinterpret_cast<const AccessedEntity*>(AccessData);
-    }
-  };
-
-  /// \brief The stack of diagnostics that were delayed due to being
-  /// produced during the parsing of a declaration.
-  llvm::SmallVector<DelayedDiagnostic, 8> DelayedDiagnostics;
-
-  /// \brief The depth of the current ParsingDeclaration stack.
-  /// If nonzero, we are currently parsing a declaration (and
-  /// hence should delay deprecation warnings).
-  unsigned ParsingDeclDepth;
-
-  /// WeakUndeclaredIdentifiers - Identifiers contained in
-  /// #pragma weak before declared. rare. may alias another
-  /// identifier, declared or undeclared
-  class WeakInfo {
-    IdentifierInfo *alias;  // alias (optional)
-    SourceLocation loc;     // for diagnostics
-    bool used;              // identifier later declared?
-  public:
-    WeakInfo()
-      : alias(0), loc(SourceLocation()), used(false) {}
-    WeakInfo(IdentifierInfo *Alias, SourceLocation Loc)
-      : alias(Alias), loc(Loc), used(false) {}
-    inline IdentifierInfo * getAlias() const { return alias; }
-    inline SourceLocation getLocation() const { return loc; }
-    void setUsed(bool Used=true) { used = Used; }
-    inline bool getUsed() { return used; }
-    bool operator==(WeakInfo RHS) const {
-      return alias == RHS.getAlias() && loc == RHS.getLocation();
-    }
-    bool operator!=(WeakInfo RHS) const { return !(*this == RHS); }
-  };
-  llvm::DenseMap<IdentifierInfo*,WeakInfo> WeakUndeclaredIdentifiers;
-
-  /// WeakTopLevelDecl - Translation-unit scoped declarations generated by
-  /// #pragma weak during processing of other Decls.
-  /// I couldn't figure out a clean way to generate these in-line, so
-  /// we store them here and handle separately -- which is a hack.
-  /// It would be best to refactor this.
-  llvm::SmallVector<Decl*,2> WeakTopLevelDecl;
-
-  IdentifierResolver IdResolver;
-
-  /// Translation Unit Scope - useful to Objective-C actions that need
-  /// to lookup file scope declarations in the "ordinary" C decl namespace.
-  /// For example, user-defined classes, built-in "id" type, etc.
-  Scope *TUScope;
-
-  /// \brief The C++ "std" namespace, where the standard library resides.
-  NamespaceDecl *StdNamespace;
-
-  /// \brief The C++ "std::bad_alloc" class, which is defined by the C++
-  /// standard library.
-  CXXRecordDecl *StdBadAlloc;
-
-  /// A flag to remember whether the implicit forms of operator new and delete
-  /// have been declared.
-  bool GlobalNewDeleteDeclared;
-
-  /// \brief The set of declarations that have been referenced within
-  /// a potentially evaluated expression.
-  typedef std::vector<std::pair<SourceLocation, Decl *> >
-    PotentiallyReferencedDecls;
-
-  /// \brief A set of diagnostics that may be emitted.
-  typedef std::vector<std::pair<SourceLocation, PartialDiagnostic> >
-    PotentiallyEmittedDiagnostics;
-
-  /// \brief Data structure used to record current or nested
-  /// expression evaluation contexts.
-  struct ExpressionEvaluationContextRecord {
-    /// \brief The expression evaluation context.
-    ExpressionEvaluationContext Context;
-
-    /// \brief The number of temporaries that were active when we
-    /// entered this expression evaluation context.
-    unsigned NumTemporaries;
-
-    /// \brief The set of declarations referenced within a
-    /// potentially potentially-evaluated context.
-    ///
-    /// When leaving a potentially potentially-evaluated context, each
-    /// of these elements will be as referenced if the corresponding
-    /// potentially potentially evaluated expression is potentially
-    /// evaluated.
-    PotentiallyReferencedDecls *PotentiallyReferenced;
-
-    /// \brief The set of diagnostics to emit should this potentially
-    /// potentially-evaluated context become evaluated.
-    PotentiallyEmittedDiagnostics *PotentiallyDiagnosed;
-
-    ExpressionEvaluationContextRecord(ExpressionEvaluationContext Context,
-                                      unsigned NumTemporaries)
-      : Context(Context), NumTemporaries(NumTemporaries),
-        PotentiallyReferenced(0), PotentiallyDiagnosed(0) { }
-
-    void addReferencedDecl(SourceLocation Loc, Decl *Decl) {
-      if (!PotentiallyReferenced)
-        PotentiallyReferenced = new PotentiallyReferencedDecls;
-      PotentiallyReferenced->push_back(std::make_pair(Loc, Decl));
-    }
-
-    void addDiagnostic(SourceLocation Loc, const PartialDiagnostic &PD) {
-      if (!PotentiallyDiagnosed)
-        PotentiallyDiagnosed = new PotentiallyEmittedDiagnostics;
-      PotentiallyDiagnosed->push_back(std::make_pair(Loc, PD));
-    }
-
-    void Destroy() {
-      delete PotentiallyReferenced;
-      delete PotentiallyDiagnosed;
-      PotentiallyReferenced = 0;
-      PotentiallyDiagnosed = 0;
-    }
-  };
-
-  /// A stack of expression evaluation contexts.
-  llvm::SmallVector<ExpressionEvaluationContextRecord, 8> ExprEvalContexts;
-
-  /// \brief Whether the code handled by Sema should be considered a
-  /// complete translation unit or not.
-  ///
-  /// When true (which is generally the case), Sema will perform
-  /// end-of-translation-unit semantic tasks (such as creating
-  /// initializers for tentative definitions in C) once parsing has
-  /// completed. This flag will be false when building PCH files,
-  /// since a PCH file is by definition not a complete translation
-  /// unit.
-  bool CompleteTranslationUnit;
-
-  llvm::BumpPtrAllocator BumpAlloc;
-
-  /// \brief The number of SFINAE diagnostics that have been trapped.
-  unsigned NumSFINAEErrors;
-
-  typedef llvm::DenseMap<Selector, ObjCMethodList> MethodPool;
-
-  /// Instance/Factory Method Pools - allows efficient lookup when typechecking
-  /// messages to "id". We need to maintain a list, since selectors can have
-  /// differing signatures across classes. In Cocoa, this happens to be
-  /// extremely uncommon (only 1% of selectors are "overloaded").
-  MethodPool InstanceMethodPool;
-  MethodPool FactoryMethodPool;
-
-  MethodPool::iterator ReadMethodPool(Selector Sel, bool isInstance);
-
-  /// Private Helper predicate to check for 'self'.
-  bool isSelfExpr(Expr *RExpr);
-public:
-  Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
-       bool CompleteTranslationUnit = true,
-       CodeCompleteConsumer *CompletionConsumer = 0);
-  ~Sema();
-
-  const LangOptions &getLangOptions() const { return LangOpts; }
-  Diagnostic &getDiagnostics() const { return Diags; }
-  SourceManager &getSourceManager() const { return SourceMgr; }
-  const TargetAttributesSema &getTargetAttributesSema() const;
-
-  /// \brief Helper class that creates diagnostics with optional
-  /// template instantiation stacks.
-  ///
-  /// This class provides a wrapper around the basic DiagnosticBuilder
-  /// class that emits diagnostics. SemaDiagnosticBuilder is
-  /// responsible for emitting the diagnostic (as DiagnosticBuilder
-  /// does) and, if the diagnostic comes from inside a template
-  /// instantiation, printing the template instantiation stack as
-  /// well.
-  class SemaDiagnosticBuilder : public DiagnosticBuilder {
-    Sema &SemaRef;
-    unsigned DiagID;
-
-  public:
-    SemaDiagnosticBuilder(DiagnosticBuilder &DB, Sema &SemaRef, unsigned DiagID)
-      : DiagnosticBuilder(DB), SemaRef(SemaRef), DiagID(DiagID) { }
-
-    explicit SemaDiagnosticBuilder(Sema &SemaRef)
-      : DiagnosticBuilder(DiagnosticBuilder::Suppress), SemaRef(SemaRef) { }
-
-    ~SemaDiagnosticBuilder();
-  };
-
-  /// \brief Emit a diagnostic.
-  SemaDiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID);
-
-  /// \brief Emit a partial diagnostic.
-  SemaDiagnosticBuilder Diag(SourceLocation Loc, const PartialDiagnostic& PD);
-
-  /// \brief Build a partial diagnostic.
-  PartialDiagnostic PDiag(unsigned DiagID = 0) {
-    return PartialDiagnostic(DiagID, Context.getDiagAllocator());
-  }
-
-  virtual void DeleteExpr(ExprTy *E);
-  virtual void DeleteStmt(StmtTy *S);
-
-  OwningExprResult Owned(Expr* E) {
-    assert(!E || E->isRetained());
-    return OwningExprResult(*this, E);
-  }
-  OwningExprResult Owned(ExprResult R) {
-    if (R.isInvalid())
-      return ExprError();
-    assert(!R.get() || ((Expr*) R.get())->isRetained());
-    return OwningExprResult(*this, R.get());
-  }
-  OwningStmtResult Owned(Stmt* S) {
-    assert(!S || S->isRetained());
-    return OwningStmtResult(*this, S);
-  }
-
-  virtual void ActOnEndOfTranslationUnit();
-
-  Scope *getScopeForContext(DeclContext *Ctx);
-
-  void PushFunctionScope();
-  void PushBlockScope(Scope *BlockScope, BlockDecl *Block);
-  void PopFunctionOrBlockScope();
-
-  /// getLabelMap() - Return the current label map.  If we're in a block, we
-  /// return it.
-  llvm::DenseMap<IdentifierInfo*, LabelStmt*> &getLabelMap() {
-    if (FunctionScopes.empty())
-      return TopFunctionScope.LabelMap;
-
-    return FunctionScopes.back()->LabelMap;
-  }
-
-  /// getSwitchStack - This is returns the switch stack for the current block or
-  /// function.
-  llvm::SmallVector<SwitchStmt*,8> &getSwitchStack() {
-    if (FunctionScopes.empty())
-      return TopFunctionScope.SwitchStack;
-
-    return FunctionScopes.back()->SwitchStack;
-  }
-
-  /// \brief Determine whether the current function or block needs scope
-  /// checking.
-  bool &FunctionNeedsScopeChecking() {
-    if (FunctionScopes.empty())
-      return TopFunctionScope.NeedsScopeChecking;
-
-    return FunctionScopes.back()->NeedsScopeChecking;
-  }
-
-  bool hasAnyErrorsInThisFunction() const;
-
-  /// \brief Retrieve the current block, if any.
-  BlockScopeInfo *getCurBlock();
-
-  /// WeakTopLevelDeclDecls - access to #pragma weak-generated Decls
-  llvm::SmallVector<Decl*,2> &WeakTopLevelDecls() { return WeakTopLevelDecl; }
-
-  //===--------------------------------------------------------------------===//
-  // Type Analysis / Processing: SemaType.cpp.
-  //
-
-  QualType adjustParameterType(QualType T);
-  QualType BuildQualifiedType(QualType T, SourceLocation Loc, Qualifiers Qs);
-  QualType BuildQualifiedType(QualType T, SourceLocation Loc, unsigned CVR) {
-    return BuildQualifiedType(T, Loc, Qualifiers::fromCVRMask(CVR));
-  }
-  QualType BuildPointerType(QualType T,
-                            SourceLocation Loc, DeclarationName Entity);
-  QualType BuildReferenceType(QualType T, bool LValueRef,
-                              SourceLocation Loc, DeclarationName Entity);
-  QualType BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
-                          Expr *ArraySize, unsigned Quals,
-                          SourceRange Brackets, DeclarationName Entity);
-  QualType BuildExtVectorType(QualType T, ExprArg ArraySize,
-                              SourceLocation AttrLoc);
-  QualType BuildFunctionType(QualType T,
-                             QualType *ParamTypes, unsigned NumParamTypes,
-                             bool Variadic, unsigned Quals,
-                             SourceLocation Loc, DeclarationName Entity);
-  QualType BuildMemberPointerType(QualType T, QualType Class,
-                                  SourceLocation Loc,
-                                  DeclarationName Entity);
-  QualType BuildBlockPointerType(QualType T,
-                                 SourceLocation Loc, DeclarationName Entity);
-  TypeSourceInfo *GetTypeForDeclarator(Declarator &D, Scope *S,
-                                       TagDecl **OwnedDecl = 0);
-  TypeSourceInfo *GetTypeSourceInfoForDeclarator(Declarator &D, QualType T,
-                                               TypeSourceInfo *ReturnTypeInfo);
-  /// \brief Create a LocInfoType to hold the given QualType and TypeSourceInfo.
-  QualType CreateLocInfoType(QualType T, TypeSourceInfo *TInfo);
-  DeclarationName GetNameForDeclarator(Declarator &D);
-  DeclarationName GetNameFromUnqualifiedId(const UnqualifiedId &Name);
-  static QualType GetTypeFromParser(TypeTy *Ty, TypeSourceInfo **TInfo = 0);
-  bool CheckSpecifiedExceptionType(QualType T, const SourceRange &Range);
-  bool CheckDistantExceptionSpec(QualType T);
-  bool CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New);
-  bool CheckEquivalentExceptionSpec(
-      const FunctionProtoType *Old, SourceLocation OldLoc,
-      const FunctionProtoType *New, SourceLocation NewLoc);
-  bool CheckEquivalentExceptionSpec(
-      const PartialDiagnostic &DiagID, const PartialDiagnostic & NoteID,
-      const FunctionProtoType *Old, SourceLocation OldLoc,
-      const FunctionProtoType *New, SourceLocation NewLoc,
-      bool *MissingExceptionSpecification = 0,
-      bool *MissingEmptyExceptionSpecification = 0);
-  bool CheckExceptionSpecSubset(
-      const PartialDiagnostic &DiagID, const PartialDiagnostic & NoteID,
-      const FunctionProtoType *Superset, SourceLocation SuperLoc,
-      const FunctionProtoType *Subset, SourceLocation SubLoc);
-  bool CheckParamExceptionSpec(const PartialDiagnostic & NoteID,
-      const FunctionProtoType *Target, SourceLocation TargetLoc,
-      const FunctionProtoType *Source, SourceLocation SourceLoc);
-
-  virtual TypeResult ActOnTypeName(Scope *S, Declarator &D);
-
-  bool RequireCompleteType(SourceLocation Loc, QualType T,
-                           const PartialDiagnostic &PD,
-                           std::pair<SourceLocation, PartialDiagnostic> Note);
-  bool RequireCompleteType(SourceLocation Loc, QualType T,
-                           const PartialDiagnostic &PD);
-  bool RequireCompleteType(SourceLocation Loc, QualType T,
-                           unsigned DiagID);
-
-  QualType getElaboratedType(ElaboratedTypeKeyword Keyword,
-                             const CXXScopeSpec &SS, QualType T);
-
-  QualType BuildTypeofExprType(Expr *E);
-  QualType BuildDecltypeType(Expr *E);
-
-  //===--------------------------------------------------------------------===//
-  // Symbol table / Decl tracking callbacks: SemaDecl.cpp.
-  //
-
-  /// getDeclName - Return a pretty name for the specified decl if possible, or
-  /// an empty string if not.  This is used for pretty crash reporting.
-  virtual std::string getDeclName(DeclPtrTy D);
-
-  DeclGroupPtrTy ConvertDeclToDeclGroup(DeclPtrTy Ptr);
-
-  virtual TypeTy *getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
-                              Scope *S, CXXScopeSpec *SS,
-                              bool isClassName = false,
-                              TypeTy *ObjectType = 0);
-  virtual DeclSpec::TST isTagName(IdentifierInfo &II, Scope *S);
-  virtual bool DiagnoseUnknownTypeName(const IdentifierInfo &II,
-                                       SourceLocation IILoc,
-                                       Scope *S,
-                                       CXXScopeSpec *SS,
-                                       TypeTy *&SuggestedType);
-
-  virtual DeclPtrTy ActOnDeclarator(Scope *S, Declarator &D) {
-    return HandleDeclarator(S, D, MultiTemplateParamsArg(*this), false);
-  }
-
-  DeclPtrTy HandleDeclarator(Scope *S, Declarator &D,
-                             MultiTemplateParamsArg TemplateParameterLists,
-                             bool IsFunctionDefinition);
-  void RegisterLocallyScopedExternCDecl(NamedDecl *ND,
-                                        const LookupResult &Previous,
-                                        Scope *S);
-  void DiagnoseFunctionSpecifiers(Declarator& D);
-  void CheckShadow(Scope *S, VarDecl *D, const LookupResult& R);
-  void CheckShadow(Scope *S, VarDecl *D);
-  NamedDecl* ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                                    QualType R, TypeSourceInfo *TInfo,
-                                    LookupResult &Previous, bool &Redeclaration);
-  NamedDecl* ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                                     QualType R, TypeSourceInfo *TInfo,
-                                     LookupResult &Previous,
-                                     MultiTemplateParamsArg TemplateParamLists,
-                                     bool &Redeclaration);
-  void CheckVariableDeclaration(VarDecl *NewVD, LookupResult &Previous,
-                                bool &Redeclaration);
-  NamedDecl* ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
-                                     QualType R, TypeSourceInfo *TInfo,
-                                     LookupResult &Previous,
-                                     MultiTemplateParamsArg TemplateParamLists,
-                                     bool IsFunctionDefinition,
-                                     bool &Redeclaration);
-  void AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD);
-  void CheckFunctionDeclaration(Scope *S,
-                                FunctionDecl *NewFD, LookupResult &Previous,
-                                bool IsExplicitSpecialization,
-                                bool &Redeclaration,
-                                bool &OverloadableAttrRequired);
-  void CheckMain(FunctionDecl *FD);
-  virtual DeclPtrTy ActOnParamDeclarator(Scope *S, Declarator &D);
-  ParmVarDecl *BuildParmVarDeclForTypedef(DeclContext *DC,
-                                          SourceLocation Loc,
-                                          QualType T);
-  ParmVarDecl *CheckParameter(DeclContext *DC,
-                              TypeSourceInfo *TSInfo, QualType T,
-                              IdentifierInfo *Name,
-                              SourceLocation NameLoc,
-                              VarDecl::StorageClass StorageClass,
-                              VarDecl::StorageClass StorageClassAsWritten);
-  virtual void ActOnParamDefaultArgument(DeclPtrTy param,
-                                         SourceLocation EqualLoc,
-                                         ExprArg defarg);
-  virtual void ActOnParamUnparsedDefaultArgument(DeclPtrTy param,
-                                                 SourceLocation EqualLoc,
-                                                 SourceLocation ArgLoc);
-  virtual void ActOnParamDefaultArgumentError(DeclPtrTy param);
-  bool SetParamDefaultArgument(ParmVarDecl *Param, ExprArg DefaultArg,
-                               SourceLocation EqualLoc);
-
-
-  // Contains the locations of the beginning of unparsed default
-  // argument locations.
-  llvm::DenseMap<ParmVarDecl *,SourceLocation> UnparsedDefaultArgLocs;
-
-  virtual void AddInitializerToDecl(DeclPtrTy dcl, ExprArg init);
-  void AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit);
-  void ActOnUninitializedDecl(DeclPtrTy dcl, bool TypeContainsUndeducedAuto);
-  virtual void ActOnInitializerError(DeclPtrTy Dcl);
-  virtual void SetDeclDeleted(DeclPtrTy dcl, SourceLocation DelLoc);
-  virtual DeclGroupPtrTy FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
-                                                 DeclPtrTy *Group,
-                                                 unsigned NumDecls);
-  virtual void ActOnFinishKNRParamDeclarations(Scope *S, Declarator &D,
-                                               SourceLocation LocAfterDecls);
-  virtual DeclPtrTy ActOnStartOfFunctionDef(Scope *S, Declarator &D);
-  virtual DeclPtrTy ActOnStartOfFunctionDef(Scope *S, DeclPtrTy D);
-  virtual void ActOnStartOfObjCMethodDef(Scope *S, DeclPtrTy D);
-
-  virtual DeclPtrTy ActOnFinishFunctionBody(DeclPtrTy Decl, StmtArg Body);
-  DeclPtrTy ActOnFinishFunctionBody(DeclPtrTy Decl, StmtArg Body,
-                                    bool IsInstantiation);
-
-  /// \brief Diagnose any unused parameters in the given sequence of
-  /// ParmVarDecl pointers.
-  template<typename InputIterator>
-  void DiagnoseUnusedParameters(InputIterator Param, InputIterator ParamEnd) {
-    if (Diags.getDiagnosticLevel(diag::warn_unused_parameter) ==
-          Diagnostic::Ignored)
-      return;
-
-    // Don't diagnose unused-parameter errors in template instantiations; we
-    // will already have done so in the template itself.
-    if (!ActiveTemplateInstantiations.empty())
-      return;
-
-    for (; Param != ParamEnd; ++Param) {
-      if (!(*Param)->isUsed() && (*Param)->getDeclName() &&
-          !(*Param)->template hasAttr<UnusedAttr>()) {
-        Diag((*Param)->getLocation(), diag::warn_unused_parameter)
-          << (*Param)->getDeclName();
-      }
-    }
-  }
-
-  void DiagnoseInvalidJumps(Stmt *Body);
-  virtual DeclPtrTy ActOnFileScopeAsmDecl(SourceLocation Loc, ExprArg expr);
-
-  /// Scope actions.
-  virtual void ActOnPopScope(SourceLocation Loc, Scope *S);
-  virtual void ActOnTranslationUnitScope(SourceLocation Loc, Scope *S);
-
-  /// ParsedFreeStandingDeclSpec - This method is invoked when a declspec with
-  /// no declarator (e.g. "struct foo;") is parsed.
-  virtual DeclPtrTy ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
-                                               DeclSpec &DS);
-
-  virtual DeclPtrTy BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
-                                                AccessSpecifier AS,
-                                                RecordDecl *Record);
-
-  bool isAcceptableTagRedeclaration(const TagDecl *Previous,
-                                    TagDecl::TagKind NewTag,
-                                    SourceLocation NewTagLoc,
-                                    const IdentifierInfo &Name);
-
-  virtual DeclPtrTy ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
-                             SourceLocation KWLoc, CXXScopeSpec &SS,
-                             IdentifierInfo *Name, SourceLocation NameLoc,
-                             AttributeList *Attr, AccessSpecifier AS,
-                             MultiTemplateParamsArg TemplateParameterLists,
-                             bool &OwnedDecl, bool &IsDependent);
-
-  virtual TypeResult ActOnDependentTag(Scope *S,
-                                       unsigned TagSpec,
-                                       TagUseKind TUK,
-                                       const CXXScopeSpec &SS,
-                                       IdentifierInfo *Name,
-                                       SourceLocation TagLoc,
-                                       SourceLocation NameLoc);
-
-  virtual void ActOnDefs(Scope *S, DeclPtrTy TagD, SourceLocation DeclStart,
-                         IdentifierInfo *ClassName,
-                         llvm::SmallVectorImpl<DeclPtrTy> &Decls);
-  virtual DeclPtrTy ActOnField(Scope *S, DeclPtrTy TagD,
-                               SourceLocation DeclStart,
-                               Declarator &D, ExprTy *BitfieldWidth);
-
-  FieldDecl *HandleField(Scope *S, RecordDecl *TagD, SourceLocation DeclStart,
-                         Declarator &D, Expr *BitfieldWidth,
-                         AccessSpecifier AS);
-
-  FieldDecl *CheckFieldDecl(DeclarationName Name, QualType T,
-                            TypeSourceInfo *TInfo,
-                            RecordDecl *Record, SourceLocation Loc,
-                            bool Mutable, Expr *BitfieldWidth,
-                            SourceLocation TSSL,
-                            AccessSpecifier AS, NamedDecl *PrevDecl,
-                            Declarator *D = 0);
-
-  enum CXXSpecialMember {
-    CXXInvalid = -1,
-    CXXConstructor = 0,
-    CXXCopyConstructor = 1,
-    CXXCopyAssignment = 2,
-    CXXDestructor = 3
-  };
-  void DiagnoseNontrivial(const RecordType* Record, CXXSpecialMember mem);
-  CXXSpecialMember getSpecialMember(const CXXMethodDecl *MD);
-
-  virtual DeclPtrTy ActOnIvar(Scope *S, SourceLocation DeclStart,
-                              DeclPtrTy IntfDecl,
-                              Declarator &D, ExprTy *BitfieldWidth,
-                              tok::ObjCKeywordKind visibility);
-
-  // This is used for both record definitions and ObjC interface declarations.
-  virtual void ActOnFields(Scope* S,
-                           SourceLocation RecLoc, DeclPtrTy TagDecl,
-                           DeclPtrTy *Fields, unsigned NumFields,
-                           SourceLocation LBrac, SourceLocation RBrac,
-                           AttributeList *AttrList);
-
-  /// ActOnTagStartDefinition - Invoked when we have entered the
-  /// scope of a tag's definition (e.g., for an enumeration, class,
-  /// struct, or union).
-  virtual void ActOnTagStartDefinition(Scope *S, DeclPtrTy TagDecl);
-
-  /// ActOnStartCXXMemberDeclarations - Invoked when we have parsed a
-  /// C++ record definition's base-specifiers clause and are starting its
-  /// member declarations.
-  virtual void ActOnStartCXXMemberDeclarations(Scope *S, DeclPtrTy TagDecl,
-                                               SourceLocation LBraceLoc);
-
-  /// ActOnTagFinishDefinition - Invoked once we have finished parsing
-  /// the definition of a tag (enumeration, class, struct, or union).
-  virtual void ActOnTagFinishDefinition(Scope *S, DeclPtrTy TagDecl,
-                                        SourceLocation RBraceLoc);
-
-  /// ActOnTagDefinitionError - Invoked when there was an unrecoverable
-  /// error parsing the definition of a tag.
-  virtual void ActOnTagDefinitionError(Scope *S, DeclPtrTy TagDecl);
-
-  EnumConstantDecl *CheckEnumConstant(EnumDecl *Enum,
-                                      EnumConstantDecl *LastEnumConst,
-                                      SourceLocation IdLoc,
-                                      IdentifierInfo *Id,
-                                      ExprArg val);
-
-  virtual DeclPtrTy ActOnEnumConstant(Scope *S, DeclPtrTy EnumDecl,
-                                      DeclPtrTy LastEnumConstant,
-                                      SourceLocation IdLoc, IdentifierInfo *Id,
-                                      SourceLocation EqualLoc, ExprTy *Val);
-  virtual void ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
-                             SourceLocation RBraceLoc, DeclPtrTy EnumDecl,
-                             DeclPtrTy *Elements, unsigned NumElements,
-                             Scope *S, AttributeList *Attr);
-
-  DeclContext *getContainingDC(DeclContext *DC);
-
-  /// Set the current declaration context until it gets popped.
-  void PushDeclContext(Scope *S, DeclContext *DC);
-  void PopDeclContext();
-
-  /// EnterDeclaratorContext - Used when we must lookup names in the context
-  /// of a declarator's nested name specifier.
-  void EnterDeclaratorContext(Scope *S, DeclContext *DC);
-  void ExitDeclaratorContext(Scope *S);
-
-  DeclContext *getFunctionLevelDeclContext();
-
-  /// getCurFunctionDecl - If inside of a function body, this returns a pointer
-  /// to the function decl for the function being parsed.  If we're currently
-  /// in a 'block', this returns the containing context.
-  FunctionDecl *getCurFunctionDecl();
-
-  /// getCurMethodDecl - If inside of a method body, this returns a pointer to
-  /// the method decl for the method being parsed.  If we're currently
-  /// in a 'block', this returns the containing context.
-  ObjCMethodDecl *getCurMethodDecl();
-
-  /// getCurFunctionOrMethodDecl - Return the Decl for the current ObjC method
-  /// or C function we're in, otherwise return null.  If we're currently
-  /// in a 'block', this returns the containing context.
-  NamedDecl *getCurFunctionOrMethodDecl();
-
-  /// Add this decl to the scope shadowed decl chains.
-  void PushOnScopeChains(NamedDecl *D, Scope *S, bool AddToContext = true);
-
-  /// isDeclInScope - If 'Ctx' is a function/method, isDeclInScope returns true
-  /// if 'D' is in Scope 'S', otherwise 'S' is ignored and isDeclInScope returns
-  /// true if 'D' belongs to the given declaration context.
-  bool isDeclInScope(NamedDecl *&D, DeclContext *Ctx, Scope *S = 0);
-
-  /// Finds the scope corresponding to the given decl context, if it
-  /// happens to be an enclosing scope.  Otherwise return NULL.
-  Scope *getScopeForDeclContext(Scope *S, DeclContext *DC) {
-    DeclContext *TargetDC = DC->getPrimaryContext();
-    do {
-      if (DeclContext *ScopeDC = (DeclContext*) S->getEntity())
-        if (ScopeDC->getPrimaryContext() == TargetDC)
-          return S;
-    } while ((S = S->getParent()));
-
-    return NULL;
-  }
-
-  /// Subroutines of ActOnDeclarator().
-  TypedefDecl *ParseTypedefDecl(Scope *S, Declarator &D, QualType T,
-                                TypeSourceInfo *TInfo);
-  void MergeTypeDefDecl(TypedefDecl *New, LookupResult &OldDecls);
-  bool MergeFunctionDecl(FunctionDecl *New, Decl *Old);
-  bool MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old);
-  void MergeVarDecl(VarDecl *New, LookupResult &OldDecls);
-  bool MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old);
-
-  // AssignmentAction - This is used by all the assignment diagnostic functions
-  // to represent what is actually causing the operation
-  enum AssignmentAction {
-    AA_Assigning,
-    AA_Passing,
-    AA_Returning,
-    AA_Converting,
-    AA_Initializing,
-    AA_Sending,
-    AA_Casting
-  };
-
-  /// C++ Overloading.
-  enum OverloadKind {
-    /// This is a legitimate overload: the existing declarations are
-    /// functions or function templates with different signatures.
-    Ovl_Overload,
-
-    /// This is not an overload because the signature exactly matches
-    /// an existing declaration.
-    Ovl_Match,
-
-    /// This is not an overload because the lookup results contain a
-    /// non-function.
-    Ovl_NonFunction
-  };
-  OverloadKind CheckOverload(Scope *S,
-                             FunctionDecl *New,
-                             const LookupResult &OldDecls,
-                             NamedDecl *&OldDecl,
-                             bool IsForUsingDecl);
-  bool IsOverload(FunctionDecl *New, FunctionDecl *Old, bool IsForUsingDecl);
-
-  bool TryImplicitConversion(InitializationSequence &Sequence,
-                             const InitializedEntity &Entity,
-                             Expr *From,
-                             bool SuppressUserConversions,
-                             bool AllowExplicit,
-                             bool InOverloadResolution);
-
-  ImplicitConversionSequence
-  TryImplicitConversion(Expr* From, QualType ToType,
-                        bool SuppressUserConversions,
-                        bool AllowExplicit,
-                        bool InOverloadResolution);
-  bool IsStandardConversion(Expr *From, QualType ToType,
-                            bool InOverloadResolution,
-                            StandardConversionSequence& SCS);
-  bool IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType);
-  bool IsFloatingPointPromotion(QualType FromType, QualType ToType);
-  bool IsComplexPromotion(QualType FromType, QualType ToType);
-  bool IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
-                           bool InOverloadResolution,
-                           QualType& ConvertedType, bool &IncompatibleObjC);
-  bool isObjCPointerConversion(QualType FromType, QualType ToType,
-                               QualType& ConvertedType, bool &IncompatibleObjC);
-  bool FunctionArgTypesAreEqual (FunctionProtoType* OldType, 
-                                 FunctionProtoType* NewType);
-  
-  bool CheckPointerConversion(Expr *From, QualType ToType,
-                              CastExpr::CastKind &Kind,
-                              CXXBaseSpecifierArray& BasePath,
-                              bool IgnoreBaseAccess);
-  bool IsMemberPointerConversion(Expr *From, QualType FromType, QualType ToType,
-                                 bool InOverloadResolution,
-                                 QualType &ConvertedType);
-  bool CheckMemberPointerConversion(Expr *From, QualType ToType,
-                                    CastExpr::CastKind &Kind,
-                                    CXXBaseSpecifierArray &BasePath,
-                                    bool IgnoreBaseAccess);
-  bool IsQualificationConversion(QualType FromType, QualType ToType);
-  OverloadingResult IsUserDefinedConversion(Expr *From, QualType ToType,
-                               UserDefinedConversionSequence& User,
-                               OverloadCandidateSet& Conversions,
-                               bool AllowExplicit);
-  bool DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType);
-
-
-  ImplicitConversionSequence::CompareKind
-  CompareImplicitConversionSequences(const ImplicitConversionSequence& ICS1,
-                                     const ImplicitConversionSequence& ICS2);
-
-  ImplicitConversionSequence::CompareKind
-  CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
-                                     const StandardConversionSequence& SCS2);
-
-  ImplicitConversionSequence::CompareKind
-  CompareQualificationConversions(const StandardConversionSequence& SCS1,
-                                  const StandardConversionSequence& SCS2);
-
-  ImplicitConversionSequence::CompareKind
-  CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
-                                  const StandardConversionSequence& SCS2);
-
-  OwningExprResult PerformCopyInitialization(const InitializedEntity &Entity,
-                                             SourceLocation EqualLoc,
-                                             OwningExprResult Init);
-  ImplicitConversionSequence
-  TryObjectArgumentInitialization(QualType FromType, CXXMethodDecl *Method,
-                                  CXXRecordDecl *ActingContext);
-  bool PerformObjectArgumentInitialization(Expr *&From,
-                                           NestedNameSpecifier *Qualifier,
-                                           NamedDecl *FoundDecl,
-                                           CXXMethodDecl *Method);
-
-  ImplicitConversionSequence TryContextuallyConvertToBool(Expr *From);
-  bool PerformContextuallyConvertToBool(Expr *&From);
-
-  ImplicitConversionSequence TryContextuallyConvertToObjCId(Expr *From);
-  bool PerformContextuallyConvertToObjCId(Expr *&From);
-
-  OwningExprResult 
-  ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
-                                     const PartialDiagnostic &NotIntDiag,
-                                     const PartialDiagnostic &IncompleteDiag,
-                                     const PartialDiagnostic &ExplicitConvDiag,
-                                     const PartialDiagnostic &ExplicitConvNote,
-                                     const PartialDiagnostic &AmbigDiag,
-                                     const PartialDiagnostic &AmbigNote,
-                                     const PartialDiagnostic &ConvDiag);
-  
-  bool PerformObjectMemberConversion(Expr *&From,
-                                     NestedNameSpecifier *Qualifier,
-                                     NamedDecl *FoundDecl,
-                                     NamedDecl *Member);
-
-  // Members have to be NamespaceDecl* or TranslationUnitDecl*.
-  // TODO: make this is a typesafe union.
-  typedef llvm::SmallPtrSet<DeclContext   *, 16> AssociatedNamespaceSet;
-  typedef llvm::SmallPtrSet<CXXRecordDecl *, 16> AssociatedClassSet;
-
-  void AddOverloadCandidate(NamedDecl *Function,
-                            DeclAccessPair FoundDecl,
-                            Expr **Args, unsigned NumArgs,
-                            OverloadCandidateSet &CandidateSet);
-
-  void AddOverloadCandidate(FunctionDecl *Function,
-                            DeclAccessPair FoundDecl,
-                            Expr **Args, unsigned NumArgs,
-                            OverloadCandidateSet& CandidateSet,
-                            bool SuppressUserConversions = false,
-                            bool PartialOverloading = false);
-  void AddFunctionCandidates(const UnresolvedSetImpl &Functions,
-                             Expr **Args, unsigned NumArgs,
-                             OverloadCandidateSet& CandidateSet,
-                             bool SuppressUserConversions = false);
-  void AddMethodCandidate(DeclAccessPair FoundDecl,
-                          QualType ObjectType,
-                          Expr **Args, unsigned NumArgs,
-                          OverloadCandidateSet& CandidateSet,
-                          bool SuppressUserConversion = false);
-  void AddMethodCandidate(CXXMethodDecl *Method,
-                          DeclAccessPair FoundDecl,
-                          CXXRecordDecl *ActingContext, QualType ObjectType,
-                          Expr **Args, unsigned NumArgs,
-                          OverloadCandidateSet& CandidateSet,
-                          bool SuppressUserConversions = false);
-  void AddMethodTemplateCandidate(FunctionTemplateDecl *MethodTmpl,
-                                  DeclAccessPair FoundDecl,
-                                  CXXRecordDecl *ActingContext,
-                         const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                                  QualType ObjectType,
-                                  Expr **Args, unsigned NumArgs,
-                                  OverloadCandidateSet& CandidateSet,
-                                  bool SuppressUserConversions = false);
-  void AddTemplateOverloadCandidate(FunctionTemplateDecl *FunctionTemplate,
-                                    DeclAccessPair FoundDecl,
-                      const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                                    Expr **Args, unsigned NumArgs,
-                                    OverloadCandidateSet& CandidateSet,
-                                    bool SuppressUserConversions = false);
-  void AddConversionCandidate(CXXConversionDecl *Conversion,
-                              DeclAccessPair FoundDecl,
-                              CXXRecordDecl *ActingContext,
-                              Expr *From, QualType ToType,
-                              OverloadCandidateSet& CandidateSet);
-  void AddTemplateConversionCandidate(FunctionTemplateDecl *FunctionTemplate,
-                                      DeclAccessPair FoundDecl,
-                                      CXXRecordDecl *ActingContext,
-                                      Expr *From, QualType ToType,
-                                      OverloadCandidateSet &CandidateSet);
-  void AddSurrogateCandidate(CXXConversionDecl *Conversion,
-                             DeclAccessPair FoundDecl,
-                             CXXRecordDecl *ActingContext,
-                             const FunctionProtoType *Proto,
-                             QualType ObjectTy, Expr **Args, unsigned NumArgs,
-                             OverloadCandidateSet& CandidateSet);
-  void AddMemberOperatorCandidates(OverloadedOperatorKind Op,
-                                   SourceLocation OpLoc,
-                                   Expr **Args, unsigned NumArgs,
-                                   OverloadCandidateSet& CandidateSet,
-                                   SourceRange OpRange = SourceRange());
-  void AddBuiltinCandidate(QualType ResultTy, QualType *ParamTys,
-                           Expr **Args, unsigned NumArgs,
-                           OverloadCandidateSet& CandidateSet,
-                           bool IsAssignmentOperator = false,
-                           unsigned NumContextualBoolArguments = 0);
-  void AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
-                                    SourceLocation OpLoc,
-                                    Expr **Args, unsigned NumArgs,
-                                    OverloadCandidateSet& CandidateSet);
-  void AddArgumentDependentLookupCandidates(DeclarationName Name,
-                                            bool Operator,
-                                            Expr **Args, unsigned NumArgs,
-                        const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                                            OverloadCandidateSet& CandidateSet,
-                                            bool PartialOverloading = false);
-  bool isBetterOverloadCandidate(const OverloadCandidate& Cand1,
-                                 const OverloadCandidate& Cand2,
-                                 SourceLocation Loc);
-  OverloadingResult BestViableFunction(OverloadCandidateSet& CandidateSet,
-                                       SourceLocation Loc,
-                                       OverloadCandidateSet::iterator& Best);
-
-  enum OverloadCandidateDisplayKind {
-    /// Requests that all candidates be shown.  Viable candidates will
-    /// be printed first.
-    OCD_AllCandidates,
-
-    /// Requests that only viable candidates be shown.
-    OCD_ViableCandidates
-  };
-  void PrintOverloadCandidates(OverloadCandidateSet& CandidateSet,
-                               OverloadCandidateDisplayKind OCD,
-                               Expr **Args, unsigned NumArgs,
-                               const char *Opc = 0,
-                               SourceLocation Loc = SourceLocation());
-
-  void NoteOverloadCandidate(FunctionDecl *Fn);
-  void DiagnoseAmbiguousConversion(const ImplicitConversionSequence &ICS,
-                                   SourceLocation CaretLoc,
-                                   const PartialDiagnostic &PDiag);
-
-  FunctionDecl *ResolveAddressOfOverloadedFunction(Expr *From, QualType ToType,
-                                                   bool Complain,
-                                                   DeclAccessPair &Found);
-  FunctionDecl *ResolveSingleFunctionTemplateSpecialization(Expr *From);
-
-  Expr *FixOverloadedFunctionReference(Expr *E,
-                                       DeclAccessPair FoundDecl,
-                                       FunctionDecl *Fn);
-  OwningExprResult FixOverloadedFunctionReference(OwningExprResult,
-                                                  DeclAccessPair FoundDecl,
-                                                  FunctionDecl *Fn);
-
-  void AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
-                                   Expr **Args, unsigned NumArgs,
-                                   OverloadCandidateSet &CandidateSet,
-                                   bool PartialOverloading = false);
-
-  OwningExprResult BuildOverloadedCallExpr(Scope *S, Expr *Fn,
-                                           UnresolvedLookupExpr *ULE,
-                                           SourceLocation LParenLoc,
-                                           Expr **Args, unsigned NumArgs,
-                                           SourceLocation *CommaLocs,
-                                           SourceLocation RParenLoc);
-
-  OwningExprResult CreateOverloadedUnaryOp(SourceLocation OpLoc,
-                                           unsigned Opc,
-                                           const UnresolvedSetImpl &Fns,
-                                           ExprArg input);
-
-  OwningExprResult CreateOverloadedBinOp(SourceLocation OpLoc,
-                                         unsigned Opc,
-                                         const UnresolvedSetImpl &Fns,
-                                         Expr *LHS, Expr *RHS);
-
-  OwningExprResult CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
-                                                      SourceLocation RLoc,
-                                                      ExprArg Base,ExprArg Idx);
-
-  OwningExprResult
-  BuildCallToMemberFunction(Scope *S, Expr *MemExpr,
-                            SourceLocation LParenLoc, Expr **Args,
-                            unsigned NumArgs, SourceLocation *CommaLocs,
-                            SourceLocation RParenLoc);
-  ExprResult
-  BuildCallToObjectOfClassType(Scope *S, Expr *Object, SourceLocation LParenLoc,
-                               Expr **Args, unsigned NumArgs,
-                               SourceLocation *CommaLocs,
-                               SourceLocation RParenLoc);
-
-  OwningExprResult BuildOverloadedArrowExpr(Scope *S, ExprArg Base,
-                                            SourceLocation OpLoc);
-
-  /// CheckCallReturnType - Checks that a call expression's return type is
-  /// complete. Returns true on failure. The location passed in is the location
-  /// that best represents the call.
-  bool CheckCallReturnType(QualType ReturnType, SourceLocation Loc,
-                           CallExpr *CE, FunctionDecl *FD);
-
-  /// Helpers for dealing with blocks and functions.
-  bool CheckParmsForFunctionDef(FunctionDecl *FD);
-  void CheckCXXDefaultArguments(FunctionDecl *FD);
-  void CheckExtraCXXDefaultArguments(Declarator &D);
-  Scope *getNonFieldDeclScope(Scope *S);
-
-  /// \name Name lookup
-  ///
-  /// These routines provide name lookup that is used during semantic
-  /// analysis to resolve the various kinds of names (identifiers,
-  /// overloaded operator names, constructor names, etc.) into zero or
-  /// more declarations within a particular scope. The major entry
-  /// points are LookupName, which performs unqualified name lookup,
-  /// and LookupQualifiedName, which performs qualified name lookup.
-  ///
-  /// All name lookup is performed based on some specific criteria,
-  /// which specify what names will be visible to name lookup and how
-  /// far name lookup should work. These criteria are important both
-  /// for capturing language semantics (certain lookups will ignore
-  /// certain names, for example) and for performance, since name
-  /// lookup is often a bottleneck in the compilation of C++. Name
-  /// lookup criteria is specified via the LookupCriteria enumeration.
-  ///
-  /// The results of name lookup can vary based on the kind of name
-  /// lookup performed, the current language, and the translation
-  /// unit. In C, for example, name lookup will either return nothing
-  /// (no entity found) or a single declaration. In C++, name lookup
-  /// can additionally refer to a set of overloaded functions or
-  /// result in an ambiguity. All of the possible results of name
-  /// lookup are captured by the LookupResult class, which provides
-  /// the ability to distinguish among them.
-  //@{
-
-  /// @brief Describes the kind of name lookup to perform.
-  enum LookupNameKind {
-    /// Ordinary name lookup, which finds ordinary names (functions,
-    /// variables, typedefs, etc.) in C and most kinds of names
-    /// (functions, variables, members, types, etc.) in C++.
-    LookupOrdinaryName = 0,
-    /// Tag name lookup, which finds the names of enums, classes,
-    /// structs, and unions.
-    LookupTagName,
-    /// Member name lookup, which finds the names of
-    /// class/struct/union members.
-    LookupMemberName,
-    // Look up of an operator name (e.g., operator+) for use with
-    // operator overloading. This lookup is similar to ordinary name
-    // lookup, but will ignore any declarations that are class
-    // members.
-    LookupOperatorName,
-    /// Look up of a name that precedes the '::' scope resolution
-    /// operator in C++. This lookup completely ignores operator, object,
-    /// function, and enumerator names (C++ [basic.lookup.qual]p1).
-    LookupNestedNameSpecifierName,
-    /// Look up a namespace name within a C++ using directive or
-    /// namespace alias definition, ignoring non-namespace names (C++
-    /// [basic.lookup.udir]p1).
-    LookupNamespaceName,
-    /// Look up all declarations in a scope with the given name,
-    /// including resolved using declarations.  This is appropriate
-    /// for checking redeclarations for a using declaration.
-    LookupUsingDeclName,
-    /// Look up an ordinary name that is going to be redeclared as a
-    /// name with linkage. This lookup ignores any declarations that
-    /// are outside of the current scope unless they have linkage. See
-    /// C99 6.2.2p4-5 and C++ [basic.link]p6.
-    LookupRedeclarationWithLinkage,
-    /// Look up the name of an Objective-C protocol.
-    LookupObjCProtocolName
-  };
-
-  /// \brief Specifies whether (or how) name lookup is being performed for a
-  /// redeclaration (vs. a reference).
-  enum RedeclarationKind {
-    /// \brief The lookup is a reference to this name that is not for the
-    /// purpose of redeclaring the name.
-    NotForRedeclaration = 0,
-    /// \brief The lookup results will be used for redeclaration of a name,
-    /// if an entity by that name already exists.
-    ForRedeclaration
-  };
-
-private:
-  bool CppLookupName(LookupResult &R, Scope *S);
-
-public:
-  /// \brief Look up a name, looking for a single declaration.  Return
-  /// null if the results were absent, ambiguous, or overloaded.
-  ///
-  /// It is preferable to use the elaborated form and explicitly handle
-  /// ambiguity and overloaded.
-  NamedDecl *LookupSingleName(Scope *S, DeclarationName Name,
-                              SourceLocation Loc,
-                              LookupNameKind NameKind,
-                              RedeclarationKind Redecl
-                                = NotForRedeclaration);
-  bool LookupName(LookupResult &R, Scope *S,
-                  bool AllowBuiltinCreation = false);
-  bool LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
-                           bool InUnqualifiedLookup = false);
-  bool LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS,
-                        bool AllowBuiltinCreation = false,
-                        bool EnteringContext = false);
-  ObjCProtocolDecl *LookupProtocol(IdentifierInfo *II, SourceLocation IdLoc);
-
-  void LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S,
-                                    QualType T1, QualType T2,
-                                    UnresolvedSetImpl &Functions);
-  DeclContext::lookup_result LookupConstructors(CXXRecordDecl *Class);
-  CXXDestructorDecl *LookupDestructor(CXXRecordDecl *Class);
-
-  void ArgumentDependentLookup(DeclarationName Name, bool Operator,
-                               Expr **Args, unsigned NumArgs,
-                               ADLResult &Functions);
-
-  void LookupVisibleDecls(Scope *S, LookupNameKind Kind,
-                          VisibleDeclConsumer &Consumer);
-  void LookupVisibleDecls(DeclContext *Ctx, LookupNameKind Kind,
-                          VisibleDeclConsumer &Consumer);
-  
-  /// \brief The context in which typo-correction occurs.
-  ///
-  /// The typo-correction context affects which keywords (if any) are
-  /// considered when trying to correct for typos.
-  enum CorrectTypoContext {
-    /// \brief An unknown context, where any keyword might be valid.
-    CTC_Unknown,
-    /// \brief A context where no keywords are used (e.g. we expect an actual
-    /// name).
-    CTC_NoKeywords,
-    /// \brief A context where we're correcting a type name.
-    CTC_Type,
-    /// \brief An expression context.
-    CTC_Expression,
-    /// \brief A type cast, or anything else that can be followed by a '<'.
-    CTC_CXXCasts,
-    /// \brief A member lookup context.
-    CTC_MemberLookup,
-    /// \brief The receiver of an Objective-C message send within an
-    /// Objective-C method where 'super' is a valid keyword.
-    CTC_ObjCMessageReceiver
-  };
-
-  DeclarationName CorrectTypo(LookupResult &R, Scope *S, CXXScopeSpec *SS,
-                              DeclContext *MemberContext = 0,
-                              bool EnteringContext = false,
-                              CorrectTypoContext CTC = CTC_Unknown,
-                              const ObjCObjectPointerType *OPT = 0);
-
-  void FindAssociatedClassesAndNamespaces(Expr **Args, unsigned NumArgs,
-                                   AssociatedNamespaceSet &AssociatedNamespaces,
-                                   AssociatedClassSet &AssociatedClasses);
-
-  bool DiagnoseAmbiguousLookup(LookupResult &Result);
-  //@}
-
-  ObjCInterfaceDecl *getObjCInterfaceDecl(IdentifierInfo *&Id,
-                                          SourceLocation IdLoc,
-                                          bool TypoCorrection = false);
-  NamedDecl *LazilyCreateBuiltin(IdentifierInfo *II, unsigned ID,
-                                 Scope *S, bool ForRedeclaration,
-                                 SourceLocation Loc);
-  NamedDecl *ImplicitlyDefineFunction(SourceLocation Loc, IdentifierInfo &II,
-                                      Scope *S);
-  void AddKnownFunctionAttributes(FunctionDecl *FD);
-
-  // More parsing and symbol table subroutines.
-
-  // Decl attributes - this routine is the top level dispatcher.
-  void ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD);
-  void ProcessDeclAttributeList(Scope *S, Decl *D, const AttributeList *AL);
-
-  void WarnUndefinedMethod(SourceLocation ImpLoc, ObjCMethodDecl *method,
-                           bool &IncompleteImpl, unsigned DiagID);
-  void WarnConflictingTypedMethods(ObjCMethodDecl *ImpMethod,
-                                   ObjCMethodDecl *IntfMethod);
-
-  bool isPropertyReadonly(ObjCPropertyDecl *PropertyDecl,
-                          ObjCInterfaceDecl *IDecl);
-
-  /// CheckProtocolMethodDefs - This routine checks unimplemented
-  /// methods declared in protocol, and those referenced by it.
-  /// \param IDecl - Used for checking for methods which may have been
-  /// inherited.
-  void CheckProtocolMethodDefs(SourceLocation ImpLoc,
-                               ObjCProtocolDecl *PDecl,
-                               bool& IncompleteImpl,
-                               const llvm::DenseSet<Selector> &InsMap,
-                               const llvm::DenseSet<Selector> &ClsMap,
-                               ObjCContainerDecl *CDecl);
-
-  /// CheckImplementationIvars - This routine checks if the instance variables
-  /// listed in the implelementation match those listed in the interface.
-  void CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
-                                ObjCIvarDecl **Fields, unsigned nIvars,
-                                SourceLocation Loc);
-
-  /// ImplMethodsVsClassMethods - This is main routine to warn if any method
-  /// remains unimplemented in the class or category @implementation.
-  void ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
-                                 ObjCContainerDecl* IDecl,
-                                 bool IncompleteImpl = false);
-
-  /// DiagnoseUnimplementedProperties - This routine warns on those properties
-  /// which must be implemented by this implementation.
-  void DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
-                                       ObjCContainerDecl *CDecl,
-                                       const llvm::DenseSet<Selector>& InsMap);
-
-  /// DefaultSynthesizeProperties - This routine default synthesizes all 
-  /// properties which must be synthesized in class's @implementation.
-  void DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
-                                    ObjCInterfaceDecl *IDecl);
-  
-  /// CollectImmediateProperties - This routine collects all properties in
-  /// the class and its conforming protocols; but not those it its super class.
-  void CollectImmediateProperties(ObjCContainerDecl *CDecl,
-            llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>& PropMap,
-            llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>& SuperPropMap);
-  
-
-  /// LookupPropertyDecl - Looks up a property in the current class and all
-  /// its protocols.
-  ObjCPropertyDecl *LookupPropertyDecl(const ObjCContainerDecl *CDecl,
-                                       IdentifierInfo *II);
-
-  /// Called by ActOnProperty to handle @property declarations in
-  ////  class extensions.
-  DeclPtrTy HandlePropertyInClassExtension(Scope *S,
-                                           ObjCCategoryDecl *CDecl,
-                                           SourceLocation AtLoc,
-                                           FieldDeclarator &FD,
-                                           Selector GetterSel,
-                                           Selector SetterSel,
-                                           const bool isAssign,
-                                           const bool isReadWrite,
-                                           const unsigned Attributes,
-                                           bool *isOverridingProperty,
-                                           TypeSourceInfo *T,
-                                           tok::ObjCKeywordKind MethodImplKind);
-
-  /// Called by ActOnProperty and HandlePropertyInClassExtension to
-  ///  handle creating the ObjcPropertyDecl for a category or @interface.
-  ObjCPropertyDecl *CreatePropertyDecl(Scope *S,
-                                       ObjCContainerDecl *CDecl,
-                                       SourceLocation AtLoc,
-                                       FieldDeclarator &FD,
-                                       Selector GetterSel,
-                                       Selector SetterSel,
-                                       const bool isAssign,
-                                       const bool isReadWrite,
-                                       const unsigned Attributes,
-                                       TypeSourceInfo *T,
-                                       tok::ObjCKeywordKind MethodImplKind,
-                                       DeclContext *lexicalDC = 0);
-
-  /// AtomicPropertySetterGetterRules - This routine enforces the rule (via
-  /// warning) when atomic property has one but not the other user-declared
-  /// setter or getter.
-  void AtomicPropertySetterGetterRules(ObjCImplDecl* IMPDecl,
-                                       ObjCContainerDecl* IDecl);
-
-  void DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID, ObjCInterfaceDecl *SID);
-
-  /// MatchTwoMethodDeclarations - Checks if two methods' type match and returns
-  /// true, or false, accordingly.
-  bool MatchTwoMethodDeclarations(const ObjCMethodDecl *Method,
-                                  const ObjCMethodDecl *PrevMethod,
-                                  bool matchBasedOnSizeAndAlignment = false);
-
-  /// MatchAllMethodDeclarations - Check methods declaraed in interface or
-  /// or protocol against those declared in their implementations.
-  void MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
-                                  const llvm::DenseSet<Selector> &ClsMap,
-                                  llvm::DenseSet<Selector> &InsMapSeen,
-                                  llvm::DenseSet<Selector> &ClsMapSeen,
-                                  ObjCImplDecl* IMPDecl,
-                                  ObjCContainerDecl* IDecl,
-                                  bool &IncompleteImpl,
-                                  bool ImmediateClass);
-
-  /// AddInstanceMethodToGlobalPool - All instance methods in a translation
-  /// unit are added to a global pool. This allows us to efficiently associate
-  /// a selector with a method declaraation for purposes of typechecking
-  /// messages sent to "id" (where the class of the object is unknown).
-  void AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method);
-
-  /// LookupInstanceMethodInGlobalPool - Returns the method and warns if
-  /// there are multiple signatures.
-  ObjCMethodDecl *LookupInstanceMethodInGlobalPool(Selector Sel, SourceRange R,
-                                                   bool warn=true);
-
-  /// LookupFactoryMethodInGlobalPool - Returns the method and warns if
-  /// there are multiple signatures.
-  ObjCMethodDecl *LookupFactoryMethodInGlobalPool(Selector Sel, SourceRange R);
-
-  /// AddFactoryMethodToGlobalPool - Same as above, but for factory methods.
-  void AddFactoryMethodToGlobalPool(ObjCMethodDecl *Method);
-
-  /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
-  /// initialization.
-  void CollectIvarsToConstructOrDestruct(const ObjCInterfaceDecl *OI,
-                                  llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars);
-  //===--------------------------------------------------------------------===//
-  // Statement Parsing Callbacks: SemaStmt.cpp.
-public:
-  virtual OwningStmtResult ActOnExprStmt(FullExprArg Expr);
-
-  virtual OwningStmtResult ActOnNullStmt(SourceLocation SemiLoc);
-  virtual OwningStmtResult ActOnCompoundStmt(SourceLocation L, SourceLocation R,
-                                             MultiStmtArg Elts,
-                                             bool isStmtExpr);
-  virtual OwningStmtResult ActOnDeclStmt(DeclGroupPtrTy Decl,
-                                         SourceLocation StartLoc,
-                                         SourceLocation EndLoc);
-  virtual void ActOnForEachDeclStmt(DeclGroupPtrTy Decl);
-  virtual OwningStmtResult ActOnCaseStmt(SourceLocation CaseLoc, ExprArg LHSVal,
-                                    SourceLocation DotDotDotLoc, ExprArg RHSVal,
-                                    SourceLocation ColonLoc);
-  virtual void ActOnCaseStmtBody(StmtTy *CaseStmt, StmtArg SubStmt);
-
-  virtual OwningStmtResult ActOnDefaultStmt(SourceLocation DefaultLoc,
-                                            SourceLocation ColonLoc,
-                                            StmtArg SubStmt, Scope *CurScope);
-  virtual OwningStmtResult ActOnLabelStmt(SourceLocation IdentLoc,
-                                          IdentifierInfo *II,
-                                          SourceLocation ColonLoc,
-                                          StmtArg SubStmt);
-  virtual OwningStmtResult ActOnIfStmt(SourceLocation IfLoc,
-                                       FullExprArg CondVal, DeclPtrTy CondVar,
-                                       StmtArg ThenVal,
-                                       SourceLocation ElseLoc, StmtArg ElseVal);
-  virtual OwningStmtResult ActOnStartOfSwitchStmt(SourceLocation SwitchLoc,
-                                                  ExprArg Cond,
-                                                  DeclPtrTy CondVar);
-  virtual OwningStmtResult ActOnFinishSwitchStmt(SourceLocation SwitchLoc,
-                                                 StmtArg Switch, StmtArg Body);
-  virtual OwningStmtResult ActOnWhileStmt(SourceLocation WhileLoc,
-                                          FullExprArg Cond,
-                                          DeclPtrTy CondVar, StmtArg Body);
-  virtual OwningStmtResult ActOnDoStmt(SourceLocation DoLoc, StmtArg Body,
-                                       SourceLocation WhileLoc,
-                                       SourceLocation CondLParen, ExprArg Cond,
-                                       SourceLocation CondRParen);
-
-  virtual OwningStmtResult ActOnForStmt(SourceLocation ForLoc,
-                                        SourceLocation LParenLoc,
-                                        StmtArg First, FullExprArg Second,
-                                        DeclPtrTy SecondVar,
-                                        FullExprArg Third,
-                                        SourceLocation RParenLoc,
-                                        StmtArg Body);
-  virtual OwningStmtResult ActOnObjCForCollectionStmt(SourceLocation ForColLoc,
-                                       SourceLocation LParenLoc,
-                                       StmtArg First, ExprArg Second,
-                                       SourceLocation RParenLoc, StmtArg Body);
-
-  virtual OwningStmtResult ActOnGotoStmt(SourceLocation GotoLoc,
-                                         SourceLocation LabelLoc,
-                                         IdentifierInfo *LabelII);
-  virtual OwningStmtResult ActOnIndirectGotoStmt(SourceLocation GotoLoc,
-                                                 SourceLocation StarLoc,
-                                                 ExprArg DestExp);
-  virtual OwningStmtResult ActOnContinueStmt(SourceLocation ContinueLoc,
-                                             Scope *CurScope);
-  virtual OwningStmtResult ActOnBreakStmt(SourceLocation GotoLoc,
-                                          Scope *CurScope);
-
-  virtual OwningStmtResult ActOnReturnStmt(SourceLocation ReturnLoc,
-                                           ExprArg RetValExp);
-  OwningStmtResult ActOnBlockReturnStmt(SourceLocation ReturnLoc,
-                                        Expr *RetValExp);
-
-  virtual OwningStmtResult ActOnAsmStmt(SourceLocation AsmLoc,
-                                        bool IsSimple,
-                                        bool IsVolatile,
-                                        unsigned NumOutputs,
-                                        unsigned NumInputs,
-                                        IdentifierInfo **Names,
-                                        MultiExprArg Constraints,
-                                        MultiExprArg Exprs,
-                                        ExprArg AsmString,
-                                        MultiExprArg Clobbers,
-                                        SourceLocation RParenLoc,
-                                        bool MSAsm = false);
-
-
-  VarDecl *BuildObjCExceptionDecl(TypeSourceInfo *TInfo, QualType ExceptionType,
-                                  IdentifierInfo *Name, SourceLocation NameLoc,
-                                  bool Invalid = false);
-
-  virtual DeclPtrTy ActOnObjCExceptionDecl(Scope *S, Declarator &D);
-
-  virtual OwningStmtResult ActOnObjCAtCatchStmt(SourceLocation AtLoc,
-                                                SourceLocation RParen,
-                                                DeclPtrTy Parm, StmtArg Body);
-
-  virtual OwningStmtResult ActOnObjCAtFinallyStmt(SourceLocation AtLoc,
-                                                  StmtArg Body);
-
-  virtual OwningStmtResult ActOnObjCAtTryStmt(SourceLocation AtLoc,
-                                              StmtArg Try,
-                                              MultiStmtArg Catch,
-                                              StmtArg Finally);
-
-  virtual OwningStmtResult BuildObjCAtThrowStmt(SourceLocation AtLoc,
-                                                ExprArg Throw);
-  virtual OwningStmtResult ActOnObjCAtThrowStmt(SourceLocation AtLoc,
-                                                ExprArg Throw,
-                                                Scope *CurScope);
-  virtual OwningStmtResult ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc,
-                                                       ExprArg SynchExpr,
-                                                       StmtArg SynchBody);
-
-  VarDecl *BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
-                                     TypeSourceInfo *TInfo,
-                                     IdentifierInfo *Name,
-                                     SourceLocation Loc,
-                                     SourceRange Range);
-  virtual DeclPtrTy ActOnExceptionDeclarator(Scope *S, Declarator &D);
-
-  virtual OwningStmtResult ActOnCXXCatchBlock(SourceLocation CatchLoc,
-                                              DeclPtrTy ExDecl,
-                                              StmtArg HandlerBlock);
-  virtual OwningStmtResult ActOnCXXTryBlock(SourceLocation TryLoc,
-                                            StmtArg TryBlock,
-                                            MultiStmtArg Handlers);
-  void DiagnoseReturnInConstructorExceptionHandler(CXXTryStmt *TryBlock);
-
-  /// DiagnoseUnusedExprResult - If the statement passed in is an expression
-  /// whose result is unused, warn.
-  void DiagnoseUnusedExprResult(const Stmt *S);
-  void DiagnoseUnusedDecl(const NamedDecl *ND);
-  
-  ParsingDeclStackState PushParsingDeclaration();
-  void PopParsingDeclaration(ParsingDeclStackState S, DeclPtrTy D);
-  void EmitDeprecationWarning(NamedDecl *D, SourceLocation Loc);
-
-  void HandleDelayedDeprecationCheck(DelayedDiagnostic &DD, Decl *Ctx);
-
-  //===--------------------------------------------------------------------===//
-  // Expression Parsing Callbacks: SemaExpr.cpp.
-
-  bool DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc);
-  bool DiagnosePropertyAccessorMismatch(ObjCPropertyDecl *PD,
-                                        ObjCMethodDecl *Getter,
-                                        SourceLocation Loc);
-  void DiagnoseSentinelCalls(NamedDecl *D, SourceLocation Loc,
-                             Expr **Args, unsigned NumArgs);
-
-  virtual void
-  PushExpressionEvaluationContext(ExpressionEvaluationContext NewContext);
-
-  virtual void PopExpressionEvaluationContext();
-
-  void MarkDeclarationReferenced(SourceLocation Loc, Decl *D);
-  void MarkDeclarationsReferencedInType(SourceLocation Loc, QualType T);
-  bool DiagRuntimeBehavior(SourceLocation Loc, const PartialDiagnostic &PD);
-
-  // Primary Expressions.
-  virtual SourceRange getExprRange(ExprTy *E) const;
-
-  virtual OwningExprResult ActOnIdExpression(Scope *S,
-                                             CXXScopeSpec &SS,
-                                             UnqualifiedId &Name,
-                                             bool HasTrailingLParen,
-                                             bool IsAddressOfOperand);
-
-  bool DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
-                           CorrectTypoContext CTC = CTC_Unknown);
-
-  OwningExprResult LookupInObjCMethod(LookupResult &R,
-                                      Scope *S,
-                                      IdentifierInfo *II,
-                                      bool AllowBuiltinCreation=false);
-
-  OwningExprResult ActOnDependentIdExpression(const CXXScopeSpec &SS,
-                                              DeclarationName Name,
-                                              SourceLocation NameLoc,
-                                              bool isAddressOfOperand,
-                                const TemplateArgumentListInfo *TemplateArgs);
-
-  OwningExprResult BuildDeclRefExpr(ValueDecl *D, QualType Ty,
-                                    SourceLocation Loc,
-                                    const CXXScopeSpec *SS = 0);
-  VarDecl *BuildAnonymousStructUnionMemberPath(FieldDecl *Field,
-                                    llvm::SmallVectorImpl<FieldDecl *> &Path);
-  OwningExprResult
-  BuildAnonymousStructUnionMemberReference(SourceLocation Loc,
-                                           FieldDecl *Field,
-                                           Expr *BaseObjectExpr = 0,
-                                      SourceLocation OpLoc = SourceLocation());
-  OwningExprResult BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
-                                                   LookupResult &R,
-                                const TemplateArgumentListInfo *TemplateArgs);
-  OwningExprResult BuildImplicitMemberExpr(const CXXScopeSpec &SS,
-                                           LookupResult &R,
-                                const TemplateArgumentListInfo *TemplateArgs,
-                                           bool IsDefiniteInstance);
-  bool UseArgumentDependentLookup(const CXXScopeSpec &SS,
-                                  const LookupResult &R,
-                                  bool HasTrailingLParen);
-
-  OwningExprResult BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS,
-                                                     DeclarationName Name,
-                                                     SourceLocation NameLoc);
-  OwningExprResult BuildDependentDeclRefExpr(const CXXScopeSpec &SS,
-                                             DeclarationName Name,
-                                             SourceLocation NameLoc,
-                                const TemplateArgumentListInfo *TemplateArgs);
-
-  OwningExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS,
-                                            LookupResult &R,
-                                            bool ADL);
-  OwningExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS,
-                                            SourceLocation Loc,
-                                            NamedDecl *D);
-
-  virtual OwningExprResult ActOnPredefinedExpr(SourceLocation Loc,
-                                               tok::TokenKind Kind);
-  virtual OwningExprResult ActOnNumericConstant(const Token &);
-  virtual OwningExprResult ActOnCharacterConstant(const Token &);
-  virtual OwningExprResult ActOnParenExpr(SourceLocation L, SourceLocation R,
-                                          ExprArg Val);
-  virtual OwningExprResult ActOnParenOrParenListExpr(SourceLocation L,
-                                              SourceLocation R,
-                                              MultiExprArg Val,
-                                              TypeTy *TypeOfCast=0);
-
-  /// ActOnStringLiteral - The specified tokens were lexed as pasted string
-  /// fragments (e.g. "foo" "bar" L"baz").
-  virtual OwningExprResult ActOnStringLiteral(const Token *Toks,
-                                              unsigned NumToks);
-
-  // Binary/Unary Operators.  'Tok' is the token for the operator.
-  OwningExprResult CreateBuiltinUnaryOp(SourceLocation OpLoc,
-                                        unsigned OpcIn,
-                                        ExprArg InputArg);
-  OwningExprResult BuildUnaryOp(Scope *S, SourceLocation OpLoc,
-                                UnaryOperator::Opcode Opc, ExprArg input);
-  virtual OwningExprResult ActOnUnaryOp(Scope *S, SourceLocation OpLoc,
-                                        tok::TokenKind Op, ExprArg Input);
-
-  OwningExprResult CreateSizeOfAlignOfExpr(TypeSourceInfo *T,
-                                           SourceLocation OpLoc,
-                                           bool isSizeOf, SourceRange R);
-  OwningExprResult CreateSizeOfAlignOfExpr(Expr *E, SourceLocation OpLoc,
-                                           bool isSizeOf, SourceRange R);
-  virtual OwningExprResult
-    ActOnSizeOfAlignOfExpr(SourceLocation OpLoc, bool isSizeof, bool isType,
-                           void *TyOrEx, const SourceRange &ArgRange);
-
-  bool CheckAlignOfExpr(Expr *E, SourceLocation OpLoc, const SourceRange &R);
-  bool CheckSizeOfAlignOfOperand(QualType type, SourceLocation OpLoc,
-                                 const SourceRange &R, bool isSizeof);
-
-  virtual OwningExprResult ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc,
-                                               tok::TokenKind Kind,
-                                               ExprArg Input);
-
-  virtual OwningExprResult ActOnArraySubscriptExpr(Scope *S, ExprArg Base,
-                                                   SourceLocation LLoc,
-                                                   ExprArg Idx,
-                                                   SourceLocation RLoc);
-  OwningExprResult CreateBuiltinArraySubscriptExpr(ExprArg Base,
-                                                   SourceLocation LLoc,
-                                                   ExprArg Idx,
-                                                   SourceLocation RLoc);
-
-  OwningExprResult BuildMemberReferenceExpr(ExprArg Base,
-                                            QualType BaseType,
-                                            SourceLocation OpLoc,
-                                            bool IsArrow,
-                                            CXXScopeSpec &SS,
-                                            NamedDecl *FirstQualifierInScope,
-                                            DeclarationName Name,
-                                            SourceLocation NameLoc,
-                                const TemplateArgumentListInfo *TemplateArgs);
-
-  OwningExprResult BuildMemberReferenceExpr(ExprArg Base,
-                                            QualType BaseType,
-                                            SourceLocation OpLoc, bool IsArrow,
-                                            const CXXScopeSpec &SS,
-                                            NamedDecl *FirstQualifierInScope,
-                                            LookupResult &R,
-                                 const TemplateArgumentListInfo *TemplateArgs,
-                                          bool SuppressQualifierCheck = false);
-
-  OwningExprResult LookupMemberExpr(LookupResult &R, Expr *&Base,
-                                    bool &IsArrow, SourceLocation OpLoc,
-                                    CXXScopeSpec &SS,
-                                    DeclPtrTy ObjCImpDecl,
-                                    bool HasTemplateArgs);
-
-  bool CheckQualifiedMemberReference(Expr *BaseExpr, QualType BaseType,
-                                     const CXXScopeSpec &SS,
-                                     const LookupResult &R);
-
-  OwningExprResult ActOnDependentMemberExpr(ExprArg Base,
-                                            QualType BaseType,
-                                            bool IsArrow,
-                                            SourceLocation OpLoc,
-                                            const CXXScopeSpec &SS,
-                                            NamedDecl *FirstQualifierInScope,
-                                            DeclarationName Name,
-                                            SourceLocation NameLoc,
-                               const TemplateArgumentListInfo *TemplateArgs);
-
-  virtual OwningExprResult ActOnMemberAccessExpr(Scope *S, ExprArg Base,
-                                                 SourceLocation OpLoc,
-                                                 tok::TokenKind OpKind,
-                                                 CXXScopeSpec &SS,
-                                                 UnqualifiedId &Member,
-                                                 DeclPtrTy ObjCImpDecl,
-                                                 bool HasTrailingLParen);
-
-  virtual void ActOnDefaultCtorInitializers(DeclPtrTy CDtorDecl);
-  bool ConvertArgumentsForCall(CallExpr *Call, Expr *Fn,
-                               FunctionDecl *FDecl,
-                               const FunctionProtoType *Proto,
-                               Expr **Args, unsigned NumArgs,
-                               SourceLocation RParenLoc);
-
-  /// ActOnCallExpr - Handle a call to Fn with the specified array of arguments.
-  /// This provides the location of the left/right parens and a list of comma
-  /// locations.
-  virtual OwningExprResult ActOnCallExpr(Scope *S, ExprArg Fn,
-                                         SourceLocation LParenLoc,
-                                         MultiExprArg Args,
-                                         SourceLocation *CommaLocs,
-                                         SourceLocation RParenLoc);
-  OwningExprResult BuildResolvedCallExpr(Expr *Fn,
-                                         NamedDecl *NDecl,
-                                         SourceLocation LParenLoc,
-                                         Expr **Args, unsigned NumArgs,
-                                         SourceLocation RParenLoc);
-
-  virtual OwningExprResult ActOnCastExpr(Scope *S, SourceLocation LParenLoc,
-                                         TypeTy *Ty, SourceLocation RParenLoc,
-                                         ExprArg Op);
-  OwningExprResult BuildCStyleCastExpr(SourceLocation LParenLoc,
-                                       TypeSourceInfo *Ty,
-                                       SourceLocation RParenLoc,
-                                       ExprArg Op);
-
-  virtual bool TypeIsVectorType(TypeTy *Ty) {
-    return GetTypeFromParser(Ty)->isVectorType();
-  }
-
-  OwningExprResult MaybeConvertParenListExprToParenExpr(Scope *S, ExprArg ME);
-  OwningExprResult ActOnCastOfParenListExpr(Scope *S, SourceLocation LParenLoc,
-                                            SourceLocation RParenLoc, ExprArg E,
-                                            TypeSourceInfo *TInfo);
-
-  virtual OwningExprResult ActOnCompoundLiteral(SourceLocation LParenLoc,
-                                                TypeTy *Ty,
-                                                SourceLocation RParenLoc,
-                                                ExprArg Op);
-
-  OwningExprResult BuildCompoundLiteralExpr(SourceLocation LParenLoc,
-                                            TypeSourceInfo *TInfo,
-                                            SourceLocation RParenLoc,
-                                            ExprArg InitExpr);
-
-  virtual OwningExprResult ActOnInitList(SourceLocation LParenLoc,
-                                         MultiExprArg InitList,
-                                         SourceLocation RParenLoc);
-
-  virtual OwningExprResult ActOnDesignatedInitializer(Designation &Desig,
-                                                      SourceLocation Loc,
-                                                      bool GNUSyntax,
-                                                      OwningExprResult Init);
-
-  virtual OwningExprResult ActOnBinOp(Scope *S, SourceLocation TokLoc,
-                                      tok::TokenKind Kind,
-                                      ExprArg LHS, ExprArg RHS);
-  OwningExprResult BuildBinOp(Scope *S, SourceLocation OpLoc,
-                              BinaryOperator::Opcode Opc,
-                              Expr *lhs, Expr *rhs);
-  OwningExprResult CreateBuiltinBinOp(SourceLocation TokLoc,
-                                      unsigned Opc, Expr *lhs, Expr *rhs);
-
-  /// ActOnConditionalOp - Parse a ?: operation.  Note that 'LHS' may be null
-  /// in the case of a the GNU conditional expr extension.
-  virtual OwningExprResult ActOnConditionalOp(SourceLocation QuestionLoc,
-                                              SourceLocation ColonLoc,
-                                              ExprArg Cond, ExprArg LHS,
-                                              ExprArg RHS);
-
-  /// ActOnAddrLabel - Parse the GNU address of label extension: "&&foo".
-  virtual OwningExprResult ActOnAddrLabel(SourceLocation OpLoc,
-                                          SourceLocation LabLoc,
-                                          IdentifierInfo *LabelII);
-
-  virtual OwningExprResult ActOnStmtExpr(SourceLocation LPLoc, StmtArg SubStmt,
-                                         SourceLocation RPLoc); // "({..})"
-
-  /// __builtin_offsetof(type, a.b[123][456].c)
-  OwningExprResult BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
-                                        TypeSourceInfo *TInfo,
-                                        OffsetOfComponent *CompPtr,
-                                        unsigned NumComponents,
-                                        SourceLocation RParenLoc);
-  virtual OwningExprResult ActOnBuiltinOffsetOf(Scope *S,
-                                                SourceLocation BuiltinLoc,
-                                                SourceLocation TypeLoc,
-                                                TypeTy *Arg1,
-                                                OffsetOfComponent *CompPtr,
-                                                unsigned NumComponents,
-                                                SourceLocation RParenLoc);
-
-  // __builtin_types_compatible_p(type1, type2)
-  virtual OwningExprResult ActOnTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                                                    TypeTy *arg1, TypeTy *arg2,
-                                                    SourceLocation RPLoc);
-
-  // __builtin_choose_expr(constExpr, expr1, expr2)
-  virtual OwningExprResult ActOnChooseExpr(SourceLocation BuiltinLoc,
-                                           ExprArg cond, ExprArg expr1,
-                                           ExprArg expr2, SourceLocation RPLoc);
-
-  // __builtin_va_arg(expr, type)
-  virtual OwningExprResult ActOnVAArg(SourceLocation BuiltinLoc,
-                                      ExprArg expr, TypeTy *type,
-                                      SourceLocation RPLoc);
-
-  // __null
-  virtual OwningExprResult ActOnGNUNullExpr(SourceLocation TokenLoc);
-
-  //===------------------------- "Block" Extension ------------------------===//
-
-  /// ActOnBlockStart - This callback is invoked when a block literal is
-  /// started.
-  virtual void ActOnBlockStart(SourceLocation CaretLoc, Scope *CurScope);
-
-  /// ActOnBlockArguments - This callback allows processing of block arguments.
-  /// If there are no arguments, this is still invoked.
-  virtual void ActOnBlockArguments(Declarator &ParamInfo, Scope *CurScope);
-
-  /// ActOnBlockError - If there is an error parsing a block, this callback
-  /// is invoked to pop the information about the block from the action impl.
-  virtual void ActOnBlockError(SourceLocation CaretLoc, Scope *CurScope);
-
-  /// ActOnBlockStmtExpr - This is called when the body of a block statement
-  /// literal was successfully completed.  ^(int x){...}
-  virtual OwningExprResult ActOnBlockStmtExpr(SourceLocation CaretLoc,
-                                              StmtArg Body, Scope *CurScope);
-
-  //===---------------------------- C++ Features --------------------------===//
-
-  // Act on C++ namespaces
-  virtual DeclPtrTy ActOnStartNamespaceDef(Scope *S, SourceLocation IdentLoc,
-                                           IdentifierInfo *Ident,
-                                           SourceLocation LBrace,
-                                           AttributeList *AttrList);
-  virtual void ActOnFinishNamespaceDef(DeclPtrTy Dcl, SourceLocation RBrace);
-
-  NamespaceDecl *getStdNamespace();
-  virtual DeclPtrTy ActOnUsingDirective(Scope *CurScope,
-                                        SourceLocation UsingLoc,
-                                        SourceLocation NamespcLoc,
-                                        CXXScopeSpec &SS,
-                                        SourceLocation IdentLoc,
-                                        IdentifierInfo *NamespcName,
-                                        AttributeList *AttrList);
-
-  void PushUsingDirective(Scope *S, UsingDirectiveDecl *UDir);
-
-  virtual DeclPtrTy ActOnNamespaceAliasDef(Scope *CurScope,
-                                           SourceLocation NamespaceLoc,
-                                           SourceLocation AliasLoc,
-                                           IdentifierInfo *Alias,
-                                           CXXScopeSpec &SS,
-                                           SourceLocation IdentLoc,
-                                           IdentifierInfo *Ident);
-
-  void HideUsingShadowDecl(Scope *S, UsingShadowDecl *Shadow);
-  bool CheckUsingShadowDecl(UsingDecl *UD, NamedDecl *Target,
-                            const LookupResult &PreviousDecls);
-  UsingShadowDecl *BuildUsingShadowDecl(Scope *S, UsingDecl *UD,
-                                        NamedDecl *Target);
-
-  bool CheckUsingDeclRedeclaration(SourceLocation UsingLoc,
-                                   bool isTypeName,
-                                   const CXXScopeSpec &SS,
-                                   SourceLocation NameLoc,
-                                   const LookupResult &Previous);
-  bool CheckUsingDeclQualifier(SourceLocation UsingLoc,
-                               const CXXScopeSpec &SS,
-                               SourceLocation NameLoc);
-
-  NamedDecl *BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
-                                   SourceLocation UsingLoc,
-                                   CXXScopeSpec &SS,
-                                   SourceLocation IdentLoc,
-                                   DeclarationName Name,
-                                   AttributeList *AttrList,
-                                   bool IsInstantiation,
-                                   bool IsTypeName,
-                                   SourceLocation TypenameLoc);
-
-  virtual DeclPtrTy ActOnUsingDeclaration(Scope *CurScope,
-                                          AccessSpecifier AS,
-                                          bool HasUsingKeyword,
-                                          SourceLocation UsingLoc,
-                                          CXXScopeSpec &SS,
-                                          UnqualifiedId &Name,
-                                          AttributeList *AttrList,
-                                          bool IsTypeName,
-                                          SourceLocation TypenameLoc);
-
-  /// AddCXXDirectInitializerToDecl - This action is called immediately after
-  /// ActOnDeclarator, when a C++ direct initializer is present.
-  /// e.g: "int x(1);"
-  virtual void AddCXXDirectInitializerToDecl(DeclPtrTy Dcl,
-                                             SourceLocation LParenLoc,
-                                             MultiExprArg Exprs,
-                                             SourceLocation *CommaLocs,
-                                             SourceLocation RParenLoc);
-
-  /// InitializeVarWithConstructor - Creates an CXXConstructExpr
-  /// and sets it as the initializer for the the passed in VarDecl.
-  bool InitializeVarWithConstructor(VarDecl *VD,
-                                    CXXConstructorDecl *Constructor,
-                                    MultiExprArg Exprs);
-
-  /// BuildCXXConstructExpr - Creates a complete call to a constructor,
-  /// including handling of its default argument expressions.
-  OwningExprResult
-  BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
-                        CXXConstructorDecl *Constructor, MultiExprArg Exprs,
-                        bool RequiresZeroInit = false,
-                        CXXConstructExpr::ConstructionKind ConstructKind =
-                        CXXConstructExpr::CK_Complete);
-
-  // FIXME: Can re remove this and have the above BuildCXXConstructExpr check if
-  // the constructor can be elidable?
-  OwningExprResult
-  BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
-                        CXXConstructorDecl *Constructor, bool Elidable,
-                        MultiExprArg Exprs, bool RequiresZeroInit = false,
-                        CXXConstructExpr::ConstructionKind ConstructKind =
-                        CXXConstructExpr::CK_Complete);
-
-  /// BuildCXXDefaultArgExpr - Creates a CXXDefaultArgExpr, instantiating
-  /// the default expr if needed.
-  OwningExprResult BuildCXXDefaultArgExpr(SourceLocation CallLoc,
-                                          FunctionDecl *FD,
-                                          ParmVarDecl *Param);
-
-  /// FinalizeVarWithDestructor - Prepare for calling destructor on the
-  /// constructed variable.
-  void FinalizeVarWithDestructor(VarDecl *VD, const RecordType *DeclInitType);
-
-  /// \brief Declare the implicit default constructor for the given class.
-  ///
-  /// \param ClassDecl The class declaration into which the implicit 
-  /// default constructor will be added.
-  ///
-  /// \returns The implicitly-declared default constructor.
-  CXXConstructorDecl *DeclareImplicitDefaultConstructor(
-                                                     CXXRecordDecl *ClassDecl);
-  
-  /// DefineImplicitDefaultConstructor - Checks for feasibility of
-  /// defining this constructor as the default constructor.
-  void DefineImplicitDefaultConstructor(SourceLocation CurrentLocation,
-                                        CXXConstructorDecl *Constructor);
-
-  /// \brief Declare the implicit destructor for the given class.
-  ///
-  /// \param ClassDecl The class declaration into which the implicit 
-  /// destructor will be added.
-  ///
-  /// \returns The implicitly-declared destructor.
-  CXXDestructorDecl *DeclareImplicitDestructor(CXXRecordDecl *ClassDecl);
-                                               
-  /// DefineImplicitDestructor - Checks for feasibility of
-  /// defining this destructor as the default destructor.
-  void DefineImplicitDestructor(SourceLocation CurrentLocation,
-                                CXXDestructorDecl *Destructor);
-
-  /// \brief Declare the implicit copy constructor for the given class.
-  ///
-  /// \param S The scope of the class, which may be NULL if this is a 
-  /// template instantiation.
-  ///
-  /// \param ClassDecl The class declaration into which the implicit 
-  /// copy constructor will be added.
-  ///
-  /// \returns The implicitly-declared copy constructor.
-  CXXConstructorDecl *DeclareImplicitCopyConstructor(CXXRecordDecl *ClassDecl);
-                                                     
-  /// DefineImplicitCopyConstructor - Checks for feasibility of
-  /// defining this constructor as the copy constructor.
-  void DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
-                                     CXXConstructorDecl *Constructor,
-                                     unsigned TypeQuals);
-
-  /// \brief Declare the implicit copy assignment operator for the given class.
-  ///
-  /// \param S The scope of the class, which may be NULL if this is a 
-  /// template instantiation.
-  ///
-  /// \param ClassDecl The class declaration into which the implicit 
-  /// copy-assignment operator will be added.
-  ///
-  /// \returns The implicitly-declared copy assignment operator.
-  CXXMethodDecl *DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl);
-  
-  /// \brief Defined an implicitly-declared copy assignment operator.
-  void DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
-                                    CXXMethodDecl *MethodDecl);
-
-  /// \brief Force the declaration of any implicitly-declared members of this
-  /// class.
-  void ForceDeclarationOfImplicitMembers(CXXRecordDecl *Class);
-  
-  /// MaybeBindToTemporary - If the passed in expression has a record type with
-  /// a non-trivial destructor, this will return CXXBindTemporaryExpr. Otherwise
-  /// it simply returns the passed in expression.
-  OwningExprResult MaybeBindToTemporary(Expr *E);
-
-  bool CompleteConstructorCall(CXXConstructorDecl *Constructor,
-                               MultiExprArg ArgsPtr,
-                               SourceLocation Loc,
-                      ASTOwningVector<&ActionBase::DeleteExpr> &ConvertedArgs);
-
-  virtual TypeTy *getDestructorName(SourceLocation TildeLoc,
-                                    IdentifierInfo &II, SourceLocation NameLoc,
-                                    Scope *S, CXXScopeSpec &SS,
-                                    TypeTy *ObjectType,
-                                    bool EnteringContext);
-
-  /// ActOnCXXNamedCast - Parse {dynamic,static,reinterpret,const}_cast's.
-  virtual OwningExprResult ActOnCXXNamedCast(SourceLocation OpLoc,
-                                             tok::TokenKind Kind,
-                                             SourceLocation LAngleBracketLoc,
-                                             TypeTy *Ty,
-                                             SourceLocation RAngleBracketLoc,
-                                             SourceLocation LParenLoc,
-                                             ExprArg E,
-                                             SourceLocation RParenLoc);
-
-  OwningExprResult BuildCXXNamedCast(SourceLocation OpLoc,
-                                     tok::TokenKind Kind,
-                                     TypeSourceInfo *Ty,
-                                     ExprArg E,
-                                     SourceRange AngleBrackets,
-                                     SourceRange Parens);
-
-  OwningExprResult BuildCXXTypeId(QualType TypeInfoType,
-                                  SourceLocation TypeidLoc,
-                                  TypeSourceInfo *Operand,
-                                  SourceLocation RParenLoc);
-  OwningExprResult BuildCXXTypeId(QualType TypeInfoType,
-                                  SourceLocation TypeidLoc,
-                                  ExprArg Operand,
-                                  SourceLocation RParenLoc);
-
-  /// ActOnCXXTypeid - Parse typeid( something ).
-  virtual OwningExprResult ActOnCXXTypeid(SourceLocation OpLoc,
-                                          SourceLocation LParenLoc, bool isType,
-                                          void *TyOrExpr,
-                                          SourceLocation RParenLoc);
-
-  //// ActOnCXXThis -  Parse 'this' pointer.
-  virtual OwningExprResult ActOnCXXThis(SourceLocation ThisLoc);
-
-  /// ActOnCXXBoolLiteral - Parse {true,false} literals.
-  virtual OwningExprResult ActOnCXXBoolLiteral(SourceLocation OpLoc,
-                                               tok::TokenKind Kind);
-
-  /// ActOnCXXNullPtrLiteral - Parse 'nullptr'.
-  virtual OwningExprResult ActOnCXXNullPtrLiteral(SourceLocation Loc);
-
-  //// ActOnCXXThrow -  Parse throw expressions.
-  virtual OwningExprResult ActOnCXXThrow(SourceLocation OpLoc,
-                                         ExprArg expr);
-  bool CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E);
-
-  /// ActOnCXXTypeConstructExpr - Parse construction of a specified type.
-  /// Can be interpreted either as function-style casting ("int(x)")
-  /// or class type construction ("ClassType(x,y,z)")
-  /// or creation of a value-initialized type ("int()").
-  virtual OwningExprResult ActOnCXXTypeConstructExpr(SourceRange TypeRange,
-                                                     TypeTy *TypeRep,
-                                                     SourceLocation LParenLoc,
-                                                     MultiExprArg Exprs,
-                                                     SourceLocation *CommaLocs,
-                                                     SourceLocation RParenLoc);
-
-  /// ActOnCXXNew - Parsed a C++ 'new' expression.
-  virtual OwningExprResult ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
-                                       SourceLocation PlacementLParen,
-                                       MultiExprArg PlacementArgs,
-                                       SourceLocation PlacementRParen,
-                                       SourceRange TypeIdParens, Declarator &D,
-                                       SourceLocation ConstructorLParen,
-                                       MultiExprArg ConstructorArgs,
-                                       SourceLocation ConstructorRParen);
-  OwningExprResult BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
-                               SourceLocation PlacementLParen,
-                               MultiExprArg PlacementArgs,
-                               SourceLocation PlacementRParen,
-                               SourceRange TypeIdParens,
-                               QualType AllocType,
-                               SourceLocation TypeLoc,
-                               SourceRange TypeRange,
-                               ExprArg ArraySize,
-                               SourceLocation ConstructorLParen,
-                               MultiExprArg ConstructorArgs,
-                               SourceLocation ConstructorRParen);
-
-  bool CheckAllocatedType(QualType AllocType, SourceLocation Loc,
-                          SourceRange R);
-  bool FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
-                               bool UseGlobal, QualType AllocType, bool IsArray,
-                               Expr **PlaceArgs, unsigned NumPlaceArgs,
-                               FunctionDecl *&OperatorNew,
-                               FunctionDecl *&OperatorDelete);
-  bool FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
-                              DeclarationName Name, Expr** Args,
-                              unsigned NumArgs, DeclContext *Ctx,
-                              bool AllowMissing, FunctionDecl *&Operator);
-  void DeclareGlobalNewDelete();
-  void DeclareGlobalAllocationFunction(DeclarationName Name, QualType Return,
-                                       QualType Argument,
-                                       bool addMallocAttr = false);
-
-  bool FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
-                                DeclarationName Name, FunctionDecl* &Operator);
-
-  /// ActOnCXXDelete - Parsed a C++ 'delete' expression
-  virtual OwningExprResult ActOnCXXDelete(SourceLocation StartLoc,
-                                          bool UseGlobal, bool ArrayForm,
-                                          ExprArg Operand);
-
-  virtual DeclResult ActOnCXXConditionDeclaration(Scope *S,
-                                                  Declarator &D);
-  OwningExprResult CheckConditionVariable(VarDecl *ConditionVar,
-                                          SourceLocation StmtLoc,
-                                          bool ConvertToBoolean);
-
-  /// ActOnUnaryTypeTrait - Parsed one of the unary type trait support
-  /// pseudo-functions.
-  virtual OwningExprResult ActOnUnaryTypeTrait(UnaryTypeTrait OTT,
-                                               SourceLocation KWLoc,
-                                               SourceLocation LParen,
-                                               TypeTy *Ty,
-                                               SourceLocation RParen);
-
-  virtual OwningExprResult ActOnStartCXXMemberReference(Scope *S,
-                                                        ExprArg Base,
-                                                        SourceLocation OpLoc,
-                                                        tok::TokenKind OpKind,
-                                                        TypeTy *&ObjectType,
-                                                   bool &MayBePseudoDestructor);
-
-  OwningExprResult DiagnoseDtorReference(SourceLocation NameLoc,
-                                         ExprArg MemExpr);
-
-  OwningExprResult BuildPseudoDestructorExpr(ExprArg Base,
-                                             SourceLocation OpLoc,
-                                             tok::TokenKind OpKind,
-                                             const CXXScopeSpec &SS,
-                                             TypeSourceInfo *ScopeType,
-                                             SourceLocation CCLoc,
-                                             SourceLocation TildeLoc,
-                                     PseudoDestructorTypeStorage DestroyedType,
-                                             bool HasTrailingLParen);
-
-  virtual OwningExprResult ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
-                                                     SourceLocation OpLoc,
-                                                     tok::TokenKind OpKind,
-                                                     CXXScopeSpec &SS,
-                                                   UnqualifiedId &FirstTypeName,
-                                                     SourceLocation CCLoc,
-                                                     SourceLocation TildeLoc,
-                                                  UnqualifiedId &SecondTypeName,
-                                                     bool HasTrailingLParen);
-
-  /// MaybeCreateCXXExprWithTemporaries - If the list of temporaries is
-  /// non-empty, will create a new CXXExprWithTemporaries expression.
-  /// Otherwise, just returs the passed in expression.
-  Expr *MaybeCreateCXXExprWithTemporaries(Expr *SubExpr);
-  OwningExprResult MaybeCreateCXXExprWithTemporaries(OwningExprResult SubExpr);
-  FullExpr CreateFullExpr(Expr *SubExpr);
-
-  virtual OwningExprResult ActOnFinishFullExpr(ExprArg Expr);
-
-  // Marks SS invalid if it represents an incomplete type.
-  bool RequireCompleteDeclContext(CXXScopeSpec &SS, DeclContext *DC);
-
-  DeclContext *computeDeclContext(QualType T);
-  DeclContext *computeDeclContext(const CXXScopeSpec &SS,
-                                  bool EnteringContext = false);
-  bool isDependentScopeSpecifier(const CXXScopeSpec &SS);
-  CXXRecordDecl *getCurrentInstantiationOf(NestedNameSpecifier *NNS);
-  bool isUnknownSpecialization(const CXXScopeSpec &SS);
-
-  /// ActOnCXXGlobalScopeSpecifier - Return the object that represents the
-  /// global scope ('::').
-  virtual CXXScopeTy *ActOnCXXGlobalScopeSpecifier(Scope *S,
-                                                   SourceLocation CCLoc);
-
-  bool isAcceptableNestedNameSpecifier(NamedDecl *SD);
-  NamedDecl *FindFirstQualifierInScope(Scope *S, NestedNameSpecifier *NNS);
-
-  virtual bool isNonTypeNestedNameSpecifier(Scope *S, CXXScopeSpec &SS,
-                                            SourceLocation IdLoc,
-                                            IdentifierInfo &II,
-                                            TypeTy *ObjectType);
-
-  CXXScopeTy *BuildCXXNestedNameSpecifier(Scope *S,
-                                          CXXScopeSpec &SS,
-                                          SourceLocation IdLoc,
-                                          SourceLocation CCLoc,
-                                          IdentifierInfo &II,
-                                          QualType ObjectType,
-                                          NamedDecl *ScopeLookupResult,
-                                          bool EnteringContext,
-                                          bool ErrorRecoveryLookup);
-
-  virtual CXXScopeTy *ActOnCXXNestedNameSpecifier(Scope *S,
-                                                  CXXScopeSpec &SS,
-                                                  SourceLocation IdLoc,
-                                                  SourceLocation CCLoc,
-                                                  IdentifierInfo &II,
-                                                  TypeTy *ObjectType,
-                                                  bool EnteringContext);
-
-  virtual bool IsInvalidUnlessNestedName(Scope *S,
-                                         CXXScopeSpec &SS,
-                                         IdentifierInfo &II,
-                                         TypeTy *ObjectType,
-                                         bool EnteringContext);
-
-  /// ActOnCXXNestedNameSpecifier - Called during parsing of a
-  /// nested-name-specifier that involves a template-id, e.g.,
-  /// "foo::bar<int, float>::", and now we need to build a scope
-  /// specifier. \p SS is empty or the previously parsed nested-name
-  /// part ("foo::"), \p Type is the already-parsed class template
-  /// specialization (or other template-id that names a type), \p
-  /// TypeRange is the source range where the type is located, and \p
-  /// CCLoc is the location of the trailing '::'.
-  virtual CXXScopeTy *ActOnCXXNestedNameSpecifier(Scope *S,
-                                                  const CXXScopeSpec &SS,
-                                                  TypeTy *Type,
-                                                  SourceRange TypeRange,
-                                                  SourceLocation CCLoc);
-
-  virtual bool ShouldEnterDeclaratorScope(Scope *S, const CXXScopeSpec &SS);
-
-  /// ActOnCXXEnterDeclaratorScope - Called when a C++ scope specifier (global
-  /// scope or nested-name-specifier) is parsed, part of a declarator-id.
-  /// After this method is called, according to [C++ 3.4.3p3], names should be
-  /// looked up in the declarator-id's scope, until the declarator is parsed and
-  /// ActOnCXXExitDeclaratorScope is called.
-  /// The 'SS' should be a non-empty valid CXXScopeSpec.
-  virtual bool ActOnCXXEnterDeclaratorScope(Scope *S, CXXScopeSpec &SS);
-
-  /// ActOnCXXExitDeclaratorScope - Called when a declarator that previously
-  /// invoked ActOnCXXEnterDeclaratorScope(), is finished. 'SS' is the same
-  /// CXXScopeSpec that was passed to ActOnCXXEnterDeclaratorScope as well.
-  /// Used to indicate that names should revert to being looked up in the
-  /// defining scope.
-  virtual void ActOnCXXExitDeclaratorScope(Scope *S, const CXXScopeSpec &SS);
-
-  /// ActOnCXXEnterDeclInitializer - Invoked when we are about to parse an
-  /// initializer for the declaration 'Dcl'.
-  /// After this method is called, according to [C++ 3.4.1p13], if 'Dcl' is a
-  /// static data member of class X, names should be looked up in the scope of
-  /// class X.
-  virtual void ActOnCXXEnterDeclInitializer(Scope *S, DeclPtrTy Dcl);
-
-  /// ActOnCXXExitDeclInitializer - Invoked after we are finished parsing an
-  /// initializer for the declaration 'Dcl'.
-  virtual void ActOnCXXExitDeclInitializer(Scope *S, DeclPtrTy Dcl);
-
-  // ParseObjCStringLiteral - Parse Objective-C string literals.
-  virtual ExprResult ParseObjCStringLiteral(SourceLocation *AtLocs,
-                                            ExprTy **Strings,
-                                            unsigned NumStrings);
-
-  Expr *BuildObjCEncodeExpression(SourceLocation AtLoc,
-                                  TypeSourceInfo *EncodedTypeInfo,
-                                  SourceLocation RParenLoc);
-  CXXMemberCallExpr *BuildCXXMemberCallExpr(Expr *Exp,
-                                            NamedDecl *FoundDecl,
-                                            CXXMethodDecl *Method);
-
-  virtual ExprResult ParseObjCEncodeExpression(SourceLocation AtLoc,
-                                               SourceLocation EncodeLoc,
-                                               SourceLocation LParenLoc,
-                                               TypeTy *Ty,
-                                               SourceLocation RParenLoc);
-
-  // ParseObjCSelectorExpression - Build selector expression for @selector
-  virtual ExprResult ParseObjCSelectorExpression(Selector Sel,
-                                                 SourceLocation AtLoc,
-                                                 SourceLocation SelLoc,
-                                                 SourceLocation LParenLoc,
-                                                 SourceLocation RParenLoc);
-
-  // ParseObjCProtocolExpression - Build protocol expression for @protocol
-  virtual ExprResult ParseObjCProtocolExpression(IdentifierInfo * ProtocolName,
-                                                 SourceLocation AtLoc,
-                                                 SourceLocation ProtoLoc,
-                                                 SourceLocation LParenLoc,
-                                                 SourceLocation RParenLoc);
-
-  //===--------------------------------------------------------------------===//
-  // C++ Declarations
-  //
-  virtual DeclPtrTy ActOnStartLinkageSpecification(Scope *S,
-                                                   SourceLocation ExternLoc,
-                                                   SourceLocation LangLoc,
-                                                   llvm::StringRef Lang,
-                                                   SourceLocation LBraceLoc);
-  virtual DeclPtrTy ActOnFinishLinkageSpecification(Scope *S,
-                                                    DeclPtrTy LinkageSpec,
-                                                    SourceLocation RBraceLoc);
-
-
-  //===--------------------------------------------------------------------===//
-  // C++ Classes
-  //
-  virtual bool isCurrentClassName(const IdentifierInfo &II, Scope *S,
-                                  const CXXScopeSpec *SS);
-
-  virtual DeclPtrTy ActOnAccessSpecifier(AccessSpecifier Access,
-                                         SourceLocation ASLoc,
-                                         SourceLocation ColonLoc);
-
-  virtual DeclPtrTy ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS,
-                                             Declarator &D,
-                                 MultiTemplateParamsArg TemplateParameterLists,
-                                             ExprTy *BitfieldWidth,
-                                             ExprTy *Init, bool IsDefinition,
-                                             bool Deleted = false);
-
-  virtual MemInitResult ActOnMemInitializer(DeclPtrTy ConstructorD,
-                                            Scope *S,
-                                            CXXScopeSpec &SS,
-                                            IdentifierInfo *MemberOrBase,
-                                            TypeTy *TemplateTypeTy,
-                                            SourceLocation IdLoc,
-                                            SourceLocation LParenLoc,
-                                            ExprTy **Args, unsigned NumArgs,
-                                            SourceLocation *CommaLocs,
-                                            SourceLocation RParenLoc);
-
-  MemInitResult BuildMemberInitializer(FieldDecl *Member, Expr **Args,
-                                       unsigned NumArgs, SourceLocation IdLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation RParenLoc);
-
-  MemInitResult BuildBaseInitializer(QualType BaseType,
-                                     TypeSourceInfo *BaseTInfo,
-                                     Expr **Args, unsigned NumArgs,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation RParenLoc,
-                                     CXXRecordDecl *ClassDecl);
-
-  bool SetBaseOrMemberInitializers(CXXConstructorDecl *Constructor,
-                                   CXXBaseOrMemberInitializer **Initializers,
-                                   unsigned NumInitializers, bool AnyErrors);
-  
-  void SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation);
-                           
-
-  /// MarkBaseAndMemberDestructorsReferenced - Given a record decl,
-  /// mark all the non-trivial destructors of its members and bases as
-  /// referenced.
-  void MarkBaseAndMemberDestructorsReferenced(SourceLocation Loc,
-                                              CXXRecordDecl *Record);
-
-  /// \brief The list of classes whose vtables have been used within
-  /// this translation unit, and the source locations at which the
-  /// first use occurred.
-  llvm::SmallVector<std::pair<CXXRecordDecl *, SourceLocation>, 16> 
-    VTableUses;
-
-  /// \brief The set of classes whose vtables have been used within
-  /// this translation unit, and a bit that will be true if the vtable is
-  /// required to be emitted (otherwise, it should be emitted only if needed
-  /// by code generation).
-  llvm::DenseMap<CXXRecordDecl *, bool> VTablesUsed;
-
-  /// \brief A list of all of the dynamic classes in this translation
-  /// unit.
-  llvm::SmallVector<CXXRecordDecl *, 16> DynamicClasses;
-
-  /// \brief Note that the vtable for the given class was used at the
-  /// given location.
-  void MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class,
-                      bool DefinitionRequired = false);
-
-  /// MarkVirtualMembersReferenced - Will mark all virtual members of the given
-  /// CXXRecordDecl referenced.
-  void MarkVirtualMembersReferenced(SourceLocation Loc,
-                                    const CXXRecordDecl *RD);
-
-  /// \brief Define all of the vtables that have been used in this
-  /// translation unit and reference any virtual members used by those
-  /// vtables.
-  ///
-  /// \returns true if any work was done, false otherwise.
-  bool DefineUsedVTables();
-
-  void AddImplicitlyDeclaredMembersToClass(CXXRecordDecl *ClassDecl);
-
-  virtual void ActOnMemInitializers(DeclPtrTy ConstructorDecl,
-                                    SourceLocation ColonLoc,
-                                    MemInitTy **MemInits, unsigned NumMemInits,
-                                    bool AnyErrors);
-
-  void CheckCompletedCXXClass(CXXRecordDecl *Record);
-  virtual void ActOnFinishCXXMemberSpecification(Scope* S, SourceLocation RLoc,
-                                                 DeclPtrTy TagDecl,
-                                                 SourceLocation LBrac,
-                                                 SourceLocation RBrac,
-                                                 AttributeList *AttrList);
-
-  virtual void ActOnReenterTemplateScope(Scope *S, DeclPtrTy Template);
-  virtual void ActOnStartDelayedMemberDeclarations(Scope *S,
-                                                   DeclPtrTy Record);
-  virtual void ActOnStartDelayedCXXMethodDeclaration(Scope *S,
-                                                     DeclPtrTy Method);
-  virtual void ActOnDelayedCXXMethodParameter(Scope *S, DeclPtrTy Param);
-  virtual void ActOnFinishDelayedCXXMethodDeclaration(Scope *S,
-                                                      DeclPtrTy Method);
-  virtual void ActOnFinishDelayedMemberDeclarations(Scope *S,
-                                                    DeclPtrTy Record);
-
-  virtual DeclPtrTy ActOnStaticAssertDeclaration(SourceLocation AssertLoc,
-                                                 ExprArg AssertExpr,
-                                                 ExprArg AssertMessageExpr);
-
-  FriendDecl *CheckFriendTypeDecl(SourceLocation FriendLoc,
-                                  TypeSourceInfo *TSInfo);
-  DeclPtrTy ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
-                                MultiTemplateParamsArg TemplateParams);
-  DeclPtrTy ActOnFriendFunctionDecl(Scope *S, Declarator &D, bool IsDefinition,
-                                    MultiTemplateParamsArg TemplateParams);
-
-  QualType CheckConstructorDeclarator(Declarator &D, QualType R,
-                                      FunctionDecl::StorageClass& SC);
-  void CheckConstructor(CXXConstructorDecl *Constructor);
-  QualType CheckDestructorDeclarator(Declarator &D, QualType R,
-                                     FunctionDecl::StorageClass& SC);
-  bool CheckDestructor(CXXDestructorDecl *Destructor);
-  void CheckConversionDeclarator(Declarator &D, QualType &R,
-                                 FunctionDecl::StorageClass& SC);
-  DeclPtrTy ActOnConversionDeclarator(CXXConversionDecl *Conversion);
-
-  //===--------------------------------------------------------------------===//
-  // C++ Derived Classes
-  //
-
-  /// ActOnBaseSpecifier - Parsed a base specifier
-  CXXBaseSpecifier *CheckBaseSpecifier(CXXRecordDecl *Class,
-                                       SourceRange SpecifierRange,
-                                       bool Virtual, AccessSpecifier Access,
-                                       QualType BaseType,
-                                       SourceLocation BaseLoc);
-
-  /// SetClassDeclAttributesFromBase - Copies class decl traits
-  /// (such as whether the class has a trivial constructor,
-  /// trivial destructor etc) from the given base class.
-  void SetClassDeclAttributesFromBase(CXXRecordDecl *Class,
-                                      const CXXRecordDecl *BaseClass,
-                                      bool BaseIsVirtual);
-
-  virtual BaseResult ActOnBaseSpecifier(DeclPtrTy classdecl,
-                                        SourceRange SpecifierRange,
-                                        bool Virtual, AccessSpecifier Access,
-                                        TypeTy *basetype, SourceLocation
-                                        BaseLoc);
-
-  bool AttachBaseSpecifiers(CXXRecordDecl *Class, CXXBaseSpecifier **Bases,
-                            unsigned NumBases);
-  virtual void ActOnBaseSpecifiers(DeclPtrTy ClassDecl, BaseTy **Bases,
-                                   unsigned NumBases);
-
-  bool IsDerivedFrom(QualType Derived, QualType Base);
-  bool IsDerivedFrom(QualType Derived, QualType Base, CXXBasePaths &Paths);
-
-  // FIXME: I don't like this name.
-  void BuildBasePathArray(const CXXBasePaths &Paths,
-                          CXXBaseSpecifierArray &BasePath);
-
-  bool BasePathInvolvesVirtualBase(const CXXBaseSpecifierArray &BasePath);
-
-  bool CheckDerivedToBaseConversion(QualType Derived, QualType Base,
-                                    SourceLocation Loc, SourceRange Range,
-                                    CXXBaseSpecifierArray *BasePath = 0,
-                                    bool IgnoreAccess = false);
-  bool CheckDerivedToBaseConversion(QualType Derived, QualType Base,
-                                    unsigned InaccessibleBaseID,
-                                    unsigned AmbigiousBaseConvID,
-                                    SourceLocation Loc, SourceRange Range,
-                                    DeclarationName Name,
-                                    CXXBaseSpecifierArray *BasePath);
-
-  std::string getAmbiguousPathsDisplayString(CXXBasePaths &Paths);
-
-  /// CheckOverridingFunctionReturnType - Checks whether the return types are
-  /// covariant, according to C++ [class.virtual]p5.
-  bool CheckOverridingFunctionReturnType(const CXXMethodDecl *New,
-                                         const CXXMethodDecl *Old);
-
-  /// CheckOverridingFunctionExceptionSpec - Checks whether the exception
-  /// spec is a subset of base spec.
-  bool CheckOverridingFunctionExceptionSpec(const CXXMethodDecl *New,
-                                            const CXXMethodDecl *Old);
-
-  /// CheckOverridingFunctionAttributes - Checks whether attributes are
-  /// incompatible or prevent overriding.
-  bool CheckOverridingFunctionAttributes(const CXXMethodDecl *New,
-                                         const CXXMethodDecl *Old);
-
-  bool CheckPureMethod(CXXMethodDecl *Method, SourceRange InitRange);
-
-  //===--------------------------------------------------------------------===//
-  // C++ Access Control
-  //
-
-  enum AccessResult {
-    AR_accessible,
-    AR_inaccessible,
-    AR_dependent,
-    AR_delayed
-  };
-
-  bool SetMemberAccessSpecifier(NamedDecl *MemberDecl,
-                                NamedDecl *PrevMemberDecl,
-                                AccessSpecifier LexicalAS);
-
-  AccessResult CheckUnresolvedMemberAccess(UnresolvedMemberExpr *E,
-                                           DeclAccessPair FoundDecl);
-  AccessResult CheckUnresolvedLookupAccess(UnresolvedLookupExpr *E,
-                                           DeclAccessPair FoundDecl);
-  AccessResult CheckAllocationAccess(SourceLocation OperatorLoc,
-                                     SourceRange PlacementRange,
-                                     CXXRecordDecl *NamingClass,
-                                     DeclAccessPair FoundDecl);
-  AccessResult CheckConstructorAccess(SourceLocation Loc,
-                                      CXXConstructorDecl *D,
-                                      const InitializedEntity &Entity,
-                                      AccessSpecifier Access,
-                                      bool IsCopyBindingRefToTemp = false);
-  AccessResult CheckDestructorAccess(SourceLocation Loc,
-                                     CXXDestructorDecl *Dtor,
-                                     const PartialDiagnostic &PDiag);
-  AccessResult CheckDirectMemberAccess(SourceLocation Loc,
-                                       NamedDecl *D,
-                                       const PartialDiagnostic &PDiag);
-  AccessResult CheckMemberOperatorAccess(SourceLocation Loc,
-                                         Expr *ObjectExpr,
-                                         Expr *ArgExpr,
-                                         DeclAccessPair FoundDecl);
-  AccessResult CheckAddressOfMemberAccess(Expr *OvlExpr,
-                                          DeclAccessPair FoundDecl);
-  AccessResult CheckBaseClassAccess(SourceLocation AccessLoc,
-                                    QualType Base, QualType Derived,
-                                    const CXXBasePath &Path,
-                                    unsigned DiagID,
-                                    bool ForceCheck = false,
-                                    bool ForceUnprivileged = false);
-  void CheckLookupAccess(const LookupResult &R);
-
-  void HandleDependentAccessCheck(const DependentDiagnostic &DD,
-                         const MultiLevelTemplateArgumentList &TemplateArgs);
-  void PerformDependentDiagnostics(const DeclContext *Pattern,
-                        const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  void HandleDelayedAccessCheck(DelayedDiagnostic &DD, Decl *Ctx);
-
-  /// A flag to suppress access checking.
-  bool SuppressAccessChecking;
-
-  void ActOnStartSuppressingAccessChecks();
-  void ActOnStopSuppressingAccessChecks();
-
-  enum AbstractDiagSelID {
-    AbstractNone = -1,
-    AbstractReturnType,
-    AbstractParamType,
-    AbstractVariableType,
-    AbstractFieldType
-  };
-
-  bool RequireNonAbstractType(SourceLocation Loc, QualType T,
-                              const PartialDiagnostic &PD,
-                              const CXXRecordDecl *CurrentRD = 0);
-
-  bool RequireNonAbstractType(SourceLocation Loc, QualType T, unsigned DiagID,
-                              AbstractDiagSelID SelID = AbstractNone,
-                              const CXXRecordDecl *CurrentRD = 0);
-
-  //===--------------------------------------------------------------------===//
-  // C++ Overloaded Operators [C++ 13.5]
-  //
-
-  bool CheckOverloadedOperatorDeclaration(FunctionDecl *FnDecl);
-
-  bool CheckLiteralOperatorDeclaration(FunctionDecl *FnDecl);
-
-  //===--------------------------------------------------------------------===//
-  // C++ Templates [C++ 14]
-  //
-  void LookupTemplateName(LookupResult &R, Scope *S, CXXScopeSpec &SS,
-                          QualType ObjectType, bool EnteringContext,
-                          bool &MemberOfUnknownSpecialization);
-
-  virtual TemplateNameKind isTemplateName(Scope *S,
-                                          CXXScopeSpec &SS,
-                                          UnqualifiedId &Name,
-                                          TypeTy *ObjectType,
-                                          bool EnteringContext,
-                                          TemplateTy &Template,
-                                          bool &MemberOfUnknownSpecialization);
-
-  virtual bool DiagnoseUnknownTemplateName(const IdentifierInfo &II,
-                                           SourceLocation IILoc,
-                                           Scope *S,
-                                           const CXXScopeSpec *SS,
-                                           TemplateTy &SuggestedTemplate,
-                                           TemplateNameKind &SuggestedKind);
-
-  bool DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl);
-  TemplateDecl *AdjustDeclIfTemplate(DeclPtrTy &Decl);
-
-  virtual DeclPtrTy ActOnTypeParameter(Scope *S, bool Typename, bool Ellipsis,
-                                       SourceLocation EllipsisLoc,
-                                       SourceLocation KeyLoc,
-                                       IdentifierInfo *ParamName,
-                                       SourceLocation ParamNameLoc,
-                                       unsigned Depth, unsigned Position,
-                                       SourceLocation EqualLoc,
-                                       TypeTy *DefaultArg);
-
-  QualType CheckNonTypeTemplateParameterType(QualType T, SourceLocation Loc);
-  virtual DeclPtrTy ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
-                                                  unsigned Depth,
-                                                  unsigned Position,
-                                                  SourceLocation EqualLoc,
-                                                  ExprArg DefaultArg);
-  virtual DeclPtrTy ActOnTemplateTemplateParameter(Scope *S,
-                                                   SourceLocation TmpLoc,
-                                                   TemplateParamsTy *Params,
-                                                   IdentifierInfo *ParamName,
-                                                   SourceLocation ParamNameLoc,
-                                                   unsigned Depth,
-                                                   unsigned Position,
-                                                   SourceLocation EqualLoc,
-                                     const ParsedTemplateArgument &DefaultArg);
-
-  virtual TemplateParamsTy *
-  ActOnTemplateParameterList(unsigned Depth,
-                             SourceLocation ExportLoc,
-                             SourceLocation TemplateLoc,
-                             SourceLocation LAngleLoc,
-                             DeclPtrTy *Params, unsigned NumParams,
-                             SourceLocation RAngleLoc);
-
-  /// \brief The context in which we are checking a template parameter
-  /// list.
-  enum TemplateParamListContext {
-    TPC_ClassTemplate,
-    TPC_FunctionTemplate,
-    TPC_ClassTemplateMember,
-    TPC_FriendFunctionTemplate
-  };
-
-  bool CheckTemplateParameterList(TemplateParameterList *NewParams,
-                                  TemplateParameterList *OldParams,
-                                  TemplateParamListContext TPC);
-  TemplateParameterList *
-  MatchTemplateParametersToScopeSpecifier(SourceLocation DeclStartLoc,
-                                          const CXXScopeSpec &SS,
-                                          TemplateParameterList **ParamLists,
-                                          unsigned NumParamLists,
-                                          bool IsFriend,
-                                          bool &IsExplicitSpecialization,
-                                          bool &Invalid);
-
-  DeclResult CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
-                                SourceLocation KWLoc, CXXScopeSpec &SS,
-                                IdentifierInfo *Name, SourceLocation NameLoc,
-                                AttributeList *Attr,
-                                TemplateParameterList *TemplateParams,
-                                AccessSpecifier AS);
-
-  void translateTemplateArguments(const ASTTemplateArgsPtr &In,
-                                  TemplateArgumentListInfo &Out);
-
-  QualType CheckTemplateIdType(TemplateName Template,
-                               SourceLocation TemplateLoc,
-                               const TemplateArgumentListInfo &TemplateArgs);
-
-  virtual TypeResult
-  ActOnTemplateIdType(TemplateTy Template, SourceLocation TemplateLoc,
-                      SourceLocation LAngleLoc,
-                      ASTTemplateArgsPtr TemplateArgs,
-                      SourceLocation RAngleLoc);
-
-  virtual TypeResult ActOnTagTemplateIdType(TypeResult Type,
-                                            TagUseKind TUK,
-                                            DeclSpec::TST TagSpec,
-                                            SourceLocation TagLoc);
-
-  OwningExprResult BuildTemplateIdExpr(const CXXScopeSpec &SS,
-                                       LookupResult &R,
-                                       bool RequiresADL,
-                               const TemplateArgumentListInfo &TemplateArgs);
-  OwningExprResult BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS,
-                                                DeclarationName Name,
-                                                SourceLocation NameLoc,
-                               const TemplateArgumentListInfo &TemplateArgs);
-
-  virtual TemplateNameKind ActOnDependentTemplateName(Scope *S,
-                                                  SourceLocation TemplateKWLoc,
-                                                      CXXScopeSpec &SS,
-                                                      UnqualifiedId &Name,
-                                                      TypeTy *ObjectType,
-                                                      bool EnteringContext,
-                                                      TemplateTy &Template);
-
-  bool CheckClassTemplatePartialSpecializationArgs(
-                                        TemplateParameterList *TemplateParams,
-                              const TemplateArgumentListBuilder &TemplateArgs,
-                                        bool &MirrorsPrimaryTemplate);
-
-  virtual DeclResult
-  ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec, TagUseKind TUK,
-                                   SourceLocation KWLoc,
-                                   CXXScopeSpec &SS,
-                                   TemplateTy Template,
-                                   SourceLocation TemplateNameLoc,
-                                   SourceLocation LAngleLoc,
-                                   ASTTemplateArgsPtr TemplateArgs,
-                                   SourceLocation RAngleLoc,
-                                   AttributeList *Attr,
-                                 MultiTemplateParamsArg TemplateParameterLists);
-
-  virtual DeclPtrTy ActOnTemplateDeclarator(Scope *S,
-                                  MultiTemplateParamsArg TemplateParameterLists,
-                                            Declarator &D);
-
-  virtual DeclPtrTy ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
-                                  MultiTemplateParamsArg TemplateParameterLists,
-                                                    Declarator &D);
-
-  bool
-  CheckSpecializationInstantiationRedecl(SourceLocation NewLoc,
-                                         TemplateSpecializationKind NewTSK,
-                                         NamedDecl *PrevDecl,
-                                         TemplateSpecializationKind PrevTSK,
-                                         SourceLocation PrevPtOfInstantiation,
-                                         bool &SuppressNew);
-
-  bool CheckDependentFunctionTemplateSpecialization(FunctionDecl *FD,
-                    const TemplateArgumentListInfo &ExplicitTemplateArgs,
-                                                    LookupResult &Previous);
-
-  bool CheckFunctionTemplateSpecialization(FunctionDecl *FD,
-                        const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                                           LookupResult &Previous);
-  bool CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous);
-
-  virtual DeclResult
-  ActOnExplicitInstantiation(Scope *S,
-                             SourceLocation ExternLoc,
-                             SourceLocation TemplateLoc,
-                             unsigned TagSpec,
-                             SourceLocation KWLoc,
-                             const CXXScopeSpec &SS,
-                             TemplateTy Template,
-                             SourceLocation TemplateNameLoc,
-                             SourceLocation LAngleLoc,
-                             ASTTemplateArgsPtr TemplateArgs,
-                             SourceLocation RAngleLoc,
-                             AttributeList *Attr);
-
-  virtual DeclResult
-  ActOnExplicitInstantiation(Scope *S,
-                             SourceLocation ExternLoc,
-                             SourceLocation TemplateLoc,
-                             unsigned TagSpec,
-                             SourceLocation KWLoc,
-                             CXXScopeSpec &SS,
-                             IdentifierInfo *Name,
-                             SourceLocation NameLoc,
-                             AttributeList *Attr);
-
-  virtual DeclResult ActOnExplicitInstantiation(Scope *S,
-                                                SourceLocation ExternLoc,
-                                                SourceLocation TemplateLoc,
-                                                Declarator &D);
-
-  TemplateArgumentLoc
-  SubstDefaultTemplateArgumentIfAvailable(TemplateDecl *Template,
-                                          SourceLocation TemplateLoc,
-                                          SourceLocation RAngleLoc,
-                                          Decl *Param,
-                                      TemplateArgumentListBuilder &Converted);
-
-  /// \brief Specifies the context in which a particular template
-  /// argument is being checked.
-  enum CheckTemplateArgumentKind {
-    /// \brief The template argument was specified in the code or was
-    /// instantiated with some deduced template arguments.
-    CTAK_Specified,
-
-    /// \brief The template argument was deduced via template argument
-    /// deduction.
-    CTAK_Deduced,
-
-    /// \brief The template argument was deduced from an array bound
-    /// via template argument deduction.
-    CTAK_DeducedFromArrayBound
-  };
-
-  bool CheckTemplateArgument(NamedDecl *Param,
-                             const TemplateArgumentLoc &Arg,
-                             TemplateDecl *Template,
-                             SourceLocation TemplateLoc,
-                             SourceLocation RAngleLoc,
-                             TemplateArgumentListBuilder &Converted,
-                             CheckTemplateArgumentKind CTAK = CTAK_Specified);
-
-  bool CheckTemplateArgumentList(TemplateDecl *Template,
-                                 SourceLocation TemplateLoc,
-                                 const TemplateArgumentListInfo &TemplateArgs,
-                                 bool PartialTemplateArgs,
-                                 TemplateArgumentListBuilder &Converted);
-
-  bool CheckTemplateTypeArgument(TemplateTypeParmDecl *Param,
-                                 const TemplateArgumentLoc &Arg,
-                                 TemplateArgumentListBuilder &Converted);
-
-  bool CheckTemplateArgument(TemplateTypeParmDecl *Param,
-                             TypeSourceInfo *Arg);
-  bool CheckTemplateArgumentPointerToMember(Expr *Arg,
-                                            TemplateArgument &Converted);
-  bool CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
-                             QualType InstantiatedParamType, Expr *&Arg,
-                             TemplateArgument &Converted,
-                             CheckTemplateArgumentKind CTAK = CTAK_Specified);
-  bool CheckTemplateArgument(TemplateTemplateParmDecl *Param,
-                             const TemplateArgumentLoc &Arg);
-
-  OwningExprResult
-  BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
-                                          QualType ParamType,
-                                          SourceLocation Loc);
-  OwningExprResult
-  BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
-                                              SourceLocation Loc);
-
-  /// \brief Enumeration describing how template parameter lists are compared
-  /// for equality.
-  enum TemplateParameterListEqualKind {
-    /// \brief We are matching the template parameter lists of two templates
-    /// that might be redeclarations.
-    ///
-    /// \code
-    /// template<typename T> struct X;
-    /// template<typename T> struct X;
-    /// \endcode
-    TPL_TemplateMatch,
-
-    /// \brief We are matching the template parameter lists of two template
-    /// template parameters as part of matching the template parameter lists
-    /// of two templates that might be redeclarations.
-    ///
-    /// \code
-    /// template<template<int I> class TT> struct X;
-    /// template<template<int Value> class Other> struct X;
-    /// \endcode
-    TPL_TemplateTemplateParmMatch,
-
-    /// \brief We are matching the template parameter lists of a template
-    /// template argument against the template parameter lists of a template
-    /// template parameter.
-    ///
-    /// \code
-    /// template<template<int Value> class Metafun> struct X;
-    /// template<int Value> struct integer_c;
-    /// X<integer_c> xic;
-    /// \endcode
-    TPL_TemplateTemplateArgumentMatch
-  };
-
-  bool TemplateParameterListsAreEqual(TemplateParameterList *New,
-                                      TemplateParameterList *Old,
-                                      bool Complain,
-                                      TemplateParameterListEqualKind Kind,
-                                      SourceLocation TemplateArgLoc
-                                        = SourceLocation());
-
-  bool CheckTemplateDeclScope(Scope *S, TemplateParameterList *TemplateParams);
-
-  /// \brief Called when the parser has parsed a C++ typename
-  /// specifier, e.g., "typename T::type".
-  ///
-  /// \param S The scope in which this typename type occurs.
-  /// \param TypenameLoc the location of the 'typename' keyword
-  /// \param SS the nested-name-specifier following the typename (e.g., 'T::').
-  /// \param II the identifier we're retrieving (e.g., 'type' in the example).
-  /// \param IdLoc the location of the identifier.
-  virtual TypeResult
-  ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, 
-                    const CXXScopeSpec &SS, const IdentifierInfo &II, 
-                    SourceLocation IdLoc);
-
-  /// \brief Called when the parser has parsed a C++ typename
-  /// specifier that ends in a template-id, e.g.,
-  /// "typename MetaFun::template apply<T1, T2>".
-  ///
-  /// \param S The scope in which this typename type occurs.
-  /// \param TypenameLoc the location of the 'typename' keyword
-  /// \param SS the nested-name-specifier following the typename (e.g., 'T::').
-  /// \param TemplateLoc the location of the 'template' keyword, if any.
-  /// \param Ty the type that the typename specifier refers to.
-  virtual TypeResult
-  ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, 
-                    const CXXScopeSpec &SS, SourceLocation TemplateLoc, 
-                    TypeTy *Ty);
-
-  QualType CheckTypenameType(ElaboratedTypeKeyword Keyword,
-                             NestedNameSpecifier *NNS,
-                             const IdentifierInfo &II,
-                             SourceLocation KeywordLoc,
-                             SourceRange NNSRange,
-                             SourceLocation IILoc);
-
-  TypeSourceInfo *RebuildTypeInCurrentInstantiation(TypeSourceInfo *T,
-                                                    SourceLocation Loc,
-                                                    DeclarationName Name);
-  bool RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS);
-
-  std::string
-  getTemplateArgumentBindingsText(const TemplateParameterList *Params,
-                                  const TemplateArgumentList &Args);
-
-  std::string
-  getTemplateArgumentBindingsText(const TemplateParameterList *Params,
-                                  const TemplateArgument *Args,
-                                  unsigned NumArgs);
-
-  /// \brief Describes the result of template argument deduction.
-  ///
-  /// The TemplateDeductionResult enumeration describes the result of
-  /// template argument deduction, as returned from
-  /// DeduceTemplateArguments(). The separate TemplateDeductionInfo
-  /// structure provides additional information about the results of
-  /// template argument deduction, e.g., the deduced template argument
-  /// list (if successful) or the specific template parameters or
-  /// deduced arguments that were involved in the failure.
-  enum TemplateDeductionResult {
-    /// \brief Template argument deduction was successful.
-    TDK_Success = 0,
-    /// \brief Template argument deduction exceeded the maximum template
-    /// instantiation depth (which has already been diagnosed).
-    TDK_InstantiationDepth,
-    /// \brief Template argument deduction did not deduce a value
-    /// for every template parameter.
-    TDK_Incomplete,
-    /// \brief Template argument deduction produced inconsistent
-    /// deduced values for the given template parameter.
-    TDK_Inconsistent,
-    /// \brief Template argument deduction failed due to inconsistent
-    /// cv-qualifiers on a template parameter type that would
-    /// otherwise be deduced, e.g., we tried to deduce T in "const T"
-    /// but were given a non-const "X".
-    TDK_InconsistentQuals,
-    /// \brief Substitution of the deduced template argument values
-    /// resulted in an error.
-    TDK_SubstitutionFailure,
-    /// \brief Substitution of the deduced template argument values
-    /// into a non-deduced context produced a type or value that
-    /// produces a type that does not match the original template
-    /// arguments provided.
-    TDK_NonDeducedMismatch,
-    /// \brief When performing template argument deduction for a function
-    /// template, there were too many call arguments.
-    TDK_TooManyArguments,
-    /// \brief When performing template argument deduction for a function
-    /// template, there were too few call arguments.
-    TDK_TooFewArguments,
-    /// \brief The explicitly-specified template arguments were not valid
-    /// template arguments for the given template.
-    TDK_InvalidExplicitArguments,
-    /// \brief The arguments included an overloaded function name that could
-    /// not be resolved to a suitable function.
-    TDK_FailedOverloadResolution
-  };
-
-  /// \brief Provides information about an attempted template argument
-  /// deduction, whose success or failure was described by a
-  /// TemplateDeductionResult value.
-  class TemplateDeductionInfo {
-    /// \brief The context in which the template arguments are stored.
-    ASTContext &Context;
-
-    /// \brief The deduced template argument list.
-    ///
-    TemplateArgumentList *Deduced;
-
-    /// \brief The source location at which template argument
-    /// deduction is occurring.
-    SourceLocation Loc;
-
-    // do not implement these
-    TemplateDeductionInfo(const TemplateDeductionInfo&);
-    TemplateDeductionInfo &operator=(const TemplateDeductionInfo&);
-
-  public:
-    TemplateDeductionInfo(ASTContext &Context, SourceLocation Loc)
-      : Context(Context), Deduced(0), Loc(Loc) { }
-
-    ~TemplateDeductionInfo() {
-      // FIXME: if (Deduced) Deduced->Destroy(Context);
-    }
-
-    /// \brief Returns the location at which template argument is
-    /// occuring.
-    SourceLocation getLocation() const {
-      return Loc;
-    }
-
-    /// \brief Take ownership of the deduced template argument list.
-    TemplateArgumentList *take() {
-      TemplateArgumentList *Result = Deduced;
-      Deduced = 0;
-      return Result;
-    }
-
-    /// \brief Provide a new template argument list that contains the
-    /// results of template argument deduction.
-    void reset(TemplateArgumentList *NewDeduced) {
-      // FIXME: if (Deduced) Deduced->Destroy(Context);
-      Deduced = NewDeduced;
-    }
-
-    /// \brief The template parameter to which a template argument
-    /// deduction failure refers.
-    ///
-    /// Depending on the result of template argument deduction, this
-    /// template parameter may have different meanings:
-    ///
-    ///   TDK_Incomplete: this is the first template parameter whose
-    ///   corresponding template argument was not deduced.
-    ///
-    ///   TDK_Inconsistent: this is the template parameter for which
-    ///   two different template argument values were deduced.
-    TemplateParameter Param;
-
-    /// \brief The first template argument to which the template
-    /// argument deduction failure refers.
-    ///
-    /// Depending on the result of the template argument deduction,
-    /// this template argument may have different meanings:
-    ///
-    ///   TDK_Inconsistent: this argument is the first value deduced
-    ///   for the corresponding template parameter.
-    ///
-    ///   TDK_SubstitutionFailure: this argument is the template
-    ///   argument we were instantiating when we encountered an error.
-    ///
-    ///   TDK_NonDeducedMismatch: this is the template argument
-    ///   provided in the source code.
-    TemplateArgument FirstArg;
-
-    /// \brief The second template argument to which the template
-    /// argument deduction failure refers.
-    ///
-    /// FIXME: Finish documenting this.
-    TemplateArgument SecondArg;
-  };
-
-  TemplateDeductionResult
-  DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial,
-                          const TemplateArgumentList &TemplateArgs,
-                          TemplateDeductionInfo &Info);
-
-  TemplateDeductionResult
-  SubstituteExplicitTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
-                        const TemplateArgumentListInfo &ExplicitTemplateArgs,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                                 llvm::SmallVectorImpl<QualType> &ParamTypes,
-                                      QualType *FunctionType,
-                                      TemplateDeductionInfo &Info);
-
-  TemplateDeductionResult
-  FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
-                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                                  unsigned NumExplicitlySpecified,
-                                  FunctionDecl *&Specialization,
-                                  TemplateDeductionInfo &Info);
-
-  TemplateDeductionResult
-  DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
-                          const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                          Expr **Args, unsigned NumArgs,
-                          FunctionDecl *&Specialization,
-                          TemplateDeductionInfo &Info);
-
-  TemplateDeductionResult
-  DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
-                          const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                          QualType ArgFunctionType,
-                          FunctionDecl *&Specialization,
-                          TemplateDeductionInfo &Info);
-
-  TemplateDeductionResult
-  DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
-                          QualType ToType,
-                          CXXConversionDecl *&Specialization,
-                          TemplateDeductionInfo &Info);
-
-  TemplateDeductionResult
-  DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
-                          const TemplateArgumentListInfo *ExplicitTemplateArgs,
-                          FunctionDecl *&Specialization,
-                          TemplateDeductionInfo &Info);
-
-  FunctionTemplateDecl *getMoreSpecializedTemplate(FunctionTemplateDecl *FT1,
-                                                   FunctionTemplateDecl *FT2,
-                                                   SourceLocation Loc,
-                                           TemplatePartialOrderingContext TPOC);
-  UnresolvedSetIterator getMostSpecialized(UnresolvedSetIterator SBegin,
-                                           UnresolvedSetIterator SEnd,
-                                           TemplatePartialOrderingContext TPOC,
-                                           SourceLocation Loc,
-                                           const PartialDiagnostic &NoneDiag,
-                                           const PartialDiagnostic &AmbigDiag,
-                                        const PartialDiagnostic &CandidateDiag);
-
-  ClassTemplatePartialSpecializationDecl *
-  getMoreSpecializedPartialSpecialization(
-                                  ClassTemplatePartialSpecializationDecl *PS1,
-                                  ClassTemplatePartialSpecializationDecl *PS2,
-                                  SourceLocation Loc);
-
-  void MarkUsedTemplateParameters(const TemplateArgumentList &TemplateArgs,
-                                  bool OnlyDeduced,
-                                  unsigned Depth,
-                                  llvm::SmallVectorImpl<bool> &Used);
-  void MarkDeducedTemplateParameters(FunctionTemplateDecl *FunctionTemplate,
-                                     llvm::SmallVectorImpl<bool> &Deduced);
-
-  //===--------------------------------------------------------------------===//
-  // C++ Template Instantiation
-  //
-
-  MultiLevelTemplateArgumentList getTemplateInstantiationArgs(NamedDecl *D,
-                                     const TemplateArgumentList *Innermost = 0,
-                                                bool RelativeToPrimary = false,
-                                               const FunctionDecl *Pattern = 0);
-
-  /// \brief A template instantiation that is currently in progress.
-  struct ActiveTemplateInstantiation {
-    /// \brief The kind of template instantiation we are performing
-    enum InstantiationKind {
-      /// We are instantiating a template declaration. The entity is
-      /// the declaration we're instantiating (e.g., a CXXRecordDecl).
-      TemplateInstantiation,
-
-      /// We are instantiating a default argument for a template
-      /// parameter. The Entity is the template, and
-      /// TemplateArgs/NumTemplateArguments provides the template
-      /// arguments as specified.
-      /// FIXME: Use a TemplateArgumentList
-      DefaultTemplateArgumentInstantiation,
-
-      /// We are instantiating a default argument for a function.
-      /// The Entity is the ParmVarDecl, and TemplateArgs/NumTemplateArgs
-      /// provides the template arguments as specified.
-      DefaultFunctionArgumentInstantiation,
-
-      /// We are substituting explicit template arguments provided for
-      /// a function template. The entity is a FunctionTemplateDecl.
-      ExplicitTemplateArgumentSubstitution,
-
-      /// We are substituting template argument determined as part of
-      /// template argument deduction for either a class template
-      /// partial specialization or a function template. The
-      /// Entity is either a ClassTemplatePartialSpecializationDecl or
-      /// a FunctionTemplateDecl.
-      DeducedTemplateArgumentSubstitution,
-
-      /// We are substituting prior template arguments into a new
-      /// template parameter. The template parameter itself is either a
-      /// NonTypeTemplateParmDecl or a TemplateTemplateParmDecl.
-      PriorTemplateArgumentSubstitution,
-
-      /// We are checking the validity of a default template argument that
-      /// has been used when naming a template-id.
-      DefaultTemplateArgumentChecking
-    } Kind;
-
-    /// \brief The point of instantiation within the source code.
-    SourceLocation PointOfInstantiation;
-
-    /// \brief The template in which we are performing the instantiation,
-    /// for substitutions of prior template arguments.
-    TemplateDecl *Template;
-
-    /// \brief The entity that is being instantiated.
-    uintptr_t Entity;
-
-    /// \brief The list of template arguments we are substituting, if they
-    /// are not part of the entity.
-    const TemplateArgument *TemplateArgs;
-
-    /// \brief The number of template arguments in TemplateArgs.
-    unsigned NumTemplateArgs;
-
-    /// \brief The source range that covers the construct that cause
-    /// the instantiation, e.g., the template-id that causes a class
-    /// template instantiation.
-    SourceRange InstantiationRange;
-
-    ActiveTemplateInstantiation()
-      : Kind(TemplateInstantiation), Template(0), Entity(0), TemplateArgs(0),
-        NumTemplateArgs(0) {}
-
-    /// \brief Determines whether this template is an actual instantiation
-    /// that should be counted toward the maximum instantiation depth.
-    bool isInstantiationRecord() const;
-
-    friend bool operator==(const ActiveTemplateInstantiation &X,
-                           const ActiveTemplateInstantiation &Y) {
-      if (X.Kind != Y.Kind)
-        return false;
-
-      if (X.Entity != Y.Entity)
-        return false;
-
-      switch (X.Kind) {
-      case TemplateInstantiation:
-        return true;
-
-      case PriorTemplateArgumentSubstitution:
-      case DefaultTemplateArgumentChecking:
-        if (X.Template != Y.Template)
-          return false;
-
-        // Fall through
-
-      case DefaultTemplateArgumentInstantiation:
-      case ExplicitTemplateArgumentSubstitution:
-      case DeducedTemplateArgumentSubstitution:
-      case DefaultFunctionArgumentInstantiation:
-        return X.TemplateArgs == Y.TemplateArgs;
-
-      }
-
-      return true;
-    }
-
-    friend bool operator!=(const ActiveTemplateInstantiation &X,
-                           const ActiveTemplateInstantiation &Y) {
-      return !(X == Y);
-    }
-  };
-
-  /// \brief List of active template instantiations.
-  ///
-  /// This vector is treated as a stack. As one template instantiation
-  /// requires another template instantiation, additional
-  /// instantiations are pushed onto the stack up to a
-  /// user-configurable limit LangOptions::InstantiationDepth.
-  llvm::SmallVector<ActiveTemplateInstantiation, 16>
-    ActiveTemplateInstantiations;
-
-  /// \brief The number of ActiveTemplateInstantiation entries in
-  /// \c ActiveTemplateInstantiations that are not actual instantiations and,
-  /// therefore, should not be counted as part of the instantiation depth.
-  unsigned NonInstantiationEntries;
-
-  /// \brief The last template from which a template instantiation
-  /// error or warning was produced.
-  ///
-  /// This value is used to suppress printing of redundant template
-  /// instantiation backtraces when there are multiple errors in the
-  /// same instantiation. FIXME: Does this belong in Sema? It's tough
-  /// to implement it anywhere else.
-  ActiveTemplateInstantiation LastTemplateInstantiationErrorContext;
-
-  /// \brief The stack of calls expression undergoing template instantiation.
-  ///
-  /// The top of this stack is used by a fixit instantiating unresolved
-  /// function calls to fix the AST to match the textual change it prints.
-  llvm::SmallVector<CallExpr *, 8> CallsUndergoingInstantiation;
-
-  /// \brief A stack object to be created when performing template
-  /// instantiation.
-  ///
-  /// Construction of an object of type \c InstantiatingTemplate
-  /// pushes the current instantiation onto the stack of active
-  /// instantiations. If the size of this stack exceeds the maximum
-  /// number of recursive template instantiations, construction
-  /// produces an error and evaluates true.
-  ///
-  /// Destruction of this object will pop the named instantiation off
-  /// the stack.
-  struct InstantiatingTemplate {
-    /// \brief Note that we are instantiating a class template,
-    /// function template, or a member thereof.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          Decl *Entity,
-                          SourceRange InstantiationRange = SourceRange());
-
-    /// \brief Note that we are instantiating a default argument in a
-    /// template-id.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          TemplateDecl *Template,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          SourceRange InstantiationRange = SourceRange());
-
-    /// \brief Note that we are instantiating a default argument in a
-    /// template-id.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          FunctionTemplateDecl *FunctionTemplate,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          ActiveTemplateInstantiation::InstantiationKind Kind,
-                          SourceRange InstantiationRange = SourceRange());
-
-    /// \brief Note that we are instantiating as part of template
-    /// argument deduction for a class template partial
-    /// specialization.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          ClassTemplatePartialSpecializationDecl *PartialSpec,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          SourceRange InstantiationRange = SourceRange());
-
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          ParmVarDecl *Param,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          SourceRange InstantiationRange = SourceRange());
-
-    /// \brief Note that we are substituting prior template arguments into a
-    /// non-type or template template parameter.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          TemplateDecl *Template,
-                          NonTypeTemplateParmDecl *Param,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          SourceRange InstantiationRange);
-
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          TemplateDecl *Template,
-                          TemplateTemplateParmDecl *Param,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          SourceRange InstantiationRange);
-
-    /// \brief Note that we are checking the default template argument
-    /// against the template parameter for a given template-id.
-    InstantiatingTemplate(Sema &SemaRef, SourceLocation PointOfInstantiation,
-                          TemplateDecl *Template,
-                          NamedDecl *Param,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs,
-                          SourceRange InstantiationRange);
-
-
-    /// \brief Note that we have finished instantiating this template.
-    void Clear();
-
-    ~InstantiatingTemplate() { Clear(); }
-
-    /// \brief Determines whether we have exceeded the maximum
-    /// recursive template instantiations.
-    operator bool() const { return Invalid; }
-
-  private:
-    Sema &SemaRef;
-    bool Invalid;
-
-    bool CheckInstantiationDepth(SourceLocation PointOfInstantiation,
-                                 SourceRange InstantiationRange);
-
-    InstantiatingTemplate(const InstantiatingTemplate&); // not implemented
-
-    InstantiatingTemplate&
-    operator=(const InstantiatingTemplate&); // not implemented
-  };
-
-  void PrintInstantiationStack();
-
-  /// \brief Determines whether we are currently in a context where
-  /// template argument substitution failures are not considered
-  /// errors.
-  ///
-  /// When this routine returns true, the emission of most diagnostics
-  /// will be suppressed and there will be no local error recovery.
-  bool isSFINAEContext() const;
-
-  /// \brief RAII class used to determine whether SFINAE has
-  /// trapped any errors that occur during template argument
-  /// deduction.
-  class SFINAETrap {
-    Sema &SemaRef;
-    unsigned PrevSFINAEErrors;
-  public:
-    explicit SFINAETrap(Sema &SemaRef)
-      : SemaRef(SemaRef), PrevSFINAEErrors(SemaRef.NumSFINAEErrors) { }
-
-    ~SFINAETrap() { SemaRef.NumSFINAEErrors = PrevSFINAEErrors; }
-
-    /// \brief Determine whether any SFINAE errors have been trapped.
-    bool hasErrorOccurred() const {
-      return SemaRef.NumSFINAEErrors > PrevSFINAEErrors;
-    }
-  };
-
-  /// \brief RAII class that determines when any errors have occurred
-  /// between the time the instance was created and the time it was
-  /// queried.
-  class ErrorTrap {
-    Sema &SemaRef;
-    unsigned PrevErrors;
-
-  public:
-    explicit ErrorTrap(Sema &SemaRef)
-      : SemaRef(SemaRef), PrevErrors(SemaRef.getDiagnostics().getNumErrors()) {}
-
-    /// \brief Determine whether any errors have occurred since this
-    /// object instance was created.
-    bool hasErrorOccurred() const {
-      return SemaRef.getDiagnostics().getNumErrors() > PrevErrors;
-    }
-  };
-
-  /// \brief A stack-allocated class that identifies which local
-  /// variable declaration instantiations are present in this scope.
-  ///
-  /// A new instance of this class type will be created whenever we
-  /// instantiate a new function declaration, which will have its own
-  /// set of parameter declarations.
-  class LocalInstantiationScope {
-    /// \brief Reference to the semantic analysis that is performing
-    /// this template instantiation.
-    Sema &SemaRef;
-
-    /// \brief A mapping from local declarations that occur
-    /// within a template to their instantiations.
-    ///
-    /// This mapping is used during instantiation to keep track of,
-    /// e.g., function parameter and variable declarations. For example,
-    /// given:
-    ///
-    /// \code
-    ///   template<typename T> T add(T x, T y) { return x + y; }
-    /// \endcode
-    ///
-    /// when we instantiate add<int>, we will introduce a mapping from
-    /// the ParmVarDecl for 'x' that occurs in the template to the
-    /// instantiated ParmVarDecl for 'x'.
-    llvm::DenseMap<const Decl *, Decl *> LocalDecls;
-
-    /// \brief The outer scope, which contains local variable
-    /// definitions from some other instantiation (that may not be
-    /// relevant to this particular scope).
-    LocalInstantiationScope *Outer;
-
-    /// \brief Whether we have already exited this scope.
-    bool Exited;
-
-    /// \brief Whether to combine this scope with the outer scope, such that
-    /// lookup will search our outer scope.
-    bool CombineWithOuterScope;
-    
-    // This class is non-copyable
-    LocalInstantiationScope(const LocalInstantiationScope &);
-    LocalInstantiationScope &operator=(const LocalInstantiationScope &);
-
-  public:
-    LocalInstantiationScope(Sema &SemaRef, bool CombineWithOuterScope = false)
-      : SemaRef(SemaRef), Outer(SemaRef.CurrentInstantiationScope),
-        Exited(false), CombineWithOuterScope(CombineWithOuterScope)
-    {
-      SemaRef.CurrentInstantiationScope = this;
-    }
-
-    ~LocalInstantiationScope() {
-      Exit();
-    }
-
-    /// \brief Exit this local instantiation scope early.
-    void Exit() {
-      if (Exited)
-        return;
-      
-      SemaRef.CurrentInstantiationScope = Outer;
-      Exited = true;
-    }
-
-    Decl *getInstantiationOf(const Decl *D);
-
-    VarDecl *getInstantiationOf(const VarDecl *Var) {
-      return cast<VarDecl>(getInstantiationOf(cast<Decl>(Var)));
-    }
-
-    ParmVarDecl *getInstantiationOf(const ParmVarDecl *Var) {
-      return cast<ParmVarDecl>(getInstantiationOf(cast<Decl>(Var)));
-    }
-
-    NonTypeTemplateParmDecl *getInstantiationOf(
-                                          const NonTypeTemplateParmDecl *Var) {
-      return cast<NonTypeTemplateParmDecl>(getInstantiationOf(cast<Decl>(Var)));
-    }
-
-    void InstantiatedLocal(const Decl *D, Decl *Inst);
-  };
-
-  /// \brief The current instantiation scope used to store local
-  /// variables.
-  LocalInstantiationScope *CurrentInstantiationScope;
-
-  /// \brief The number of typos corrected by CorrectTypo.
-  unsigned TyposCorrected;
-
-  /// \brief Worker object for performing CFG-based warnings.
-  sema::AnalysisBasedWarnings AnalysisWarnings;
-
-  /// \brief An entity for which implicit template instantiation is required.
-  ///
-  /// The source location associated with the declaration is the first place in
-  /// the source code where the declaration was "used". It is not necessarily
-  /// the point of instantiation (which will be either before or after the
-  /// namespace-scope declaration that triggered this implicit instantiation),
-  /// However, it is the location that diagnostics should generally refer to,
-  /// because users will need to know what code triggered the instantiation.
-  typedef std::pair<ValueDecl *, SourceLocation> PendingImplicitInstantiation;
-
-  /// \brief The queue of implicit template instantiations that are required
-  /// but have not yet been performed.
-  std::deque<PendingImplicitInstantiation> PendingImplicitInstantiations;
-
-  /// \brief The queue of implicit template instantiations that are required
-  /// and must be performed within the current local scope.
-  ///
-  /// This queue is only used for member functions of local classes in
-  /// templates, which must be instantiated in the same scope as their
-  /// enclosing function, so that they can reference function-local
-  /// types, static variables, enumerators, etc.
-  std::deque<PendingImplicitInstantiation> PendingLocalImplicitInstantiations;
-
-  void PerformPendingImplicitInstantiations(bool LocalOnly = false);
-
-  TypeSourceInfo *SubstType(TypeSourceInfo *T,
-                            const MultiLevelTemplateArgumentList &TemplateArgs,
-                            SourceLocation Loc, DeclarationName Entity);
-
-  QualType SubstType(QualType T,
-                     const MultiLevelTemplateArgumentList &TemplateArgs,
-                     SourceLocation Loc, DeclarationName Entity);
-
-  TypeSourceInfo *SubstFunctionDeclType(TypeSourceInfo *T,
-                            const MultiLevelTemplateArgumentList &TemplateArgs,
-                                        SourceLocation Loc,
-                                        DeclarationName Entity);
-  ParmVarDecl *SubstParmVarDecl(ParmVarDecl *D,
-                            const MultiLevelTemplateArgumentList &TemplateArgs);
-  OwningExprResult SubstExpr(Expr *E,
-                            const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  OwningStmtResult SubstStmt(Stmt *S,
-                            const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  Decl *SubstDecl(Decl *D, DeclContext *Owner,
-                  const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  bool
-  SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
-                      CXXRecordDecl *Pattern,
-                      const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  bool
-  InstantiateClass(SourceLocation PointOfInstantiation,
-                   CXXRecordDecl *Instantiation, CXXRecordDecl *Pattern,
-                   const MultiLevelTemplateArgumentList &TemplateArgs,
-                   TemplateSpecializationKind TSK,
-                   bool Complain = true);
-
-  bool
-  InstantiateClassTemplateSpecialization(SourceLocation PointOfInstantiation,
-                           ClassTemplateSpecializationDecl *ClassTemplateSpec,
-                           TemplateSpecializationKind TSK,
-                           bool Complain = true);
-
-  void InstantiateClassMembers(SourceLocation PointOfInstantiation,
-                               CXXRecordDecl *Instantiation,
-                            const MultiLevelTemplateArgumentList &TemplateArgs,
-                               TemplateSpecializationKind TSK);
-
-  void InstantiateClassTemplateSpecializationMembers(
-                                          SourceLocation PointOfInstantiation,
-                           ClassTemplateSpecializationDecl *ClassTemplateSpec,
-                                                TemplateSpecializationKind TSK);
-
-  NestedNameSpecifier *
-  SubstNestedNameSpecifier(NestedNameSpecifier *NNS,
-                           SourceRange Range,
-                           const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  TemplateName
-  SubstTemplateName(TemplateName Name, SourceLocation Loc,
-                    const MultiLevelTemplateArgumentList &TemplateArgs);
-  bool Subst(const TemplateArgumentLoc &Arg, TemplateArgumentLoc &Result,
-             const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  void InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
-                                     FunctionDecl *Function,
-                                     bool Recursive = false,
-                                     bool DefinitionRequired = false);
-  void InstantiateStaticDataMemberDefinition(
-                                     SourceLocation PointOfInstantiation,
-                                     VarDecl *Var,
-                                     bool Recursive = false,
-                                     bool DefinitionRequired = false);
-
-  void InstantiateMemInitializers(CXXConstructorDecl *New,
-                                  const CXXConstructorDecl *Tmpl,
-                            const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  NamedDecl *FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
-                          const MultiLevelTemplateArgumentList &TemplateArgs);
-  DeclContext *FindInstantiatedContext(SourceLocation Loc, DeclContext *DC,
-                          const MultiLevelTemplateArgumentList &TemplateArgs);
-
-  // Objective-C declarations.
-  virtual DeclPtrTy ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
-                                             IdentifierInfo *ClassName,
-                                             SourceLocation ClassLoc,
-                                             IdentifierInfo *SuperName,
-                                             SourceLocation SuperLoc,
-                                             const DeclPtrTy *ProtoRefs,
-                                             unsigned NumProtoRefs,
-                                             const SourceLocation *ProtoLocs,
-                                             SourceLocation EndProtoLoc,
-                                             AttributeList *AttrList);
-
-  virtual DeclPtrTy ActOnCompatiblityAlias(
-                    SourceLocation AtCompatibilityAliasLoc,
-                    IdentifierInfo *AliasName,  SourceLocation AliasLocation,
-                    IdentifierInfo *ClassName, SourceLocation ClassLocation);
-
-  void CheckForwardProtocolDeclarationForCircularDependency(
-    IdentifierInfo *PName,
-    SourceLocation &PLoc, SourceLocation PrevLoc,
-    const ObjCList<ObjCProtocolDecl> &PList);
-
-  virtual DeclPtrTy ActOnStartProtocolInterface(
-                    SourceLocation AtProtoInterfaceLoc,
-                    IdentifierInfo *ProtocolName, SourceLocation ProtocolLoc,
-                    const DeclPtrTy *ProtoRefNames, unsigned NumProtoRefs,
-                    const SourceLocation *ProtoLocs,
-                    SourceLocation EndProtoLoc,
-                    AttributeList *AttrList);
-
-  virtual DeclPtrTy ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
-                                                IdentifierInfo *ClassName,
-                                                SourceLocation ClassLoc,
-                                                IdentifierInfo *CategoryName,
-                                                SourceLocation CategoryLoc,
-                                                const DeclPtrTy *ProtoRefs,
-                                                unsigned NumProtoRefs,
-                                                const SourceLocation *ProtoLocs,
-                                                SourceLocation EndProtoLoc);
-
-  virtual DeclPtrTy ActOnStartClassImplementation(
-                    SourceLocation AtClassImplLoc,
-                    IdentifierInfo *ClassName, SourceLocation ClassLoc,
-                    IdentifierInfo *SuperClassname,
-                    SourceLocation SuperClassLoc);
-
-  virtual DeclPtrTy ActOnStartCategoryImplementation(
-                                                  SourceLocation AtCatImplLoc,
-                                                  IdentifierInfo *ClassName,
-                                                  SourceLocation ClassLoc,
-                                                  IdentifierInfo *CatName,
-                                                  SourceLocation CatLoc);
-
-  virtual DeclPtrTy ActOnForwardClassDeclaration(SourceLocation Loc,
-                                                 IdentifierInfo **IdentList,
-                                                 SourceLocation *IdentLocs,
-                                                 unsigned NumElts);
-
-  virtual DeclPtrTy ActOnForwardProtocolDeclaration(SourceLocation AtProtoclLoc,
-                                            const IdentifierLocPair *IdentList,
-                                                  unsigned NumElts,
-                                                  AttributeList *attrList);
-
-  virtual void FindProtocolDeclaration(bool WarnOnDeclarations,
-                                       const IdentifierLocPair *ProtocolId,
-                                       unsigned NumProtocols,
-                                   llvm::SmallVectorImpl<DeclPtrTy> &Protocols);
-
-  /// Ensure attributes are consistent with type.
-  /// \param [in, out] Attributes The attributes to check; they will
-  /// be modified to be consistent with \arg PropertyTy.
-  void CheckObjCPropertyAttributes(DeclPtrTy PropertyPtrTy,
-                                   SourceLocation Loc,
-                                   unsigned &Attributes);
-  void ProcessPropertyDecl(ObjCPropertyDecl *property, ObjCContainerDecl *DC);
-  void DiagnosePropertyMismatch(ObjCPropertyDecl *Property,
-                                ObjCPropertyDecl *SuperProperty,
-                                const IdentifierInfo *Name);
-  void ComparePropertiesInBaseAndSuper(ObjCInterfaceDecl *IDecl);
-
-  void CompareMethodParamsInBaseAndSuper(Decl *IDecl,
-                                         ObjCMethodDecl *MethodDecl,
-                                         bool IsInstance);
-
-  void CompareProperties(Decl *CDecl, DeclPtrTy MergeProtocols);
-
-  void DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
-                                        ObjCInterfaceDecl *ID);
-
-  void MatchOneProtocolPropertiesInClass(Decl *CDecl,
-                                         ObjCProtocolDecl *PDecl);
-
-  virtual void ActOnAtEnd(Scope *S, SourceRange AtEnd,
-                          DeclPtrTy classDecl,
-                          DeclPtrTy *allMethods = 0, unsigned allNum = 0,
-                          DeclPtrTy *allProperties = 0, unsigned pNum = 0,
-                          DeclGroupPtrTy *allTUVars = 0, unsigned tuvNum = 0);
-
-  virtual DeclPtrTy ActOnProperty(Scope *S, SourceLocation AtLoc,
-                                  FieldDeclarator &FD, ObjCDeclSpec &ODS,
-                                  Selector GetterSel, Selector SetterSel,
-                                  DeclPtrTy ClassCategory,
-                                  bool *OverridingProperty,
-                                  tok::ObjCKeywordKind MethodImplKind);
-
-  virtual DeclPtrTy ActOnPropertyImplDecl(Scope *S,
-                                          SourceLocation AtLoc,
-                                          SourceLocation PropertyLoc,
-                                          bool ImplKind,DeclPtrTy ClassImplDecl,
-                                          IdentifierInfo *PropertyId,
-                                          IdentifierInfo *PropertyIvar);
-
-  virtual DeclPtrTy ActOnMethodDeclaration(
-    SourceLocation BeginLoc, // location of the + or -.
-    SourceLocation EndLoc,   // location of the ; or {.
-    tok::TokenKind MethodType,
-    DeclPtrTy ClassDecl, ObjCDeclSpec &ReturnQT, TypeTy *ReturnType,
-    Selector Sel,
-    // optional arguments. The number of types/arguments is obtained
-    // from the Sel.getNumArgs().
-    ObjCArgInfo *ArgInfo,
-    DeclaratorChunk::ParamInfo *CParamInfo, unsigned CNumArgs, // c-style args
-    AttributeList *AttrList, tok::ObjCKeywordKind MethodImplKind,
-    bool isVariadic = false);
-
-  // Helper method for ActOnClassMethod/ActOnInstanceMethod.
-  // Will search "local" class/category implementations for a method decl.
-  // Will also search in class's root looking for instance method.
-  // Returns 0 if no method is found.
-  ObjCMethodDecl *LookupPrivateClassMethod(Selector Sel,
-                                           ObjCInterfaceDecl *CDecl);
-  ObjCMethodDecl *LookupPrivateInstanceMethod(Selector Sel,
-                                              ObjCInterfaceDecl *ClassDecl);
-
-  OwningExprResult
-  HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
-                            Expr *BaseExpr,
-                            DeclarationName MemberName,
-                            SourceLocation MemberLoc);
-
-  virtual OwningExprResult
-  ActOnClassPropertyRefExpr(IdentifierInfo &receiverName,
-                            IdentifierInfo &propertyName,
-                            SourceLocation receiverNameLoc,
-                            SourceLocation propertyNameLoc);
-
-  virtual ObjCMessageKind getObjCMessageKind(Scope *S,
-                                             IdentifierInfo *Name,
-                                             SourceLocation NameLoc,
-                                             bool IsSuper,
-                                             bool HasTrailingDot,
-                                             TypeTy *&ReceiverType);
-
-  virtual OwningExprResult ActOnSuperMessage(Scope *S, SourceLocation SuperLoc,
-                                             Selector Sel,
-                                             SourceLocation LBracLoc,
-                                             SourceLocation SelectorLoc,
-                                             SourceLocation RBracLoc,
-                                             MultiExprArg Args);
-
-  OwningExprResult BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
-                                     QualType ReceiverType,
-                                     SourceLocation SuperLoc,
-                                     Selector Sel,
-                                     ObjCMethodDecl *Method,
-                                     SourceLocation LBracLoc,
-                                     SourceLocation RBracLoc,
-                                     MultiExprArg Args);
-
-  virtual OwningExprResult ActOnClassMessage(Scope *S,
-                                             TypeTy *Receiver,
-                                             Selector Sel,
-                                             SourceLocation LBracLoc,
-                                             SourceLocation SelectorLoc,
-                                             SourceLocation RBracLoc,
-                                             MultiExprArg Args);
-
-  OwningExprResult BuildInstanceMessage(ExprArg Receiver,
-                                        QualType ReceiverType,
-                                        SourceLocation SuperLoc,
-                                        Selector Sel,
-                                        ObjCMethodDecl *Method,
-                                        SourceLocation LBracLoc,
-                                        SourceLocation RBracLoc,
-                                        MultiExprArg Args);
-
-  virtual OwningExprResult ActOnInstanceMessage(Scope *S,
-                                                ExprArg Receiver,
-                                                Selector Sel,
-                                                SourceLocation LBracLoc,
-                                                SourceLocation SelectorLoc,
-                                                SourceLocation RBracLoc,
-                                                MultiExprArg Args);
-
-
-  /// ActOnPragmaOptionsAlign - Called on well formed #pragma options align.
-  virtual void ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind,
-                                       SourceLocation PragmaLoc,
-                                       SourceLocation KindLoc);
-
-  /// ActOnPragmaPack - Called on well formed #pragma pack(...).
-  virtual void ActOnPragmaPack(PragmaPackKind Kind,
-                               IdentifierInfo *Name,
-                               ExprTy *Alignment,
-                               SourceLocation PragmaLoc,
-                               SourceLocation LParenLoc,
-                               SourceLocation RParenLoc);
-
-  /// ActOnPragmaUnused - Called on well-formed '#pragma unused'.
-  virtual void ActOnPragmaUnused(const Token *Identifiers,
-                                 unsigned NumIdentifiers, Scope *curScope,
-                                 SourceLocation PragmaLoc,
-                                 SourceLocation LParenLoc,
-                                 SourceLocation RParenLoc);
-
-  NamedDecl *DeclClonePragmaWeak(NamedDecl *ND, IdentifierInfo *II);
-  void DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, WeakInfo &W);
-
-  /// ActOnPragmaWeakID - Called on well formed #pragma weak ident.
-  virtual void ActOnPragmaWeakID(IdentifierInfo* WeakName,
-                                 SourceLocation PragmaLoc,
-                                 SourceLocation WeakNameLoc);
-
-  /// ActOnPragmaWeakAlias - Called on well formed #pragma weak ident = ident.
-  virtual void ActOnPragmaWeakAlias(IdentifierInfo* WeakName,
-                                    IdentifierInfo* AliasName,
-                                    SourceLocation PragmaLoc,
-                                    SourceLocation WeakNameLoc,
-                                    SourceLocation AliasNameLoc);
-
-  /// AddAlignmentAttributesForRecord - Adds any needed alignment attributes to
-  /// a the record decl, to handle '#pragma pack' and '#pragma options align'.
-  void AddAlignmentAttributesForRecord(RecordDecl *RD);
-
-  /// FreePackedContext - Deallocate and null out PackContext.
-  void FreePackedContext();
-
-  /// AddAlignedAttr - Adds an aligned attribute to a particular declaration.
-  void AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E);
-
-  /// ImpCastExprToType - If Expr is not of type 'Type', insert an implicit
-  /// cast.  If there is already an implicit cast, merge into the existing one.
-  /// If isLvalue, the result of the cast is an lvalue.
-  void ImpCastExprToType(Expr *&Expr, QualType Type, CastExpr::CastKind Kind,
-                         bool isLvalue = false,
-                         CXXBaseSpecifierArray BasePath =
-                          CXXBaseSpecifierArray());
-
-  // UsualUnaryConversions - promotes integers (C99 6.3.1.1p2) and converts
-  // functions and arrays to their respective pointers (C99 6.3.2.1).
-  Expr *UsualUnaryConversions(Expr *&expr);
-
-  // DefaultFunctionArrayConversion - converts functions and arrays
-  // to their respective pointers (C99 6.3.2.1).
-  void DefaultFunctionArrayConversion(Expr *&expr);
-
-  // DefaultFunctionArrayLvalueConversion - converts functions and
-  // arrays to their respective pointers and performs the
-  // lvalue-to-rvalue conversion.
-  void DefaultFunctionArrayLvalueConversion(Expr *&expr);
-
-  // DefaultArgumentPromotion (C99 6.5.2.2p6). Used for function calls that
-  // do not have a prototype. Integer promotions are performed on each
-  // argument, and arguments that have type float are promoted to double.
-  void DefaultArgumentPromotion(Expr *&Expr);
-
-  // Used for emitting the right warning by DefaultVariadicArgumentPromotion
-  enum VariadicCallType {
-    VariadicFunction,
-    VariadicBlock,
-    VariadicMethod,
-    VariadicConstructor,
-    VariadicDoesNotApply
-  };
-
-  /// GatherArgumentsForCall - Collector argument expressions for various
-  /// form of call prototypes.
-  bool GatherArgumentsForCall(SourceLocation CallLoc,
-                              FunctionDecl *FDecl,
-                              const FunctionProtoType *Proto,
-                              unsigned FirstProtoArg,
-                              Expr **Args, unsigned NumArgs,
-                              llvm::SmallVector<Expr *, 8> &AllArgs,
-                              VariadicCallType CallType = VariadicDoesNotApply);
-
-  // DefaultVariadicArgumentPromotion - Like DefaultArgumentPromotion, but
-  // will warn if the resulting type is not a POD type.
-  bool DefaultVariadicArgumentPromotion(Expr *&Expr, VariadicCallType CT,
-                                        FunctionDecl *FDecl);
-
-  // UsualArithmeticConversions - performs the UsualUnaryConversions on it's
-  // operands and then handles various conversions that are common to binary
-  // operators (C99 6.3.1.8). If both operands aren't arithmetic, this
-  // routine returns the first non-arithmetic type found. The client is
-  // responsible for emitting appropriate error diagnostics.
-  QualType UsualArithmeticConversions(Expr *&lExpr, Expr *&rExpr,
-                                      bool isCompAssign = false);
-
-  /// AssignConvertType - All of the 'assignment' semantic checks return this
-  /// enum to indicate whether the assignment was allowed.  These checks are
-  /// done for simple assignments, as well as initialization, return from
-  /// function, argument passing, etc.  The query is phrased in terms of a
-  /// source and destination type.
-  enum AssignConvertType {
-    /// Compatible - the types are compatible according to the standard.
-    Compatible,
-
-    /// PointerToInt - The assignment converts a pointer to an int, which we
-    /// accept as an extension.
-    PointerToInt,
-
-    /// IntToPointer - The assignment converts an int to a pointer, which we
-    /// accept as an extension.
-    IntToPointer,
-
-    /// FunctionVoidPointer - The assignment is between a function pointer and
-    /// void*, which the standard doesn't allow, but we accept as an extension.
-    FunctionVoidPointer,
-
-    /// IncompatiblePointer - The assignment is between two pointers types that
-    /// are not compatible, but we accept them as an extension.
-    IncompatiblePointer,
-
-    /// IncompatiblePointer - The assignment is between two pointers types which
-    /// point to integers which have a different sign, but are otherwise identical.
-    /// This is a subset of the above, but broken out because it's by far the most
-    /// common case of incompatible pointers.
-    IncompatiblePointerSign,
-
-    /// CompatiblePointerDiscardsQualifiers - The assignment discards
-    /// c/v/r qualifiers, which we accept as an extension.
-    CompatiblePointerDiscardsQualifiers,
-
-    /// IncompatibleNestedPointerQualifiers - The assignment is between two
-    /// nested pointer types, and the qualifiers other than the first two
-    /// levels differ e.g. char ** -> const char **, but we accept them as an
-    /// extension.
-    IncompatibleNestedPointerQualifiers,
-
-    /// IncompatibleVectors - The assignment is between two vector types that
-    /// have the same size, which we accept as an extension.
-    IncompatibleVectors,
-
-    /// IntToBlockPointer - The assignment converts an int to a block
-    /// pointer. We disallow this.
-    IntToBlockPointer,
-
-    /// IncompatibleBlockPointer - The assignment is between two block
-    /// pointers types that are not compatible.
-    IncompatibleBlockPointer,
-
-    /// IncompatibleObjCQualifiedId - The assignment is between a qualified
-    /// id type and something else (that is incompatible with it). For example,
-    /// "id <XXX>" = "Foo *", where "Foo *" doesn't implement the XXX protocol.
-    IncompatibleObjCQualifiedId,
-
-    /// Incompatible - We reject this conversion outright, it is invalid to
-    /// represent it in the AST.
-    Incompatible
-  };
-
-  /// DiagnoseAssignmentResult - Emit a diagnostic, if required, for the
-  /// assignment conversion type specified by ConvTy.  This returns true if the
-  /// conversion was invalid or false if the conversion was accepted.
-  bool DiagnoseAssignmentResult(AssignConvertType ConvTy,
-                                SourceLocation Loc,
-                                QualType DstType, QualType SrcType,
-                                Expr *SrcExpr, AssignmentAction Action,
-                                bool *Complained = 0);
-
-  /// CheckAssignmentConstraints - Perform type checking for assignment,
-  /// argument passing, variable initialization, and function return values.
-  /// This routine is only used by the following two methods. C99 6.5.16.
-  AssignConvertType CheckAssignmentConstraints(QualType lhs, QualType rhs);
-
-  // CheckSingleAssignmentConstraints - Currently used by
-  // CheckAssignmentOperands, and ActOnReturnStmt. Prior to type checking,
-  // this routine performs the default function/array converions.
-  AssignConvertType CheckSingleAssignmentConstraints(QualType lhs,
-                                                     Expr *&rExpr);
-
-  // \brief If the lhs type is a transparent union, check whether we
-  // can initialize the transparent union with the given expression.
-  AssignConvertType CheckTransparentUnionArgumentConstraints(QualType lhs,
-                                                             Expr *&rExpr);
-
-  // Helper function for CheckAssignmentConstraints (C99 6.5.16.1p1)
-  AssignConvertType CheckPointerTypesForAssignment(QualType lhsType,
-                                                   QualType rhsType);
-
-  AssignConvertType CheckObjCPointerTypesForAssignment(QualType lhsType,
-                                                       QualType rhsType);
-
-  // Helper function for CheckAssignmentConstraints involving two
-  // block pointer types.
-  AssignConvertType CheckBlockPointerTypesForAssignment(QualType lhsType,
-                                                        QualType rhsType);
-
-  bool IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType);
-
-  bool CheckExceptionSpecCompatibility(Expr *From, QualType ToType);
-
-  bool PerformImplicitConversion(Expr *&From, QualType ToType,
-                                 AssignmentAction Action,
-                                 bool AllowExplicit = false);
-  bool PerformImplicitConversion(Expr *&From, QualType ToType,
-                                 AssignmentAction Action,
-                                 bool AllowExplicit,
-                                 ImplicitConversionSequence& ICS);
-  bool PerformImplicitConversion(Expr *&From, QualType ToType,
-                                 const ImplicitConversionSequence& ICS,
-                                 AssignmentAction Action,
-                                 bool IgnoreBaseAccess = false);
-  bool PerformImplicitConversion(Expr *&From, QualType ToType,
-                                 const StandardConversionSequence& SCS,
-                                 AssignmentAction Action,bool IgnoreBaseAccess);
-
-  /// the following "Check" methods will return a valid/converted QualType
-  /// or a null QualType (indicating an error diagnostic was issued).
-
-  /// type checking binary operators (subroutines of CreateBuiltinBinOp).
-  QualType InvalidOperands(SourceLocation l, Expr *&lex, Expr *&rex);
-  QualType CheckPointerToMemberOperands( // C++ 5.5
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isIndirect);
-  QualType CheckMultiplyDivideOperands( // C99 6.5.5
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isCompAssign,
-                                       bool isDivide);
-  QualType CheckRemainderOperands( // C99 6.5.5
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isCompAssign = false);
-  QualType CheckAdditionOperands( // C99 6.5.6
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, QualType* CompLHSTy = 0);
-  QualType CheckSubtractionOperands( // C99 6.5.6
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, QualType* CompLHSTy = 0);
-  QualType CheckShiftOperands( // C99 6.5.7
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isCompAssign = false);
-  QualType CheckCompareOperands( // C99 6.5.8/9
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, unsigned Opc,
-                                bool isRelational);
-  QualType CheckBitwiseOperands( // C99 6.5.[10...12]
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, bool isCompAssign = false);
-  QualType CheckLogicalOperands( // C99 6.5.[13,14]
-    Expr *&lex, Expr *&rex, SourceLocation OpLoc, unsigned Opc);
-  // CheckAssignmentOperands is used for both simple and compound assignment.
-  // For simple assignment, pass both expressions and a null converted type.
-  // For compound assignment, pass both expressions and the converted type.
-  QualType CheckAssignmentOperands( // C99 6.5.16.[1,2]
-    Expr *lex, Expr *&rex, SourceLocation OpLoc, QualType convertedType);
-  QualType CheckCommaOperands( // C99 6.5.17
-    Expr *lex, Expr *&rex, SourceLocation OpLoc);
-  QualType CheckConditionalOperands( // C99 6.5.15
-    Expr *&cond, Expr *&lhs, Expr *&rhs, SourceLocation questionLoc);
-  QualType CXXCheckConditionalOperands( // C++ 5.16
-    Expr *&cond, Expr *&lhs, Expr *&rhs, SourceLocation questionLoc);
-  QualType FindCompositePointerType(SourceLocation Loc, Expr *&E1, Expr *&E2,
-                                    bool *NonStandardCompositeType = 0);
-
-  QualType FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
-                                        SourceLocation questionLoc);
-
-  /// type checking for vector binary operators.
-  QualType CheckVectorOperands(SourceLocation l, Expr *&lex, Expr *&rex);
-  QualType CheckVectorCompareOperands(Expr *&lex, Expr *&rx,
-                                      SourceLocation l, bool isRel);
-
-  /// type checking unary operators (subroutines of ActOnUnaryOp).
-  /// C99 6.5.3.1, 6.5.3.2, 6.5.3.4
-  QualType CheckIncrementDecrementOperand(Expr *op, SourceLocation OpLoc,
-                                          bool isInc, bool isPrefix);
-  QualType CheckAddressOfOperand(Expr *op, SourceLocation OpLoc);
-  QualType CheckIndirectionOperand(Expr *op, SourceLocation OpLoc);
-  QualType CheckRealImagOperand(Expr *&Op, SourceLocation OpLoc, bool isReal);
-
-  /// type checking primary expressions.
-  QualType CheckExtVectorComponent(QualType baseType, SourceLocation OpLoc,
-                                   const IdentifierInfo *Comp,
-                                   SourceLocation CmpLoc);
-
-  /// type checking declaration initializers (C99 6.7.8)
-  bool CheckInitList(const InitializedEntity &Entity,
-                     InitListExpr *&InitList, QualType &DeclType);
-  bool CheckForConstantInitializer(Expr *e, QualType t);
-
-  // type checking C++ declaration initializers (C++ [dcl.init]).
-
-  /// ReferenceCompareResult - Expresses the result of comparing two
-  /// types (cv1 T1 and cv2 T2) to determine their compatibility for the
-  /// purposes of initialization by reference (C++ [dcl.init.ref]p4).
-  enum ReferenceCompareResult {
-    /// Ref_Incompatible - The two types are incompatible, so direct
-    /// reference binding is not possible.
-    Ref_Incompatible = 0,
-    /// Ref_Related - The two types are reference-related, which means
-    /// that their unqualified forms (T1 and T2) are either the same
-    /// or T1 is a base class of T2.
-    Ref_Related,
-    /// Ref_Compatible_With_Added_Qualification - The two types are
-    /// reference-compatible with added qualification, meaning that
-    /// they are reference-compatible and the qualifiers on T1 (cv1)
-    /// are greater than the qualifiers on T2 (cv2).
-    Ref_Compatible_With_Added_Qualification,
-    /// Ref_Compatible - The two types are reference-compatible and
-    /// have equivalent qualifiers (cv1 == cv2).
-    Ref_Compatible
-  };
-
-  ReferenceCompareResult CompareReferenceRelationship(SourceLocation Loc,
-                                                      QualType T1, QualType T2,
-                                                      bool& DerivedToBase);
-
-  /// CheckCastTypes - Check type constraints for casting between types under
-  /// C semantics, or forward to CXXCheckCStyleCast in C++.
-  bool CheckCastTypes(SourceRange TyRange, QualType CastTy, Expr *&CastExpr,
-                      CastExpr::CastKind &Kind, CXXBaseSpecifierArray &BasePath,
-                      bool FunctionalStyle = false);
-
-  // CheckVectorCast - check type constraints for vectors.
-  // Since vectors are an extension, there are no C standard reference for this.
-  // We allow casting between vectors and integer datatypes of the same size.
-  // returns true if the cast is invalid
-  bool CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
-                       CastExpr::CastKind &Kind);
-
-  // CheckExtVectorCast - check type constraints for extended vectors.
-  // Since vectors are an extension, there are no C standard reference for this.
-  // We allow casting between vectors and integer datatypes of the same size,
-  // or vectors and the element type of that vector.
-  // returns true if the cast is invalid
-  bool CheckExtVectorCast(SourceRange R, QualType VectorTy, Expr *&CastExpr,
-                          CastExpr::CastKind &Kind);
-
-  /// CXXCheckCStyleCast - Check constraints of a C-style or function-style
-  /// cast under C++ semantics.
-  bool CXXCheckCStyleCast(SourceRange R, QualType CastTy, Expr *&CastExpr,
-                          CastExpr::CastKind &Kind,
-                          CXXBaseSpecifierArray &BasePath,
-                          bool FunctionalStyle);
-
-  /// CheckMessageArgumentTypes - Check types in an Obj-C message send.
-  /// \param Method - May be null.
-  /// \param [out] ReturnType - The return type of the send.
-  /// \return true iff there were any incompatible types.
-  bool CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs, Selector Sel,
-                                 ObjCMethodDecl *Method, bool isClassMessage,
-                                 SourceLocation lbrac, SourceLocation rbrac,
-                                 QualType &ReturnType);
-
-  /// CheckBooleanCondition - Diagnose problems involving the use of
-  /// the given expression as a boolean condition (e.g. in an if
-  /// statement).  Also performs the standard function and array
-  /// decays, possibly changing the input variable.
-  ///
-  /// \param Loc - A location associated with the condition, e.g. the
-  /// 'if' keyword.
-  /// \return true iff there were any errors
-  bool CheckBooleanCondition(Expr *&CondExpr, SourceLocation Loc);
-
-  virtual OwningExprResult ActOnBooleanCondition(Scope *S, SourceLocation Loc,
-                                                 ExprArg SubExpr);
-  
-  /// DiagnoseAssignmentAsCondition - Given that an expression is
-  /// being used as a boolean condition, warn if it's an assignment.
-  void DiagnoseAssignmentAsCondition(Expr *E);
-
-  /// CheckCXXBooleanCondition - Returns true if conversion to bool is invalid.
-  bool CheckCXXBooleanCondition(Expr *&CondExpr);
-
-  /// ConvertIntegerToTypeWarnOnOverflow - Convert the specified APInt to have
-  /// the specified width and sign.  If an overflow occurs, detect it and emit
-  /// the specified diagnostic.
-  void ConvertIntegerToTypeWarnOnOverflow(llvm::APSInt &OldVal,
-                                          unsigned NewWidth, bool NewSign,
-                                          SourceLocation Loc, unsigned DiagID);
-
-  /// Checks that the Objective-C declaration is declared in the global scope.
-  /// Emits an error and marks the declaration as invalid if it's not declared
-  /// in the global scope.
-  bool CheckObjCDeclScope(Decl *D);
-
-  void InitBuiltinVaListType();
-
-  /// VerifyIntegerConstantExpression - verifies that an expression is an ICE,
-  /// and reports the appropriate diagnostics. Returns false on success.
-  /// Can optionally return the value of the expression.
-  bool VerifyIntegerConstantExpression(const Expr *E, llvm::APSInt *Result = 0);
-
-  /// VerifyBitField - verifies that a bit field expression is an ICE and has
-  /// the correct width, and that the field type is valid.
-  /// Returns false on success.
-  /// Can optionally return whether the bit-field is of width 0
-  bool VerifyBitField(SourceLocation FieldLoc, IdentifierInfo *FieldName,
-                      QualType FieldTy, const Expr *BitWidth,
-                      bool *ZeroWidth = 0);
-
-  /// \name Code completion
-  //@{
-  virtual void CodeCompleteOrdinaryName(Scope *S,
-                                     CodeCompletionContext CompletionContext);
-  virtual void CodeCompleteExpression(Scope *S, QualType T);
-  virtual void CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *Base,
-                                               SourceLocation OpLoc,
-                                               bool IsArrow);
-  virtual void CodeCompleteTag(Scope *S, unsigned TagSpec);
-  virtual void CodeCompleteCase(Scope *S);
-  virtual void CodeCompleteCall(Scope *S, ExprTy *Fn,
-                                ExprTy **Args, unsigned NumArgs);
-  virtual void CodeCompleteInitializer(Scope *S, DeclPtrTy D);
-  virtual void CodeCompleteReturn(Scope *S);
-  virtual void CodeCompleteAssignmentRHS(Scope *S, ExprTy *LHS);
-  
-  virtual void CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
-                                       bool EnteringContext);
-  virtual void CodeCompleteUsing(Scope *S);
-  virtual void CodeCompleteUsingDirective(Scope *S);
-  virtual void CodeCompleteNamespaceDecl(Scope *S);
-  virtual void CodeCompleteNamespaceAliasDecl(Scope *S);
-  virtual void CodeCompleteOperatorName(Scope *S);
-
-  virtual void CodeCompleteObjCAtDirective(Scope *S, DeclPtrTy ObjCImpDecl,
-                                           bool InInterface);
-  virtual void CodeCompleteObjCAtVisibility(Scope *S);
-  virtual void CodeCompleteObjCAtStatement(Scope *S);
-  virtual void CodeCompleteObjCAtExpression(Scope *S);
-  virtual void CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS);
-  virtual void CodeCompleteObjCPropertyGetter(Scope *S, DeclPtrTy ClassDecl,
-                                              DeclPtrTy *Methods,
-                                              unsigned NumMethods);
-  virtual void CodeCompleteObjCPropertySetter(Scope *S, DeclPtrTy ClassDecl,
-                                              DeclPtrTy *Methods,
-                                              unsigned NumMethods);
-  virtual void CodeCompleteObjCMessageReceiver(Scope *S);
-  virtual void CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
-                                            IdentifierInfo **SelIdents,
-                                            unsigned NumSelIdents);
-  virtual void CodeCompleteObjCClassMessage(Scope *S, TypeTy *Receiver,
-                                            IdentifierInfo **SelIdents,
-                                            unsigned NumSelIdents);
-  virtual void CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
-                                               IdentifierInfo **SelIdents,
-                                               unsigned NumSelIdents);
-  virtual void CodeCompleteObjCProtocolReferences(IdentifierLocPair *Protocols,
-                                                  unsigned NumProtocols);
-  virtual void CodeCompleteObjCProtocolDecl(Scope *S);
-  virtual void CodeCompleteObjCInterfaceDecl(Scope *S);
-  virtual void CodeCompleteObjCSuperclass(Scope *S,
-                                          IdentifierInfo *ClassName,
-                                          SourceLocation ClassNameLoc);
-  virtual void CodeCompleteObjCImplementationDecl(Scope *S);
-  virtual void CodeCompleteObjCInterfaceCategory(Scope *S,
-                                                 IdentifierInfo *ClassName,
-                                                 SourceLocation ClassNameLoc);
-  virtual void CodeCompleteObjCImplementationCategory(Scope *S,
-                                                  IdentifierInfo *ClassName,
-                                                  SourceLocation ClassNameLoc);
-  virtual void CodeCompleteObjCPropertyDefinition(Scope *S,
-                                                  DeclPtrTy ObjCImpDecl);
-  virtual void CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
-                                                  IdentifierInfo *PropertyName,
-                                                      DeclPtrTy ObjCImpDecl);
-  virtual void CodeCompleteObjCMethodDecl(Scope *S,
-                                          bool IsInstanceMethod,
-                                          TypeTy *ReturnType,
-                                          DeclPtrTy IDecl);
-  virtual void CodeCompleteObjCMethodDeclSelector(Scope *S, 
-                                                  bool IsInstanceMethod,
-                                                  bool AtParameterName,
-                                                  TypeTy *ReturnType,
-                                                  IdentifierInfo **SelIdents,
-                                                  unsigned NumSelIdents);
-  
-  //@}
-
-  //===--------------------------------------------------------------------===//
-  // Extra semantic analysis beyond the C type system
-
-public:
-  SourceLocation getLocationOfStringLiteralByte(const StringLiteral *SL,
-                                                unsigned ByteNo) const;
-
-private:
-  bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall);
-  bool CheckBlockCall(NamedDecl *NDecl, CallExpr *TheCall);
-
-  bool CheckablePrintfAttr(const FormatAttr *Format, CallExpr *TheCall);
-  bool CheckObjCString(Expr *Arg);
-
-  Action::OwningExprResult CheckBuiltinFunctionCall(unsigned BuiltinID,
-                                                    CallExpr *TheCall);
-  bool CheckARMBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
-  bool CheckX86BuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
-
-  bool SemaBuiltinVAStart(CallExpr *TheCall);
-  bool SemaBuiltinUnorderedCompare(CallExpr *TheCall);
-  bool SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs);
-
-public:
-  // Used by C++ template instantiation.
-  Action::OwningExprResult SemaBuiltinShuffleVector(CallExpr *TheCall);
-
-private:
-  bool SemaBuiltinPrefetch(CallExpr *TheCall);
-  bool SemaBuiltinObjectSize(CallExpr *TheCall);
-  bool SemaBuiltinLongjmp(CallExpr *TheCall);
-  OwningExprResult SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult);
-  bool SemaBuiltinConstantArg(CallExpr *TheCall, int ArgNum,
-                              llvm::APSInt &Result);
-  bool SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
-                              bool HasVAListArg, unsigned format_idx,
-                              unsigned firstDataArg);
-  void CheckPrintfString(const StringLiteral *FExpr, const Expr *OrigFormatExpr,
-                         const CallExpr *TheCall, bool HasVAListArg,
-                         unsigned format_idx, unsigned firstDataArg);
-  void CheckNonNullArguments(const NonNullAttr *NonNull,
-                             const CallExpr *TheCall);
-  void CheckPrintfArguments(const CallExpr *TheCall, bool HasVAListArg,
-                            unsigned format_idx, unsigned firstDataArg);
-  void CheckReturnStackAddr(Expr *RetValExp, QualType lhsType,
-                            SourceLocation ReturnLoc);
-  void CheckFloatComparison(SourceLocation loc, Expr* lex, Expr* rex);
-  void CheckImplicitConversions(Expr *E);
-};
-
-//===--------------------------------------------------------------------===//
-// Typed version of Parser::ExprArg (smart pointer for wrapping Expr pointers).
-template <typename T>
-class ExprOwningPtr : public Action::ExprArg {
-public:
-  ExprOwningPtr(Sema *S, T *expr) : Action::ExprArg(*S, expr) {}
-
-  void reset(T* p) { Action::ExprArg::operator=(p); }
-  T* get() const { return static_cast<T*>(Action::ExprArg::get()); }
-  T* take() { return static_cast<T*>(Action::ExprArg::take()); }
-  T* release() { return take(); }
-
-  T& operator*() const { return *get(); }
-  T* operator->() const { return get(); }
-};
-
-}  // end namespace clang
-
-#endif
diff --git a/lib/Sema/SemaAccess.cpp b/lib/Sema/SemaAccess.cpp
index e110e3dfa471..e629f0fd35bf 100644
--- a/lib/Sema/SemaAccess.cpp
+++ b/lib/Sema/SemaAccess.cpp
@@ -11,9 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/DelayedDiagnostic.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclCXX.h"
@@ -22,6 +23,7 @@
 #include "clang/AST/ExprCXX.h"
 
 using namespace clang;
+using namespace sema;
 
 /// A copy of Sema's enum without AR_delayed.
 enum AccessResult {
@@ -132,10 +134,10 @@ struct EffectiveContext {
   bool Dependent;
 };
 
-/// Like Sema's AccessedEntity, but kindly lets us scribble all over
+/// Like sema:;AccessedEntity, but kindly lets us scribble all over
 /// it.
-struct AccessTarget : public Sema::AccessedEntity {
-  AccessTarget(const Sema::AccessedEntity &Entity)
+struct AccessTarget : public AccessedEntity {
+  AccessTarget(const AccessedEntity &Entity)
     : AccessedEntity(Entity) {
     initialize();
   }
@@ -562,6 +564,130 @@ static AccessResult GetFriendKind(Sema &S,
   return OnFailure;
 }
 
+namespace {
+
+/// A helper class for checking for a friend which will grant access
+/// to a protected instance member.
+struct ProtectedFriendContext {
+  Sema &S;
+  const EffectiveContext &EC;
+  const CXXRecordDecl *NamingClass;
+  bool CheckDependent;
+  bool EverDependent;
+
+  /// The path down to the current base class.
+  llvm::SmallVector<const CXXRecordDecl*, 20> CurPath;
+
+  ProtectedFriendContext(Sema &S, const EffectiveContext &EC,
+                         const CXXRecordDecl *InstanceContext,
+                         const CXXRecordDecl *NamingClass)
+    : S(S), EC(EC), NamingClass(NamingClass),
+      CheckDependent(InstanceContext->isDependentContext() ||
+                     NamingClass->isDependentContext()),
+      EverDependent(false) {}
+
+  /// Check classes in the current path for friendship, starting at
+  /// the given index.
+  bool checkFriendshipAlongPath(unsigned I) {
+    assert(I < CurPath.size());
+    for (unsigned E = CurPath.size(); I != E; ++I) {
+      switch (GetFriendKind(S, EC, CurPath[I])) {
+      case AR_accessible:   return true;
+      case AR_inaccessible: continue;
+      case AR_dependent:    EverDependent = true; continue;
+      }
+    }
+    return false;
+  }
+
+  /// Perform a search starting at the given class.
+  ///
+  /// PrivateDepth is the index of the last (least derived) class
+  /// along the current path such that a notional public member of
+  /// the final class in the path would have access in that class.
+  bool findFriendship(const CXXRecordDecl *Cur, unsigned PrivateDepth) {
+    // If we ever reach the naming class, check the current path for
+    // friendship.  We can also stop recursing because we obviously
+    // won't find the naming class there again.
+    if (Cur == NamingClass)
+      return checkFriendshipAlongPath(PrivateDepth);
+
+    if (CheckDependent && MightInstantiateTo(Cur, NamingClass))
+      EverDependent = true;
+
+    // Recurse into the base classes.
+    for (CXXRecordDecl::base_class_const_iterator
+           I = Cur->bases_begin(), E = Cur->bases_end(); I != E; ++I) {
+
+      // If this is private inheritance, then a public member of the
+      // base will not have any access in classes derived from Cur.
+      unsigned BasePrivateDepth = PrivateDepth;
+      if (I->getAccessSpecifier() == AS_private)
+        BasePrivateDepth = CurPath.size() - 1;
+
+      const CXXRecordDecl *RD;
+
+      QualType T = I->getType();
+      if (const RecordType *RT = T->getAs<RecordType>()) {
+        RD = cast<CXXRecordDecl>(RT->getDecl());
+      } else if (const InjectedClassNameType *IT
+                   = T->getAs<InjectedClassNameType>()) {
+        RD = IT->getDecl();
+      } else {
+        assert(T->isDependentType() && "non-dependent base wasn't a record?");
+        EverDependent = true;
+        continue;
+      }
+
+      // Recurse.  We don't need to clean up if this returns true.
+      CurPath.push_back(RD);
+      if (findFriendship(RD->getCanonicalDecl(), BasePrivateDepth))
+        return true;
+      CurPath.pop_back();
+    }
+
+    return false;
+  }
+
+  bool findFriendship(const CXXRecordDecl *Cur) {
+    assert(CurPath.empty());
+    CurPath.push_back(Cur);
+    return findFriendship(Cur, 0);
+  }
+};
+}
+
+/// Search for a class P that EC is a friend of, under the constraint
+///   InstanceContext <= P <= NamingClass
+/// and with the additional restriction that a protected member of
+/// NamingClass would have some natural access in P.
+///
+/// That second condition isn't actually quite right: the condition in
+/// the standard is whether the target would have some natural access
+/// in P.  The difference is that the target might be more accessible
+/// along some path not passing through NamingClass.  Allowing that
+/// introduces two problems:
+///   - It breaks encapsulation because you can suddenly access a
+///     forbidden base class's members by subclassing it elsewhere.
+///   - It makes access substantially harder to compute because it
+///     breaks the hill-climbing algorithm: knowing that the target is
+///     accessible in some base class would no longer let you change
+///     the question solely to whether the base class is accessible,
+///     because the original target might have been more accessible
+///     because of crazy subclassing.
+/// So we don't implement that.
+static AccessResult GetProtectedFriendKind(Sema &S, const EffectiveContext &EC,
+                                           const CXXRecordDecl *InstanceContext,
+                                           const CXXRecordDecl *NamingClass) {
+  assert(InstanceContext->getCanonicalDecl() == InstanceContext);
+  assert(NamingClass->getCanonicalDecl() == NamingClass);
+
+  ProtectedFriendContext PRC(S, EC, InstanceContext, NamingClass);
+  if (PRC.findFriendship(InstanceContext)) return AR_accessible;
+  if (PRC.EverDependent) return AR_dependent;
+  return AR_inaccessible;
+}
+
 static AccessResult HasAccess(Sema &S,
                               const EffectiveContext &EC,
                               const CXXRecordDecl *NamingClass,
@@ -629,20 +755,25 @@ static AccessResult HasAccess(Sema &S,
     }
   }
 
-  if (!NamingClass->hasFriends())
-    return OnFailure;
-
-  // Don't consider friends if we're under the [class.protected]
-  // restriction, above.
+  // [M3] and [B3] say that, if the target is protected in N, we grant
+  // access if the access occurs in a friend or member of some class P
+  // that's a subclass of N and where the target has some natural
+  // access in P.  The 'member' aspect is easy to handle because P
+  // would necessarily be one of the effective-context records, and we
+  // address that above.  The 'friend' aspect is completely ridiculous
+  // to implement because there are no restrictions at all on P
+  // *unless* the [class.protected] restriction applies.  If it does,
+  // however, we should ignore whether the naming class is a friend,
+  // and instead rely on whether any potential P is a friend.
   if (Access == AS_protected && Target.hasInstanceContext()) {
     const CXXRecordDecl *InstanceContext = Target.resolveInstanceContext(S);
     if (!InstanceContext) return AR_dependent;
-
-    switch (IsDerivedFromInclusive(InstanceContext, NamingClass)) {
-    case AR_accessible: break;
+    switch (GetProtectedFriendKind(S, EC, InstanceContext, NamingClass)) {
+    case AR_accessible: return AR_accessible;
     case AR_inaccessible: return OnFailure;
     case AR_dependent: return AR_dependent;
     }
+    llvm_unreachable("impossible friendship kind");
   }
 
   switch (GetFriendKind(S, EC, NamingClass)) {
@@ -799,6 +930,57 @@ static CXXBasePath *FindBestPath(Sema &S,
   return BestPath;
 }
 
+/// Given that an entity has protected natural access, check whether
+/// access might be denied because of the protected member access
+/// restriction.
+///
+/// \return true if a note was emitted
+static bool TryDiagnoseProtectedAccess(Sema &S, const EffectiveContext &EC,
+                                       AccessTarget &Target) {
+  // Only applies to instance accesses.
+  if (!Target.hasInstanceContext())
+    return false;
+  assert(Target.isMemberAccess());
+  NamedDecl *D = Target.getTargetDecl();
+
+  const CXXRecordDecl *DeclaringClass = Target.getDeclaringClass();
+  DeclaringClass = DeclaringClass->getCanonicalDecl();
+
+  for (EffectiveContext::record_iterator
+         I = EC.Records.begin(), E = EC.Records.end(); I != E; ++I) {
+    const CXXRecordDecl *ECRecord = *I;
+    switch (IsDerivedFromInclusive(ECRecord, DeclaringClass)) {
+    case AR_accessible: break;
+    case AR_inaccessible: continue;
+    case AR_dependent: continue;
+    }
+
+    // The effective context is a subclass of the declaring class.
+    // If that class isn't a superclass of the instance context,
+    // then the [class.protected] restriction applies.
+
+    // To get this exactly right, this might need to be checked more
+    // holistically;  it's not necessarily the case that gaining
+    // access here would grant us access overall.
+
+    const CXXRecordDecl *InstanceContext = Target.resolveInstanceContext(S);
+    assert(InstanceContext && "diagnosing dependent access");
+
+    switch (IsDerivedFromInclusive(InstanceContext, ECRecord)) {
+    case AR_accessible: continue;
+    case AR_dependent: continue;
+    case AR_inaccessible:
+      S.Diag(D->getLocation(), diag::note_access_protected_restricted)
+        << (InstanceContext != Target.getNamingClass()->getCanonicalDecl())
+        << S.Context.getTypeDeclType(InstanceContext)
+        << S.Context.getTypeDeclType(ECRecord);
+      return true;
+    }
+  }
+
+  return false;
+}
+
 /// Diagnose the path which caused the given declaration or base class
 /// to become inaccessible.
 static void DiagnoseAccessPath(Sema &S,
@@ -817,6 +999,10 @@ static void DiagnoseAccessPath(Sema &S,
   if (D && (Access == D->getAccess() || D->getAccess() == AS_private)) {
     switch (HasAccess(S, EC, DeclaringClass, D->getAccess(), Entity)) {
     case AR_inaccessible: {
+      if (Access == AS_protected &&
+          TryDiagnoseProtectedAccess(S, EC, Entity))
+        return;
+
       S.Diag(D->getLocation(), diag::note_access_natural)
         << (unsigned) (Access == AS_protected)
         << /*FIXME: not implicitly*/ 0;
@@ -1033,7 +1219,7 @@ static Sema::AccessResult CheckAccess(Sema &S, SourceLocation Loc,
   // access control.
   if (S.CurContext->isFileContext() && S.ParsingDeclDepth) {
     S.DelayedDiagnostics.push_back(
-        Sema::DelayedDiagnostic::makeAccess(Loc, Entity));
+        DelayedDiagnostic::makeAccess(Loc, Entity));
     return Sema::AR_delayed;
   }
 
@@ -1266,7 +1452,7 @@ Sema::AccessResult Sema::CheckAddressOfMemberAccess(Expr *OvlExpr,
       Found.getAccess() == AS_public)
     return AR_accessible;
 
-  OverloadExpr *Ovl = OverloadExpr::find(OvlExpr).getPointer();
+  OverloadExpr *Ovl = OverloadExpr::find(OvlExpr).Expression;
   CXXRecordDecl *NamingClass = Ovl->getNamingClass();
 
   AccessTarget Entity(Context, AccessTarget::Member, NamingClass, Found,
diff --git a/lib/Sema/SemaAttr.cpp b/lib/Sema/SemaAttr.cpp
index 69f27b0ada7d..0921156b932f 100644
--- a/lib/Sema/SemaAttr.cpp
+++ b/lib/Sema/SemaAttr.cpp
@@ -12,8 +12,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/Expr.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
@@ -62,18 +63,30 @@ namespace {
     /// alignment to the previous value. If \arg Name is non-zero then
     /// the first such named record is popped, otherwise the top record
     /// is popped. Returns true if the pop succeeded.
-    bool pop(IdentifierInfo *Name);
+    bool pop(IdentifierInfo *Name, bool IsReset);
   };
 }  // end anonymous namespace.
 
-bool PragmaPackStack::pop(IdentifierInfo *Name) {
-  if (Stack.empty())
-    return false;
-
+bool PragmaPackStack::pop(IdentifierInfo *Name, bool IsReset) {
   // If name is empty just pop top.
   if (!Name) {
-    Alignment = Stack.back().Alignment;
-    Stack.pop_back();
+    // An empty stack is a special case...
+    if (Stack.empty()) {
+      // If this isn't a reset, it is always an error.
+      if (!IsReset)
+        return false;
+
+      // Otherwise, it is an error only if some alignment has been set.
+      if (!Alignment)
+        return false;
+
+      // Otherwise, reset to the default alignment.
+      Alignment = 0;
+    } else {
+      Alignment = Stack.back().Alignment;
+      Stack.pop_back();
+    }
+
     return true;
   }
 
@@ -108,9 +121,11 @@ void Sema::AddAlignmentAttributesForRecord(RecordDecl *RD) {
   // Otherwise, check to see if we need a max field alignment attribute.
   if (unsigned Alignment = Stack->getAlignment()) {
     if (Alignment == PackStackEntry::kMac68kAlignmentSentinel)
-      RD->addAttr(::new (Context) AlignMac68kAttr());
+      RD->addAttr(::new (Context) AlignMac68kAttr(SourceLocation(), Context));
     else
-      RD->addAttr(::new (Context) MaxFieldAlignmentAttr(Alignment * 8));
+      RD->addAttr(::new (Context) MaxFieldAlignmentAttr(SourceLocation(),
+                                                        Context,
+                                                        Alignment * 8));
   }
 }
 
@@ -122,13 +137,10 @@ void Sema::ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind,
 
   PragmaPackStack *Context = static_cast<PragmaPackStack*>(PackContext);
 
-  // Reset just pops the top of the stack.
-  if (Kind == Action::POAK_Reset) {
-    // Do the pop.
-    if (!Context->pop(0)) {
-      // If a name was specified then failure indicates the name
-      // wasn't found. Otherwise failure indicates the stack was
-      // empty.
+  // Reset just pops the top of the stack, or resets the current alignment to
+  // default.
+  if (Kind == Sema::POAK_Reset) {
+    if (!Context->pop(0, /*IsReset=*/true)) {
       Diag(PragmaLoc, diag::warn_pragma_options_align_reset_failed)
         << "stack empty";
     }
@@ -188,7 +200,6 @@ void Sema::ActOnPragmaPack(PragmaPackKind Kind, IdentifierInfo *Name,
         !(Val == 0 || Val.isPowerOf2()) ||
         Val.getZExtValue() > 16) {
       Diag(PragmaLoc, diag::warn_pragma_pack_invalid_alignment);
-      Alignment->Destroy(Context);
       return; // Ignore
     }
 
@@ -201,11 +212,11 @@ void Sema::ActOnPragmaPack(PragmaPackKind Kind, IdentifierInfo *Name,
   PragmaPackStack *Context = static_cast<PragmaPackStack*>(PackContext);
 
   switch (Kind) {
-  case Action::PPK_Default: // pack([n])
+  case Sema::PPK_Default: // pack([n])
     Context->setAlignment(AlignmentVal);
     break;
 
-  case Action::PPK_Show: // pack(show)
+  case Sema::PPK_Show: // pack(show)
     // Show the current alignment, making sure to show the right value
     // for the default.
     AlignmentVal = Context->getAlignment();
@@ -218,21 +229,21 @@ void Sema::ActOnPragmaPack(PragmaPackKind Kind, IdentifierInfo *Name,
       Diag(PragmaLoc, diag::warn_pragma_pack_show) << AlignmentVal;
     break;
 
-  case Action::PPK_Push: // pack(push [, id] [, [n])
+  case Sema::PPK_Push: // pack(push [, id] [, [n])
     Context->push(Name);
     // Set the new alignment if specified.
     if (Alignment)
       Context->setAlignment(AlignmentVal);
     break;
 
-  case Action::PPK_Pop: // pack(pop [, id] [,  n])
+  case Sema::PPK_Pop: // pack(pop [, id] [,  n])
     // MSDN, C/C++ Preprocessor Reference > Pragma Directives > pack:
     // "#pragma pack(pop, identifier, n) is undefined"
     if (Alignment && Name)
       Diag(PragmaLoc, diag::warn_pragma_pack_pop_identifer_and_alignment);
 
     // Do the pop.
-    if (!Context->pop(Name)) {
+    if (!Context->pop(Name, /*IsReset=*/false)) {
       // If a name was specified then failure indicates the name
       // wasn't found. Otherwise failure indicates the stack was
       // empty.
@@ -277,6 +288,80 @@ void Sema::ActOnPragmaUnused(const Token *Identifiers, unsigned NumIdentifiers,
       continue;
     }
 
-    VD->addAttr(::new (Context) UnusedAttr());
+    VD->addAttr(::new (Context) UnusedAttr(Tok.getLocation(), Context));
+  }
+}
+
+typedef std::vector<std::pair<VisibilityAttr::VisibilityType,
+                              SourceLocation> > VisStack;
+
+void Sema::AddPushedVisibilityAttribute(Decl *D) {
+  if (!VisContext)
+    return;
+
+  if (D->hasAttr<VisibilityAttr>())
+    return;
+
+  VisStack *Stack = static_cast<VisStack*>(VisContext);
+  VisibilityAttr::VisibilityType type = Stack->back().first;
+  SourceLocation loc = Stack->back().second;
+
+  D->addAttr(::new (Context) VisibilityAttr(loc, Context, type));
+}
+
+/// FreeVisContext - Deallocate and null out VisContext.
+void Sema::FreeVisContext() {
+  delete static_cast<VisStack*>(VisContext);
+  VisContext = 0;
+}
+
+static void PushPragmaVisibility(Sema &S, VisibilityAttr::VisibilityType type,
+                                 SourceLocation loc) {
+  // Put visibility on stack.
+  if (!S.VisContext)
+    S.VisContext = new VisStack;
+
+  VisStack *Stack = static_cast<VisStack*>(S.VisContext);
+  Stack->push_back(std::make_pair(type, loc));
+}
+
+void Sema::ActOnPragmaVisibility(bool IsPush, const IdentifierInfo* VisType,
+                                 SourceLocation PragmaLoc) {
+  if (IsPush) {
+    // Compute visibility to use.
+    VisibilityAttr::VisibilityType type;
+    if (VisType->isStr("default"))
+      type = VisibilityAttr::Default;
+    else if (VisType->isStr("hidden"))
+      type = VisibilityAttr::Hidden;
+    else if (VisType->isStr("internal"))
+      type = VisibilityAttr::Hidden; // FIXME
+    else if (VisType->isStr("protected"))
+      type = VisibilityAttr::Protected;
+    else {
+      Diag(PragmaLoc, diag::warn_attribute_unknown_visibility) <<
+        VisType->getName();
+      return;
+    }
+    PushPragmaVisibility(*this, type, PragmaLoc);
+  } else {
+    PopPragmaVisibility();
+  }
+}
+
+void Sema::PushVisibilityAttr(const VisibilityAttr *Attr) {
+  PushPragmaVisibility(*this, Attr->getVisibility(), Attr->getLocation());
+}
+
+void Sema::PopPragmaVisibility() {
+  // Pop visibility from stack, if there is one on the stack.
+  if (VisContext) {
+    VisStack *Stack = static_cast<VisStack*>(VisContext);
+
+    Stack->pop_back();
+    // To simplify the implementation, never keep around an empty stack.
+    if (Stack->empty())
+      FreeVisContext();
   }
+  // FIXME: Add diag for pop without push.
 }
diff --git a/lib/Sema/SemaCXXCast.cpp b/lib/Sema/SemaCXXCast.cpp
index b8e27e7b7285..21b1a73aa3bc 100644
--- a/lib/Sema/SemaCXXCast.cpp
+++ b/lib/Sema/SemaCXXCast.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Initialization.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
@@ -43,16 +43,16 @@ static void CheckConstCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 static void CheckReinterpretCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                                  const SourceRange &OpRange,
                                  const SourceRange &DestRange,
-                                 CastExpr::CastKind &Kind);
+                                 CastKind &Kind);
 static void CheckStaticCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                             const SourceRange &OpRange,
-                            CastExpr::CastKind &Kind,
-                            CXXBaseSpecifierArray &BasePath);
+                            CastKind &Kind,
+                            CXXCastPath &BasePath);
 static void CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                              const SourceRange &OpRange,
                              const SourceRange &DestRange,
-                             CastExpr::CastKind &Kind,
-                             CXXBaseSpecifierArray &BasePath);
+                             CastKind &Kind,
+                             CXXCastPath &BasePath);
 
 static bool CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType);
 
@@ -73,54 +73,54 @@ static TryCastResult TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr,
                                                QualType DestType, bool CStyle,
                                                const SourceRange &OpRange,
                                                unsigned &msg,
-                                               CastExpr::CastKind &Kind,
-                                               CXXBaseSpecifierArray &BasePath);
+                                               CastKind &Kind,
+                                               CXXCastPath &BasePath);
 static TryCastResult TryStaticPointerDowncast(Sema &Self, QualType SrcType,
                                               QualType DestType, bool CStyle,
                                               const SourceRange &OpRange,
                                               unsigned &msg,
-                                              CastExpr::CastKind &Kind,
-                                              CXXBaseSpecifierArray &BasePath);
+                                              CastKind &Kind,
+                                              CXXCastPath &BasePath);
 static TryCastResult TryStaticDowncast(Sema &Self, CanQualType SrcType,
                                        CanQualType DestType, bool CStyle,
                                        const SourceRange &OpRange,
                                        QualType OrigSrcType,
                                        QualType OrigDestType, unsigned &msg,
-                                       CastExpr::CastKind &Kind,
-                                       CXXBaseSpecifierArray &BasePath);
+                                       CastKind &Kind,
+                                       CXXCastPath &BasePath);
 static TryCastResult TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr,
                                                QualType SrcType,
                                                QualType DestType,bool CStyle,
                                                const SourceRange &OpRange,
                                                unsigned &msg,
-                                               CastExpr::CastKind &Kind,
-                                               CXXBaseSpecifierArray &BasePath);
+                                               CastKind &Kind,
+                                               CXXCastPath &BasePath);
 
 static TryCastResult TryStaticImplicitCast(Sema &Self, Expr *&SrcExpr,
                                            QualType DestType, bool CStyle,
                                            const SourceRange &OpRange,
                                            unsigned &msg,
-                                           CastExpr::CastKind &Kind);
+                                           CastKind &Kind);
 static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
                                    QualType DestType, bool CStyle,
                                    const SourceRange &OpRange,
                                    unsigned &msg,
-                                   CastExpr::CastKind &Kind,
-                                   CXXBaseSpecifierArray &BasePath);
+                                   CastKind &Kind,
+                                   CXXCastPath &BasePath);
 static TryCastResult TryConstCast(Sema &Self, Expr *SrcExpr, QualType DestType,
                                   bool CStyle, unsigned &msg);
 static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
                                         QualType DestType, bool CStyle,
                                         const SourceRange &OpRange,
                                         unsigned &msg,
-                                        CastExpr::CastKind &Kind);
+                                        CastKind &Kind);
 
 /// ActOnCXXNamedCast - Parse {dynamic,static,reinterpret,const}_cast's.
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
-                        SourceLocation LAngleBracketLoc, TypeTy *Ty,
+                        SourceLocation LAngleBracketLoc, ParsedType Ty,
                         SourceLocation RAngleBracketLoc,
-                        SourceLocation LParenLoc, ExprArg E,
+                        SourceLocation LParenLoc, Expr *E,
                         SourceLocation RParenLoc) {
   
   TypeSourceInfo *DestTInfo;
@@ -133,11 +133,10 @@ Sema::ActOnCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
                            SourceRange(LParenLoc, RParenLoc));
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
-                        TypeSourceInfo *DestTInfo, ExprArg E,
+                        TypeSourceInfo *DestTInfo, Expr *Ex,
                         SourceRange AngleBrackets, SourceRange Parens) {
-  Expr *Ex = E.takeAs<Expr>();
   QualType DestType = DestTInfo->getType();
 
   SourceRange OpRange(OpLoc, Parens.getEnd());
@@ -153,39 +152,39 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
   case tok::kw_const_cast:
     if (!TypeDependent)
       CheckConstCast(*this, Ex, DestType, OpRange, DestRange);
-    return Owned(new (Context) CXXConstCastExpr(
+    return Owned(CXXConstCastExpr::Create(Context,
                                         DestType.getNonLValueExprType(Context),
-                                                Ex, DestTInfo, OpLoc));
+                                          Ex, DestTInfo, OpLoc));
 
   case tok::kw_dynamic_cast: {
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-    CXXBaseSpecifierArray BasePath;
+    CastKind Kind = CK_Unknown;
+    CXXCastPath BasePath;
     if (!TypeDependent)
       CheckDynamicCast(*this, Ex, DestType, OpRange, DestRange, Kind, BasePath);
-    return Owned(new (Context)CXXDynamicCastExpr(
+    return Owned(CXXDynamicCastExpr::Create(Context,
                                           DestType.getNonLValueExprType(Context),
-                                                 Kind, Ex, BasePath, DestTInfo,
-                                                 OpLoc));
+                                            Kind, Ex, &BasePath, DestTInfo,
+                                            OpLoc));
   }
   case tok::kw_reinterpret_cast: {
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
+    CastKind Kind = CK_Unknown;
     if (!TypeDependent)
       CheckReinterpretCast(*this, Ex, DestType, OpRange, DestRange, Kind);
-    return Owned(new (Context) CXXReinterpretCastExpr(
+    return Owned(CXXReinterpretCastExpr::Create(Context,
                                   DestType.getNonLValueExprType(Context),
-                                  Kind, Ex, CXXBaseSpecifierArray(), 
+                                  Kind, Ex, 0,
                                   DestTInfo, OpLoc));
   }
   case tok::kw_static_cast: {
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-    CXXBaseSpecifierArray BasePath;
+    CastKind Kind = CK_Unknown;
+    CXXCastPath BasePath;
     if (!TypeDependent)
       CheckStaticCast(*this, Ex, DestType, OpRange, Kind, BasePath);
     
-    return Owned(new (Context) CXXStaticCastExpr(
+    return Owned(CXXStaticCastExpr::Create(Context,
                                          DestType.getNonLValueExprType(Context),
-                                                 Kind, Ex, BasePath,
-                                                 DestTInfo, OpLoc));
+                                           Kind, Ex, &BasePath,
+                                           DestTInfo, OpLoc));
   }
   }
 
@@ -197,7 +196,7 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
 /// the same kind of pointer (plain or to-member). Unlike the Sema function,
 /// this one doesn't care if the two pointers-to-member don't point into the
 /// same class. This is because CastsAwayConstness doesn't care.
-bool UnwrapDissimilarPointerTypes(QualType& T1, QualType& T2) {
+static bool UnwrapDissimilarPointerTypes(QualType& T1, QualType& T2) {
   const PointerType *T1PtrType = T1->getAs<PointerType>(),
                     *T2PtrType = T2->getAs<PointerType>();
   if (T1PtrType && T2PtrType) {
@@ -307,8 +306,8 @@ CastsAwayConstness(Sema &Self, QualType SrcType, QualType DestType) {
 static void
 CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                  const SourceRange &OpRange,
-                 const SourceRange &DestRange, CastExpr::CastKind &Kind,
-                 CXXBaseSpecifierArray &BasePath) {
+                 const SourceRange &DestRange, CastKind &Kind,
+                 CXXCastPath &BasePath) {
   QualType OrigDestType = DestType, OrigSrcType = SrcExpr->getType();
   DestType = Self.Context.getCanonicalType(DestType);
 
@@ -396,6 +395,7 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
   // C++ 5.2.7p3: If the type of v is the same as the required result type,
   //   [except for cv].
   if (DestRecord == SrcRecord) {
+    Kind = CK_NoOp;
     return;
   }
 
@@ -407,7 +407,7 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                                            &BasePath))
         return;
         
-    Kind = CastExpr::CK_DerivedToBase;
+    Kind = CK_DerivedToBase;
 
     // If we are casting to or through a virtual base class, we need a
     // vtable.
@@ -428,7 +428,7 @@ CheckDynamicCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                       cast<CXXRecordDecl>(SrcRecord->getDecl()));
 
   // Done. Everything else is run-time checks.
-  Kind = CastExpr::CK_Dynamic;
+  Kind = CK_Dynamic;
 }
 
 /// CheckConstCast - Check that a const_cast\<DestType\>(SrcExpr) is valid.
@@ -457,7 +457,7 @@ CheckConstCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 void
 CheckReinterpretCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                      const SourceRange &OpRange, const SourceRange &DestRange,
-                     CastExpr::CastKind &Kind) {
+                     CastKind &Kind) {
   if (!DestType->isLValueReferenceType())
     Self.DefaultFunctionArrayLvalueConversion(SrcExpr);
 
@@ -475,13 +475,13 @@ CheckReinterpretCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 /// implicit conversions explicit and getting rid of data loss warnings.
 void
 CheckStaticCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
-                const SourceRange &OpRange, CastExpr::CastKind &Kind,
-                CXXBaseSpecifierArray &BasePath) {
+                const SourceRange &OpRange, CastKind &Kind,
+                CXXCastPath &BasePath) {
   // This test is outside everything else because it's the only case where
   // a non-lvalue-reference target type does not lead to decay.
   // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
   if (DestType->isVoidType()) {
-    Kind = CastExpr::CK_ToVoid;
+    Kind = CK_ToVoid;
     return;
   }
 
@@ -493,6 +493,8 @@ CheckStaticCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                     Kind, BasePath) != TC_Success && msg != 0)
     Self.Diag(OpRange.getBegin(), msg) << CT_Static
       << SrcExpr->getType() << DestType << OpRange;
+  else if (Kind == CK_Unknown || Kind == CK_BitCast)
+    Self.CheckCastAlign(SrcExpr, DestType, OpRange);
 }
 
 /// TryStaticCast - Check if a static cast can be performed, and do so if
@@ -501,8 +503,8 @@ CheckStaticCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
 static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
                                    QualType DestType, bool CStyle,
                                    const SourceRange &OpRange, unsigned &msg,
-                                   CastExpr::CastKind &Kind,
-                                   CXXBaseSpecifierArray &BasePath) {
+                                   CastKind &Kind,
+                                   CXXCastPath &BasePath) {
   // The order the tests is not entirely arbitrary. There is one conversion
   // that can be handled in two different ways. Given:
   // struct A {};
@@ -532,7 +534,7 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
   //   reference to cv2 T2" if "cv2 T2" is reference-compatible with "cv1 T1".
   tcr = TryLValueToRValueCast(Self, SrcExpr, DestType, msg);
   if (tcr != TC_NotApplicable) {
-    Kind = CastExpr::CK_NoOp;
+    Kind = CK_NoOp;
     return tcr;
   }
 
@@ -567,7 +569,7 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
     if (SrcType->isComplexType() || SrcType->isVectorType()) {
       // Fall through - these cannot be converted.
     } else if (SrcType->isArithmeticType() || SrcType->isEnumeralType()) {
-      Kind = CastExpr::CK_IntegralCast;
+      Kind = CK_IntegralCast;
       return TC_Success;
     }
   }
@@ -602,19 +604,19 @@ static TryCastResult TryStaticCast(Sema &Self, Expr *&SrcExpr,
             msg = diag::err_bad_cxx_cast_const_away;
             return TC_Failed;
           }
-          Kind = CastExpr::CK_BitCast;
+          Kind = CK_BitCast;
           return TC_Success;
         }
       }
       else if (DestType->isObjCObjectPointerType()) {
         // allow both c-style cast and static_cast of objective-c pointers as 
         // they are pervasive.
-        Kind = CastExpr::CK_AnyPointerToObjCPointerCast;
+        Kind = CK_AnyPointerToObjCPointerCast;
         return TC_Success;
       }
       else if (CStyle && DestType->isBlockPointerType()) {
         // allow c-style cast of void * to block pointers.
-        Kind = CastExpr::CK_AnyPointerToBlockPointerCast;
+        Kind = CK_AnyPointerToBlockPointerCast;
         return TC_Success;
       }
     }
@@ -645,9 +647,10 @@ TryLValueToRValueCast(Sema &Self, Expr *SrcExpr, QualType DestType,
   // this is the only cast possibility, so we issue an error if we fail now.
   // FIXME: Should allow casting away constness if CStyle.
   bool DerivedToBase;
+  bool ObjCConversion;
   if (Self.CompareReferenceRelationship(SrcExpr->getLocStart(),
                                         SrcExpr->getType(), R->getPointeeType(),
-                                        DerivedToBase) <
+                                        DerivedToBase, ObjCConversion) <
         Sema::Ref_Compatible_With_Added_Qualification) {
     msg = diag::err_bad_lvalue_to_rvalue_cast;
     return TC_Failed;
@@ -662,8 +665,8 @@ TryLValueToRValueCast(Sema &Self, Expr *SrcExpr, QualType DestType,
 TryCastResult
 TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr, QualType DestType,
                            bool CStyle, const SourceRange &OpRange,
-                           unsigned &msg, CastExpr::CastKind &Kind,
-                           CXXBaseSpecifierArray &BasePath) {
+                           unsigned &msg, CastKind &Kind,
+                           CXXCastPath &BasePath) {
   // C++ 5.2.9p5: An lvalue of type "cv1 B", where B is a class type, can be
   //   cast to type "reference to cv2 D", where D is a class derived from B,
   //   if a valid standard conversion from "pointer to D" to "pointer to B"
@@ -697,8 +700,8 @@ TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr, QualType DestType,
 TryCastResult
 TryStaticPointerDowncast(Sema &Self, QualType SrcType, QualType DestType,
                          bool CStyle, const SourceRange &OpRange,
-                         unsigned &msg, CastExpr::CastKind &Kind,
-                         CXXBaseSpecifierArray &BasePath) {
+                         unsigned &msg, CastKind &Kind,
+                         CXXCastPath &BasePath) {
   // C++ 5.2.9p8: An rvalue of type "pointer to cv1 B", where B is a class
   //   type, can be converted to an rvalue of type "pointer to cv2 D", where D
   //   is a class derived from B, if a valid standard conversion from "pointer
@@ -732,7 +735,7 @@ TryCastResult
 TryStaticDowncast(Sema &Self, CanQualType SrcType, CanQualType DestType,
                   bool CStyle, const SourceRange &OpRange, QualType OrigSrcType,
                   QualType OrigDestType, unsigned &msg, 
-                  CastExpr::CastKind &Kind, CXXBaseSpecifierArray &BasePath) {
+                  CastKind &Kind, CXXCastPath &BasePath) {
   // We can only work with complete types. But don't complain if it doesn't work
   if (Self.RequireCompleteType(OpRange.getBegin(), SrcType, Self.PDiag(0)) ||
       Self.RequireCompleteType(OpRange.getBegin(), DestType, Self.PDiag(0)))
@@ -824,7 +827,7 @@ TryStaticDowncast(Sema &Self, CanQualType SrcType, CanQualType DestType,
   }
 
   Self.BuildBasePathArray(Paths, BasePath);
-  Kind = CastExpr::CK_BaseToDerived;
+  Kind = CK_BaseToDerived;
   return TC_Success;
 }
 
@@ -839,8 +842,8 @@ TryCastResult
 TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr, QualType SrcType, 
                              QualType DestType, bool CStyle, 
                              const SourceRange &OpRange,
-                             unsigned &msg, CastExpr::CastKind &Kind,
-                             CXXBaseSpecifierArray &BasePath) {
+                             unsigned &msg, CastKind &Kind,
+                             CXXCastPath &BasePath) {
   const MemberPointerType *DestMemPtr = DestType->getAs<MemberPointerType>();
   if (!DestMemPtr)
     return TC_NotApplicable;
@@ -900,7 +903,7 @@ TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr, QualType SrcType,
   }
 
   if (!CStyle && Self.CheckBaseClassAccess(OpRange.getBegin(),
-                                           DestType, SrcType,
+                                           DestClass, SrcClass,
                                            Paths.front(),
                                      diag::err_upcast_to_inaccessible_base)) {
     msg = 0;
@@ -927,7 +930,7 @@ TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr, QualType SrcType,
   }
 
   Self.BuildBasePathArray(Paths, BasePath);
-  Kind = CastExpr::CK_DerivedToBaseMemberPointer;
+  Kind = CK_DerivedToBaseMemberPointer;
   return TC_Success;
 }
 
@@ -939,7 +942,7 @@ TryStaticMemberPointerUpcast(Sema &Self, Expr *&SrcExpr, QualType SrcType,
 TryCastResult
 TryStaticImplicitCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
                       bool CStyle, const SourceRange &OpRange, unsigned &msg,
-                      CastExpr::CastKind &Kind) {
+                      CastKind &Kind) {
   if (DestType->isRecordType()) {
     if (Self.RequireCompleteType(OpRange.getBegin(), DestType,
                                  diag::err_bad_dynamic_cast_incomplete)) {
@@ -961,18 +964,17 @@ TryStaticImplicitCast(Sema &Self, Expr *&SrcExpr, QualType DestType,
       (CStyle || !DestType->isReferenceType()))
     return TC_NotApplicable;
     
-  Sema::OwningExprResult Result
-    = InitSeq.Perform(Self, Entity, InitKind,
-                      Action::MultiExprArg(Self, (void**)&SrcExpr, 1));
+  ExprResult Result
+    = InitSeq.Perform(Self, Entity, InitKind, MultiExprArg(Self, &SrcExpr, 1));
   if (Result.isInvalid()) {
     msg = 0;
     return TC_Failed;
   }
   
   if (InitSeq.isConstructorInitialization())
-    Kind = CastExpr::CK_ConstructorConversion;
+    Kind = CK_ConstructorConversion;
   else
-    Kind = CastExpr::CK_NoOp;
+    Kind = CK_NoOp;
   
   SrcExpr = Result.takeAs<Expr>();
   return TC_Success;
@@ -1051,7 +1053,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
                                         QualType DestType, bool CStyle,
                                         const SourceRange &OpRange,
                                         unsigned &msg,
-                                        CastExpr::CastKind &Kind) {
+                                        CastKind &Kind) {
   bool IsLValueCast = false;
   
   DestType = Self.Context.getCanonicalType(DestType);
@@ -1097,8 +1099,15 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
       return TC_Failed;
     }
 
+    // Don't allow casting between member pointers of different sizes.
+    if (Self.Context.getTypeSize(DestMemPtr) !=
+        Self.Context.getTypeSize(SrcMemPtr)) {
+      msg = diag::err_bad_cxx_cast_member_pointer_size;
+      return TC_Failed;
+    }
+
     // A valid member pointer cast.
-    Kind = IsLValueCast? CastExpr::CK_LValueBitCast : CastExpr::CK_BitCast;
+    Kind = IsLValueCast? CK_LValueBitCast : CK_BitCast;
     return TC_Success;
   }
 
@@ -1113,7 +1122,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
       msg = diag::err_bad_reinterpret_cast_small_int;
       return TC_Failed;
     }
-    Kind = CastExpr::CK_PointerToIntegral;
+    Kind = CK_PointerToIntegral;
     return TC_Success;
   }
 
@@ -1132,7 +1141,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     // If both types have the same size, we can successfully cast.
     if (Self.Context.getTypeSize(SrcType)
           == Self.Context.getTypeSize(DestType)) {
-      Kind = CastExpr::CK_BitCast;
+      Kind = CK_BitCast;
       return TC_Success;
     }
     
@@ -1163,7 +1172,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     // to the same type. However, the behavior of compilers is pretty consistent
     // on this point: allow same-type conversion if the involved types are
     // pointers, disallow otherwise.
-    Kind = CastExpr::CK_NoOp;
+    Kind = CK_NoOp;
     return TC_Success;
   }
 
@@ -1176,7 +1185,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
       msg = diag::err_bad_reinterpret_cast_small_int;
       return TC_Failed;
     }
-    Kind = CastExpr::CK_PointerToIntegral;
+    Kind = CK_PointerToIntegral;
     return TC_Success;
   }
 
@@ -1184,7 +1193,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
     assert(destIsPtr && "One type must be a pointer");
     // C++ 5.2.10p5: A value of integral or enumeration type can be explicitly
     //   converted to a pointer.
-    Kind = CastExpr::CK_IntegralToPointer;
+    Kind = CK_IntegralToPointer;
     return TC_Success;
   }
 
@@ -1209,13 +1218,13 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
   // Any pointer can be cast to an Objective-C pointer type with a C-style
   // cast.
   if (CStyle && DestType->isObjCObjectPointerType()) {
-    Kind = CastExpr::CK_AnyPointerToObjCPointerCast;
+    Kind = CK_AnyPointerToObjCPointerCast;
     return TC_Success;
   }
     
   // Not casting away constness, so the only remaining check is for compatible
   // pointer categories.
-  Kind = IsLValueCast? CastExpr::CK_LValueBitCast : CastExpr::CK_BitCast;
+  Kind = IsLValueCast? CK_LValueBitCast : CK_BitCast;
 
   if (SrcType->isFunctionPointerType()) {
     if (DestType->isFunctionPointerType()) {
@@ -1252,14 +1261,14 @@ static TryCastResult TryReinterpretCast(Sema &Self, Expr *SrcExpr,
 
 bool 
 Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, Expr *&CastExpr,
-                         CastExpr::CastKind &Kind, 
-                         CXXBaseSpecifierArray &BasePath,
+                         CastKind &Kind, 
+                         CXXCastPath &BasePath,
                          bool FunctionalStyle) {
   // This test is outside everything else because it's the only case where
   // a non-lvalue-reference target type does not lead to decay.
   // C++ 5.2.9p4: Any expression can be explicitly converted to type "cv void".
   if (CastTy->isVoidType()) {
-    Kind = CastExpr::CK_ToVoid;
+    Kind = CK_ToVoid;
     return false;
   }
 
@@ -1285,7 +1294,7 @@ Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, Expr *&CastExpr,
   TryCastResult tcr = TryConstCast(*this, CastExpr, CastTy, /*CStyle*/true,
                                    msg);
   if (tcr == TC_Success)
-    Kind = CastExpr::CK_NoOp;
+    Kind = CK_NoOp;
 
   if (tcr == TC_NotApplicable) {
     // ... or if that is not possible, a static_cast, ignoring const, ...
@@ -1301,6 +1310,8 @@ Sema::CXXCheckCStyleCast(SourceRange R, QualType CastTy, Expr *&CastExpr,
   if (tcr != TC_Success && msg != 0)
     Diag(R.getBegin(), msg) << (FunctionalStyle ? CT_Functional : CT_CStyle)
       << CastExpr->getType() << CastTy << R;
+  else if (Kind == CK_Unknown || Kind == CK_BitCast)
+    CheckCastAlign(CastExpr, CastTy, R);
 
   return tcr != TC_Success;
 }
diff --git a/lib/Sema/SemaCXXScopeSpec.cpp b/lib/Sema/SemaCXXScopeSpec.cpp
index f56573a8de26..631308eebb04 100644
--- a/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/lib/Sema/SemaCXXScopeSpec.cpp
@@ -11,14 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/Basic/PartialDiagnostic.h"
-#include "clang/Parse/DeclSpec.h"
+#include "clang/Sema/DeclSpec.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
@@ -283,7 +283,7 @@ NamedDecl *Sema::FindFirstQualifierInScope(Scope *S, NestedNameSpecifier *NNS) {
 bool Sema::isNonTypeNestedNameSpecifier(Scope *S, CXXScopeSpec &SS,
                                         SourceLocation IdLoc,
                                         IdentifierInfo &II,
-                                        TypeTy *ObjectTypePtr) {
+                                        ParsedType ObjectTypePtr) {
   QualType ObjectType = GetTypeFromParser(ObjectTypePtr);
   LookupResult Found(*this, &II, IdLoc, LookupNestedNameSpecifierName);
   
@@ -416,7 +416,17 @@ Sema::CXXScopeTy *Sema::BuildCXXNestedNameSpecifier(Scope *S,
 
       ObjectTypeSearchedInScope = true;
     }
-  } else if (isDependent) {
+  } else if (!isDependent) {
+    // Perform unqualified name lookup in the current scope.
+    LookupName(Found, S);
+  }
+
+  // If we performed lookup into a dependent context and did not find anything,
+  // that's fine: just build a dependent nested-name-specifier.
+  if (Found.empty() && isDependent &&
+      !(LookupCtx && LookupCtx->isRecord() &&
+        (!cast<CXXRecordDecl>(LookupCtx)->hasDefinition() ||
+         !cast<CXXRecordDecl>(LookupCtx)->hasAnyDependentBases()))) {
     // Don't speculate if we're just trying to improve error recovery.
     if (ErrorRecoveryLookup)
       return 0;
@@ -429,11 +439,8 @@ Sema::CXXScopeTy *Sema::BuildCXXNestedNameSpecifier(Scope *S,
       return NestedNameSpecifier::Create(Context, &II);
 
     return NestedNameSpecifier::Create(Context, Prefix, &II);
-  } else {
-    // Perform unqualified name lookup in the current scope.
-    LookupName(Found, S);
-  }
-
+  } 
+  
   // FIXME: Deal with ambiguities cleanly.
 
   if (Found.empty() && !ErrorRecoveryLookup) {
@@ -560,10 +567,10 @@ Sema::CXXScopeTy *Sema::ActOnCXXNestedNameSpecifier(Scope *S,
                                                     SourceLocation IdLoc,
                                                     SourceLocation CCLoc,
                                                     IdentifierInfo &II,
-                                                    TypeTy *ObjectTypePtr,
+                                                    ParsedType ObjectTypePtr,
                                                     bool EnteringContext) {
   return BuildCXXNestedNameSpecifier(S, SS, IdLoc, CCLoc, II,
-                                     QualType::getFromOpaquePtr(ObjectTypePtr),
+                                     GetTypeFromParser(ObjectTypePtr),
                                      /*ScopeLookupResult=*/0, EnteringContext,
                                      false);
 }
@@ -575,21 +582,20 @@ Sema::CXXScopeTy *Sema::ActOnCXXNestedNameSpecifier(Scope *S,
 ///
 /// The arguments are the same as those passed to ActOnCXXNestedNameSpecifier.
 bool Sema::IsInvalidUnlessNestedName(Scope *S, CXXScopeSpec &SS,
-                                     IdentifierInfo &II, TypeTy *ObjectType,
+                                     IdentifierInfo &II, ParsedType ObjectType,
                                      bool EnteringContext) {
   return BuildCXXNestedNameSpecifier(S, SS, SourceLocation(), SourceLocation(),
-                                     II, QualType::getFromOpaquePtr(ObjectType),
+                                     II, GetTypeFromParser(ObjectType),
                                      /*ScopeLookupResult=*/0, EnteringContext,
                                      true);
 }
 
 Sema::CXXScopeTy *Sema::ActOnCXXNestedNameSpecifier(Scope *S,
                                                     const CXXScopeSpec &SS,
-                                                    TypeTy *Ty,
+                                                    ParsedType Ty,
                                                     SourceRange TypeRange,
                                                     SourceLocation CCLoc) {
-  NestedNameSpecifier *Prefix
-    = static_cast<NestedNameSpecifier *>(SS.getScopeRep());
+  NestedNameSpecifier *Prefix = SS.getScopeRep();
   QualType T = GetTypeFromParser(Ty);
   return NestedNameSpecifier::Create(Context, Prefix, /*FIXME:*/false,
                                      T.getTypePtr());
@@ -620,7 +626,7 @@ bool Sema::ShouldEnterDeclaratorScope(Scope *S, const CXXScopeSpec &SS) {
   case NestedNameSpecifier::Namespace:
     // These are always namespace scopes.  We never want to enter a
     // namespace scope from anything but a file context.
-    return CurContext->getLookupContext()->isFileContext();
+    return CurContext->getRedeclContext()->isFileContext();
 
   case NestedNameSpecifier::Identifier:
   case NestedNameSpecifier::TypeSpec:
diff --git a/lib/Sema/SemaChecking.cpp b/lib/Sema/SemaChecking.cpp
index 7a39f058c5ec..a0b4b988dbeb 100644
--- a/lib/Sema/SemaChecking.cpp
+++ b/lib/Sema/SemaChecking.cpp
@@ -12,10 +12,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "clang/Analysis/Analyses/PrintfFormatString.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Analysis/Analyses/FormatString.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CharUnits.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
@@ -26,12 +29,12 @@
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "clang/Basic/TargetInfo.h"
 #include <limits>
 using namespace clang;
+using namespace sema;
 
 /// getLocationOfStringLiteralByte - Return a source location that points to the
 /// specified byte of the specified string literal.
@@ -122,9 +125,9 @@ bool Sema::CheckablePrintfAttr(const FormatAttr *Format, CallExpr *TheCall) {
   return false;
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
-  OwningExprResult TheCallResult(Owned(TheCall));
+  ExprResult TheCallResult(Owned(TheCall));
 
   switch (BuiltinID) {
   case Builtin::BI__builtin___CFStringMakeConstantString:
@@ -298,6 +301,10 @@ bool Sema::CheckARMBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   unsigned i = 0, l = 0, u = 0;
   switch (BuiltinID) {
   default: return false;
+  case ARM::BI__builtin_arm_ssat: i = 1; l = 1; u = 31; break;
+  case ARM::BI__builtin_arm_usat: i = 1; u = 31; break;
+  case ARM::BI__builtin_arm_vcvtr_f:
+  case ARM::BI__builtin_arm_vcvtr_d: i = 1; u = 1; break;
 #define GET_NEON_IMMEDIATE_CHECK
 #include "clang/Basic/arm_neon.inc"
 #undef GET_NEON_IMMEDIATE_CHECK
@@ -311,9 +318,9 @@ bool Sema::CheckARMBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   unsigned Val = Result.getZExtValue();
   if (Val < l || Val > (u + l))
     return Diag(TheCall->getLocStart(), diag::err_argument_invalid_range)
-      << llvm::utostr(l) << llvm::utostr(u+l)  
-      << TheCall->getArg(i)->getSourceRange();
+      << l << u+l << TheCall->getArg(i)->getSourceRange();
 
+  // FIXME: VFP Intrinsics should error if VFP not present.
   return false;
 }
 
@@ -334,16 +341,22 @@ bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall) {
 
   // Printf checking.
   if (const FormatAttr *Format = FDecl->getAttr<FormatAttr>()) {
-    if (CheckablePrintfAttr(Format, TheCall)) {
+    const bool b = Format->getType() == "scanf";
+    if (b || CheckablePrintfAttr(Format, TheCall)) {
       bool HasVAListArg = Format->getFirstArg() == 0;
-      CheckPrintfArguments(TheCall, HasVAListArg, Format->getFormatIdx() - 1,
-                           HasVAListArg ? 0 : Format->getFirstArg() - 1);
+      CheckPrintfScanfArguments(TheCall, HasVAListArg,
+                                Format->getFormatIdx() - 1,
+                                HasVAListArg ? 0 : Format->getFirstArg() - 1,
+                                !b);
     }
   }
 
-  for (const NonNullAttr *NonNull = FDecl->getAttr<NonNullAttr>(); NonNull;
-       NonNull = NonNull->getNext<NonNullAttr>())
-    CheckNonNullArguments(NonNull, TheCall);
+  specific_attr_iterator<NonNullAttr>
+    i = FDecl->specific_attr_begin<NonNullAttr>(),
+    e = FDecl->specific_attr_end<NonNullAttr>();
+
+  for (; i != e; ++i)
+    CheckNonNullArguments(*i, TheCall);
 
   return false;
 }
@@ -362,12 +375,13 @@ bool Sema::CheckBlockCall(NamedDecl *NDecl, CallExpr *TheCall) {
   if (!Ty->isBlockPointerType())
     return false;
 
-  if (!CheckablePrintfAttr(Format, TheCall))
+  const bool b = Format->getType() == "scanf";
+  if (!b && !CheckablePrintfAttr(Format, TheCall))
     return false;
 
   bool HasVAListArg = Format->getFirstArg() == 0;
-  CheckPrintfArguments(TheCall, HasVAListArg, Format->getFormatIdx() - 1,
-                       HasVAListArg ? 0 : Format->getFirstArg() - 1);
+  CheckPrintfScanfArguments(TheCall, HasVAListArg, Format->getFormatIdx() - 1,
+                            HasVAListArg ? 0 : Format->getFirstArg() - 1, !b);
 
   return false;
 }
@@ -380,8 +394,8 @@ bool Sema::CheckBlockCall(NamedDecl *NDecl, CallExpr *TheCall) {
 ///
 /// This function goes through and does final semantic checking for these
 /// builtins,
-Sema::OwningExprResult
-Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
+ExprResult
+Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   CallExpr *TheCall = (CallExpr *)TheCallResult.get();
   DeclRefExpr *DRE =cast<DeclRefExpr>(TheCall->getCallee()->IgnoreParenCasts());
   FunctionDecl *FDecl = cast<FunctionDecl>(DRE->getDecl());
@@ -415,6 +429,10 @@ Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
     return ExprError();
   }
 
+  // The majority of builtins return a value, but a few have special return
+  // types, so allow them to override appropriately below.
+  QualType ResultType = ValType;
+
   // We need to figure out which concrete builtin this maps onto.  For example,
   // __sync_fetch_and_add with a 2 byte object turns into
   // __sync_fetch_and_add_2.
@@ -483,11 +501,13 @@ Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
   case Builtin::BI__sync_bool_compare_and_swap:
     BuiltinIndex = 11;
     NumFixed = 2;
+    ResultType = Context.BoolTy;
     break;
   case Builtin::BI__sync_lock_test_and_set: BuiltinIndex = 12; break;
   case Builtin::BI__sync_lock_release:
     BuiltinIndex = 13;
     NumFixed = 0;
+    ResultType = Context.VoidTy;
     break;
   }
 
@@ -508,19 +528,10 @@ Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
   FunctionDecl *NewBuiltinDecl =
     cast<FunctionDecl>(LazilyCreateBuiltin(NewBuiltinII, NewBuiltinID,
                                            TUScope, false, DRE->getLocStart()));
-  const FunctionProtoType *BuiltinFT =
-    NewBuiltinDecl->getType()->getAs<FunctionProtoType>();
-
-  QualType OrigValType = ValType;
-  ValType = BuiltinFT->getArgType(0)->getAs<PointerType>()->getPointeeType();
-
-  // If the first type needs to be converted (e.g. void** -> int*), do it now.
-  if (BuiltinFT->getArgType(0) != FirstArg->getType()) {
-    ImpCastExprToType(FirstArg, BuiltinFT->getArgType(0), CastExpr::CK_BitCast);
-    TheCall->setArg(0, FirstArg);
-  }
 
-  // Next, walk the valid ones promoting to the right type.
+  // The first argument --- the pointer --- has a fixed type; we
+  // deduce the types of the rest of the arguments accordingly.  Walk
+  // the remaining arguments, converting them to the deduced value type.
   for (unsigned i = 0; i != NumFixed; ++i) {
     Expr *Arg = TheCall->getArg(i+1);
 
@@ -529,14 +540,13 @@ Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
     if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Arg)) {
       Arg = ICE->getSubExpr();
       ICE->setSubExpr(0);
-      ICE->Destroy(Context);
       TheCall->setArg(i+1, Arg);
     }
 
     // GCC does an implicit conversion to the pointer or integer ValType.  This
     // can fail in some cases (1i -> int**), check for this error case now.
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-    CXXBaseSpecifierArray BasePath;
+    CastKind Kind = CK_Unknown;
+    CXXCastPath BasePath;
     if (CheckCastTypes(Arg->getSourceRange(), ValType, Arg, Kind, BasePath))
       return ExprError();
 
@@ -546,7 +556,7 @@ Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
     // pass in 42.  The 42 gets converted to char.  This is even more strange
     // for things like 45.123 -> char, etc.
     // FIXME: Do this check.
-    ImpCastExprToType(Arg, ValType, Kind);
+    ImpCastExprToType(Arg, ValType, Kind, VK_RValue, &BasePath);
     TheCall->setArg(i+1, Arg);
   }
 
@@ -560,28 +570,10 @@ Sema::SemaBuiltinAtomicOverloaded(OwningExprResult TheCallResult) {
   UsualUnaryConversions(PromotedCall);
   TheCall->setCallee(PromotedCall);
 
-  // Change the result type of the call to match the result type of the decl.
-  TheCall->setType(NewBuiltinDecl->getCallResultType());
-
-  // If the value type was converted to an integer when processing the
-  // arguments (e.g. void* -> int), we need to convert the result back.
-  if (!Context.hasSameUnqualifiedType(ValType, OrigValType)) {
-    Expr *E = TheCallResult.takeAs<Expr>();
-
-    assert(ValType->isIntegerType() &&
-           "We always convert atomic operation values to integers.");
-    // FIXME: Handle floating point value type here too.
-    CastExpr::CastKind Kind;
-    if (OrigValType->isIntegerType())
-      Kind = CastExpr::CK_IntegralCast;
-    else if (OrigValType->hasPointerRepresentation())
-      Kind = CastExpr::CK_IntegralToPointer;
-    else
-      llvm_unreachable("Unhandled original value type!");
-
-    ImpCastExprToType(E, OrigValType, Kind);
-    return Owned(E);
-  }
+  // Change the result type of the call to match the original value type. This
+  // is arbitrary, but the codegen for these builtins ins design to handle it
+  // gracefully.
+  TheCall->setType(ResultType);
 
   return move(TheCallResult);
 }
@@ -604,16 +596,11 @@ bool Sema::CheckObjCString(Expr *Arg) {
     return true;
   }
 
-  const char *Data = Literal->getStrData();
-  unsigned Length = Literal->getByteLength();
-
-  for (unsigned i = 0; i < Length; ++i) {
-    if (!Data[i]) {
-      Diag(getLocationOfStringLiteralByte(Literal, i),
-           diag::warn_cfstring_literal_contains_nul_character)
-        << Arg->getSourceRange();
-      break;
-    }
+  size_t NulPos = Literal->getString().find('\0');
+  if (NulPos != llvm::StringRef::npos) {
+    Diag(getLocationOfStringLiteralByte(Literal, NulPos),
+         diag::warn_cfstring_literal_contains_nul_character)
+      << Arg->getSourceRange();
   }
 
   return false;
@@ -753,7 +740,6 @@ bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs) {
       assert(Cast->getType()->isSpecificBuiltinType(BuiltinType::Double) &&
              "promotion from float to double is the only expected cast here");
       Cast->setSubExpr(0);
-      Cast->Destroy(Context);
       TheCall->setArg(NumArgs-1, CastArg);
       OrigArg = CastArg;
     }
@@ -764,7 +750,7 @@ bool Sema::SemaBuiltinFPClassification(CallExpr *TheCall, unsigned NumArgs) {
 
 /// SemaBuiltinShuffleVector - Handle __builtin_shufflevector.
 // This is declared to take (...), so we have to check everything.
-Action::OwningExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
+ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
   if (TheCall->getNumArgs() < 2)
     return ExprError(Diag(TheCall->getLocEnd(),
                           diag::err_typecheck_call_too_few_args_at_least)
@@ -797,7 +783,7 @@ Action::OwningExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
     // with mask.  If so, verify that RHS is an integer vector type with the
     // same number of elts as lhs.
     if (TheCall->getNumArgs() == 2) {
-      if (!RHSType->isIntegerType() || 
+      if (!RHSType->hasIntegerRepresentation() || 
           RHSType->getAs<VectorType>()->getNumElements() != numElements)
         Diag(TheCall->getLocStart(), diag::err_shufflevector_incompatible_vector)
           << SourceRange(TheCall->getArg(1)->getLocStart(),
@@ -941,29 +927,31 @@ bool Sema::SemaBuiltinLongjmp(CallExpr *TheCall) {
 // Handle i > 1 ? "x" : "y", recursivelly
 bool Sema::SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
                                   bool HasVAListArg,
-                                  unsigned format_idx, unsigned firstDataArg) {
+                                  unsigned format_idx, unsigned firstDataArg,
+                                  bool isPrintf) {
+
   if (E->isTypeDependent() || E->isValueDependent())
     return false;
 
   switch (E->getStmtClass()) {
   case Stmt::ConditionalOperatorClass: {
     const ConditionalOperator *C = cast<ConditionalOperator>(E);
-    return SemaCheckStringLiteral(C->getTrueExpr(), TheCall,
-                                  HasVAListArg, format_idx, firstDataArg)
-        && SemaCheckStringLiteral(C->getRHS(), TheCall,
-                                  HasVAListArg, format_idx, firstDataArg);
+    return SemaCheckStringLiteral(C->getTrueExpr(), TheCall, HasVAListArg,
+                                  format_idx, firstDataArg, isPrintf)
+        && SemaCheckStringLiteral(C->getRHS(), TheCall, HasVAListArg,
+                                  format_idx, firstDataArg, isPrintf);
   }
 
   case Stmt::ImplicitCastExprClass: {
     const ImplicitCastExpr *Expr = cast<ImplicitCastExpr>(E);
     return SemaCheckStringLiteral(Expr->getSubExpr(), TheCall, HasVAListArg,
-                                  format_idx, firstDataArg);
+                                  format_idx, firstDataArg, isPrintf);
   }
 
   case Stmt::ParenExprClass: {
     const ParenExpr *Expr = cast<ParenExpr>(E);
     return SemaCheckStringLiteral(Expr->getSubExpr(), TheCall, HasVAListArg,
-                                  format_idx, firstDataArg);
+                                  format_idx, firstDataArg, isPrintf);
   }
 
   case Stmt::DeclRefExprClass: {
@@ -985,7 +973,8 @@ bool Sema::SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
       if (isConstant) {
         if (const Expr *Init = VD->getAnyInitializer())
           return SemaCheckStringLiteral(Init, TheCall,
-                                        HasVAListArg, format_idx, firstDataArg);
+                                        HasVAListArg, format_idx, firstDataArg,
+                                        isPrintf);
       }
 
       // For vprintf* functions (i.e., HasVAListArg==true), we add a
@@ -1025,7 +1014,7 @@ bool Sema::SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
             const Expr *Arg = CE->getArg(ArgIndex - 1);
 
             return SemaCheckStringLiteral(Arg, TheCall, HasVAListArg,
-                                          format_idx, firstDataArg);
+                                          format_idx, firstDataArg, isPrintf);
           }
         }
       }
@@ -1043,8 +1032,8 @@ bool Sema::SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
       StrE = cast<StringLiteral>(E);
 
     if (StrE) {
-      CheckPrintfString(StrE, E, TheCall, HasVAListArg, format_idx,
-                        firstDataArg);
+      CheckFormatString(StrE, E, TheCall, HasVAListArg, format_idx,
+                        firstDataArg, isPrintf);
       return true;
     }
 
@@ -1059,7 +1048,8 @@ bool Sema::SemaCheckStringLiteral(const Expr *E, const CallExpr *TheCall,
 void
 Sema::CheckNonNullArguments(const NonNullAttr *NonNull,
                             const CallExpr *TheCall) {
-  for (NonNullAttr::iterator i = NonNull->begin(), e = NonNull->end();
+  for (NonNullAttr::args_iterator i = NonNull->args_begin(),
+                                  e = NonNull->args_end();
        i != e; ++i) {
     const Expr *ArgExpr = TheCall->getArg(*i);
     if (ArgExpr->isNullPointerConstant(Context,
@@ -1069,55 +1059,13 @@ Sema::CheckNonNullArguments(const NonNullAttr *NonNull,
   }
 }
 
-/// CheckPrintfArguments - Check calls to printf (and similar functions) for
-/// correct use of format strings.
-///
-///  HasVAListArg - A predicate indicating whether the printf-like
-///    function is passed an explicit va_arg argument (e.g., vprintf)
-///
-///  format_idx - The index into Args for the format string.
-///
-/// Improper format strings to functions in the printf family can be
-/// the source of bizarre bugs and very serious security holes.  A
-/// good source of information is available in the following paper
-/// (which includes additional references):
-///
-///  FormatGuard: Automatic Protection From printf Format String
-///  Vulnerabilities, Proceedings of the 10th USENIX Security Symposium, 2001.
-///
-/// TODO:
-/// Functionality implemented:
-///
-///  We can statically check the following properties for string
-///  literal format strings for non v.*printf functions (where the
-///  arguments are passed directly):
-//
-///  (1) Are the number of format conversions equal to the number of
-///      data arguments?
-///
-///  (2) Does each format conversion correctly match the type of the
-///      corresponding data argument?
-///
-/// Moreover, for all printf functions we can:
-///
-///  (3) Check for a missing format string (when not caught by type checking).
-///
-///  (4) Check for no-operation flags; e.g. using "#" with format
-///      conversion 'c'  (TODO)
-///
-///  (5) Check the use of '%n', a major source of security holes.
-///
-///  (6) Check for malformed format conversions that don't specify anything.
-///
-///  (7) Check for empty format strings.  e.g: printf("");
-///
-///  (8) Check that the format string is a wide literal.
-///
-/// All of these checks can be done by parsing the format string.
-///
+/// CheckPrintfScanfArguments - Check calls to printf and scanf (and similar
+/// functions) for correct use of format strings.
 void
-Sema::CheckPrintfArguments(const CallExpr *TheCall, bool HasVAListArg,
-                           unsigned format_idx, unsigned firstDataArg) {
+Sema::CheckPrintfScanfArguments(const CallExpr *TheCall, bool HasVAListArg,
+                                unsigned format_idx, unsigned firstDataArg,
+                                bool isPrintf) {
+
   const Expr *Fn = TheCall->getCallee();
 
   // The way the format attribute works in GCC, the implicit this argument
@@ -1132,9 +1080,9 @@ Sema::CheckPrintfArguments(const CallExpr *TheCall, bool HasVAListArg,
       --firstDataArg;
   }
 
-  // CHECK: printf-like function is called with no format string.
+  // CHECK: printf/scanf-like function is called with no format string.
   if (format_idx >= TheCall->getNumArgs()) {
-    Diag(TheCall->getRParenLoc(), diag::warn_printf_missing_format_string)
+    Diag(TheCall->getRParenLoc(), diag::warn_missing_format_string)
       << Fn->getSourceRange();
     return;
   }
@@ -1154,23 +1102,24 @@ Sema::CheckPrintfArguments(const CallExpr *TheCall, bool HasVAListArg,
   // ObjC string uses the same format specifiers as C string, so we can use
   // the same format string checking logic for both ObjC and C strings.
   if (SemaCheckStringLiteral(OrigFormatExpr, TheCall, HasVAListArg, format_idx,
-                             firstDataArg))
+                             firstDataArg, isPrintf))
     return;  // Literal format string found, check done!
 
   // If there are no arguments specified, warn with -Wformat-security, otherwise
   // warn only with -Wformat-nonliteral.
   if (TheCall->getNumArgs() == format_idx+1)
     Diag(TheCall->getArg(format_idx)->getLocStart(),
-         diag::warn_printf_nonliteral_noargs)
+         diag::warn_format_nonliteral_noargs)
       << OrigFormatExpr->getSourceRange();
   else
     Diag(TheCall->getArg(format_idx)->getLocStart(),
-         diag::warn_printf_nonliteral)
+         diag::warn_format_nonliteral)
            << OrigFormatExpr->getSourceRange();
 }
 
 namespace {
-class CheckPrintfHandler : public analyze_printf::FormatStringHandler {
+class CheckFormatHandler : public analyze_format_string::FormatStringHandler {
+protected:
   Sema &S;
   const StringLiteral *FExpr;
   const Expr *OrigFormatExpr;
@@ -1185,7 +1134,7 @@ class CheckPrintfHandler : public analyze_printf::FormatStringHandler {
   bool usesPositionalArgs;
   bool atFirstArg;
 public:
-  CheckPrintfHandler(Sema &s, const StringLiteral *fexpr,
+  CheckFormatHandler(Sema &s, const StringLiteral *fexpr,
                      const Expr *origFormatExpr, unsigned firstDataArg,
                      unsigned numDataArgs, bool isObjCLiteral,
                      const char *beg, bool hasVAListArg,
@@ -1203,55 +1152,43 @@ public:
 
   void DoneProcessing();
 
-  void HandleIncompleteFormatSpecifier(const char *startSpecifier,
-                                       unsigned specifierLen);
-
-  bool
-  HandleInvalidConversionSpecifier(const analyze_printf::FormatSpecifier &FS,
-                                   const char *startSpecifier,
-                                   unsigned specifierLen);
-
+  void HandleIncompleteSpecifier(const char *startSpecifier,
+                                 unsigned specifierLen);
+    
   virtual void HandleInvalidPosition(const char *startSpecifier,
                                      unsigned specifierLen,
-                                     analyze_printf::PositionContext p);
+                                     analyze_format_string::PositionContext p);
 
   virtual void HandleZeroPosition(const char *startPos, unsigned posLen);
 
   void HandleNullChar(const char *nullCharacter);
 
-  bool HandleFormatSpecifier(const analyze_printf::FormatSpecifier &FS,
-                             const char *startSpecifier,
-                             unsigned specifierLen);
-private:
+protected:
+  bool HandleInvalidConversionSpecifier(unsigned argIndex, SourceLocation Loc,
+                                        const char *startSpec,
+                                        unsigned specifierLen,
+                                        const char *csStart, unsigned csLen);
+  
   SourceRange getFormatStringRange();
-  CharSourceRange getFormatSpecifierRange(const char *startSpecifier,
-                                          unsigned specifierLen);
+  CharSourceRange getSpecifierRange(const char *startSpecifier,
+                                    unsigned specifierLen);
   SourceLocation getLocationOfByte(const char *x);
 
-  bool HandleAmount(const analyze_printf::OptionalAmount &Amt, unsigned k,
-                    const char *startSpecifier, unsigned specifierLen);
-  void HandleInvalidAmount(const analyze_printf::FormatSpecifier &FS,
-                           const analyze_printf::OptionalAmount &Amt,
-                           unsigned type,
-                           const char *startSpecifier, unsigned specifierLen);
-  void HandleFlag(const analyze_printf::FormatSpecifier &FS,
-                  const analyze_printf::OptionalFlag &flag,
-                  const char *startSpecifier, unsigned specifierLen);
-  void HandleIgnoredFlag(const analyze_printf::FormatSpecifier &FS,
-                         const analyze_printf::OptionalFlag &ignoredFlag,
-                         const analyze_printf::OptionalFlag &flag,
-                         const char *startSpecifier, unsigned specifierLen);
-
   const Expr *getDataArg(unsigned i) const;
+  
+  bool CheckNumArgs(const analyze_format_string::FormatSpecifier &FS,
+                    const analyze_format_string::ConversionSpecifier &CS,
+                    const char *startSpecifier, unsigned specifierLen,
+                    unsigned argIndex);
 };
 }
 
-SourceRange CheckPrintfHandler::getFormatStringRange() {
+SourceRange CheckFormatHandler::getFormatStringRange() {
   return OrigFormatExpr->getSourceRange();
 }
 
-CharSourceRange CheckPrintfHandler::
-getFormatSpecifierRange(const char *startSpecifier, unsigned specifierLen) {
+CharSourceRange CheckFormatHandler::
+getSpecifierRange(const char *startSpecifier, unsigned specifierLen) {
   SourceLocation Start = getLocationOfByte(startSpecifier);
   SourceLocation End   = getLocationOfByte(startSpecifier + specifierLen - 1);
 
@@ -1261,39 +1198,67 @@ getFormatSpecifierRange(const char *startSpecifier, unsigned specifierLen) {
   return CharSourceRange::getCharRange(Start, End);
 }
 
-SourceLocation CheckPrintfHandler::getLocationOfByte(const char *x) {
+SourceLocation CheckFormatHandler::getLocationOfByte(const char *x) {
   return S.getLocationOfStringLiteralByte(FExpr, x - Beg);
 }
 
-void CheckPrintfHandler::
-HandleIncompleteFormatSpecifier(const char *startSpecifier,
-                                unsigned specifierLen) {
+void CheckFormatHandler::HandleIncompleteSpecifier(const char *startSpecifier,
+                                                   unsigned specifierLen){
   SourceLocation Loc = getLocationOfByte(startSpecifier);
   S.Diag(Loc, diag::warn_printf_incomplete_specifier)
-    << getFormatSpecifierRange(startSpecifier, specifierLen);
+    << getSpecifierRange(startSpecifier, specifierLen);
 }
 
 void
-CheckPrintfHandler::HandleInvalidPosition(const char *startPos, unsigned posLen,
-                                          analyze_printf::PositionContext p) {
+CheckFormatHandler::HandleInvalidPosition(const char *startPos, unsigned posLen,
+                                     analyze_format_string::PositionContext p) {
   SourceLocation Loc = getLocationOfByte(startPos);
-  S.Diag(Loc, diag::warn_printf_invalid_positional_specifier)
-    << (unsigned) p << getFormatSpecifierRange(startPos, posLen);
+  S.Diag(Loc, diag::warn_format_invalid_positional_specifier)
+    << (unsigned) p << getSpecifierRange(startPos, posLen);
 }
 
-void CheckPrintfHandler::HandleZeroPosition(const char *startPos,
+void CheckFormatHandler::HandleZeroPosition(const char *startPos,
                                             unsigned posLen) {
   SourceLocation Loc = getLocationOfByte(startPos);
-  S.Diag(Loc, diag::warn_printf_zero_positional_specifier)
-    << getFormatSpecifierRange(startPos, posLen);
+  S.Diag(Loc, diag::warn_format_zero_positional_specifier)
+    << getSpecifierRange(startPos, posLen);
+}
+
+void CheckFormatHandler::HandleNullChar(const char *nullCharacter) {
+  // The presence of a null character is likely an error.
+  S.Diag(getLocationOfByte(nullCharacter),
+         diag::warn_printf_format_string_contains_null_char)
+    << getFormatStringRange();
 }
 
-bool CheckPrintfHandler::
-HandleInvalidConversionSpecifier(const analyze_printf::FormatSpecifier &FS,
-                                 const char *startSpecifier,
-                                 unsigned specifierLen) {
+const Expr *CheckFormatHandler::getDataArg(unsigned i) const {
+  return TheCall->getArg(FirstDataArg + i);
+}
 
-  unsigned argIndex = FS.getArgIndex();
+void CheckFormatHandler::DoneProcessing() {
+    // Does the number of data arguments exceed the number of
+    // format conversions in the format string?
+  if (!HasVAListArg) {
+      // Find any arguments that weren't covered.
+    CoveredArgs.flip();
+    signed notCoveredArg = CoveredArgs.find_first();
+    if (notCoveredArg >= 0) {
+      assert((unsigned)notCoveredArg < NumDataArgs);
+      S.Diag(getDataArg((unsigned) notCoveredArg)->getLocStart(),
+             diag::warn_printf_data_arg_not_used)
+      << getFormatStringRange();
+    }
+  }
+}
+
+bool
+CheckFormatHandler::HandleInvalidConversionSpecifier(unsigned argIndex,
+                                                     SourceLocation Loc,
+                                                     const char *startSpec,
+                                                     unsigned specifierLen,
+                                                     const char *csStart,
+                                                     unsigned csLen) {
+  
   bool keepGoing = true;
   if (argIndex < NumDataArgs) {
     // Consider the argument coverered, even though the specifier doesn't
@@ -1308,32 +1273,95 @@ HandleInvalidConversionSpecifier(const analyze_printf::FormatSpecifier &FS,
     // gibberish when trying to match arguments.
     keepGoing = false;
   }
+  
+  S.Diag(Loc, diag::warn_format_invalid_conversion)
+    << llvm::StringRef(csStart, csLen)
+    << getSpecifierRange(startSpec, specifierLen);
+  
+  return keepGoing;
+}
 
-  const analyze_printf::ConversionSpecifier &CS =
-    FS.getConversionSpecifier();
-  SourceLocation Loc = getLocationOfByte(CS.getStart());
-  S.Diag(Loc, diag::warn_printf_invalid_conversion)
-      << llvm::StringRef(CS.getStart(), CS.getLength())
-      << getFormatSpecifierRange(startSpecifier, specifierLen);
+bool
+CheckFormatHandler::CheckNumArgs(
+  const analyze_format_string::FormatSpecifier &FS,
+  const analyze_format_string::ConversionSpecifier &CS,
+  const char *startSpecifier, unsigned specifierLen, unsigned argIndex) {
 
-  return keepGoing;
+  if (argIndex >= NumDataArgs) {
+    if (FS.usesPositionalArg())  {
+      S.Diag(getLocationOfByte(CS.getStart()),
+             diag::warn_printf_positional_arg_exceeds_data_args)
+      << (argIndex+1) << NumDataArgs
+      << getSpecifierRange(startSpecifier, specifierLen);
+    }
+    else {
+      S.Diag(getLocationOfByte(CS.getStart()),
+             diag::warn_printf_insufficient_data_args)
+      << getSpecifierRange(startSpecifier, specifierLen);
+    }
+    
+    return false;
+  }
+  return true;
 }
 
-void CheckPrintfHandler::HandleNullChar(const char *nullCharacter) {
-  // The presence of a null character is likely an error.
-  S.Diag(getLocationOfByte(nullCharacter),
-         diag::warn_printf_format_string_contains_null_char)
-    << getFormatStringRange();
+//===--- CHECK: Printf format string checking ------------------------------===//
+
+namespace {
+class CheckPrintfHandler : public CheckFormatHandler {
+public:
+  CheckPrintfHandler(Sema &s, const StringLiteral *fexpr,
+                     const Expr *origFormatExpr, unsigned firstDataArg,
+                     unsigned numDataArgs, bool isObjCLiteral,
+                     const char *beg, bool hasVAListArg,
+                     const CallExpr *theCall, unsigned formatIdx)
+  : CheckFormatHandler(s, fexpr, origFormatExpr, firstDataArg,
+                       numDataArgs, isObjCLiteral, beg, hasVAListArg,
+                       theCall, formatIdx) {}
+  
+  
+  bool HandleInvalidPrintfConversionSpecifier(
+                                      const analyze_printf::PrintfSpecifier &FS,
+                                      const char *startSpecifier,
+                                      unsigned specifierLen);
+  
+  bool HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier &FS,
+                             const char *startSpecifier,
+                             unsigned specifierLen);
+  
+  bool HandleAmount(const analyze_format_string::OptionalAmount &Amt, unsigned k,
+                    const char *startSpecifier, unsigned specifierLen);
+  void HandleInvalidAmount(const analyze_printf::PrintfSpecifier &FS,
+                           const analyze_printf::OptionalAmount &Amt,
+                           unsigned type,
+                           const char *startSpecifier, unsigned specifierLen);
+  void HandleFlag(const analyze_printf::PrintfSpecifier &FS,
+                  const analyze_printf::OptionalFlag &flag,
+                  const char *startSpecifier, unsigned specifierLen);
+  void HandleIgnoredFlag(const analyze_printf::PrintfSpecifier &FS,
+                         const analyze_printf::OptionalFlag &ignoredFlag,
+                         const analyze_printf::OptionalFlag &flag,
+                         const char *startSpecifier, unsigned specifierLen);
+};  
 }
 
-const Expr *CheckPrintfHandler::getDataArg(unsigned i) const {
-  return TheCall->getArg(FirstDataArg + i);
+bool CheckPrintfHandler::HandleInvalidPrintfConversionSpecifier(
+                                      const analyze_printf::PrintfSpecifier &FS,
+                                      const char *startSpecifier,
+                                      unsigned specifierLen) {
+  const analyze_printf::PrintfConversionSpecifier &CS =
+    FS.getConversionSpecifier();
+  
+  return HandleInvalidConversionSpecifier(FS.getArgIndex(),
+                                          getLocationOfByte(CS.getStart()),
+                                          startSpecifier, specifierLen,
+                                          CS.getStart(), CS.getLength());
 }
 
-bool
-CheckPrintfHandler::HandleAmount(const analyze_printf::OptionalAmount &Amt,
-                                 unsigned k, const char *startSpecifier,
-                                 unsigned specifierLen) {
+bool CheckPrintfHandler::HandleAmount(
+                               const analyze_format_string::OptionalAmount &Amt,
+                               unsigned k, const char *startSpecifier,
+                               unsigned specifierLen) {
 
   if (Amt.hasDataArgument()) {
     if (!HasVAListArg) {
@@ -1341,7 +1369,7 @@ CheckPrintfHandler::HandleAmount(const analyze_printf::OptionalAmount &Amt,
       if (argIndex >= NumDataArgs) {
         S.Diag(getLocationOfByte(Amt.getStart()),
                diag::warn_printf_asterisk_missing_arg)
-          << k << getFormatSpecifierRange(startSpecifier, specifierLen);
+          << k << getSpecifierRange(startSpecifier, specifierLen);
         // Don't do any more checking.  We will just emit
         // spurious errors.
         return false;
@@ -1363,7 +1391,7 @@ CheckPrintfHandler::HandleAmount(const analyze_printf::OptionalAmount &Amt,
                diag::warn_printf_asterisk_wrong_type)
           << k
           << ATR.getRepresentativeType(S.Context) << T
-          << getFormatSpecifierRange(startSpecifier, specifierLen)
+          << getSpecifierRange(startSpecifier, specifierLen)
           << Arg->getSourceRange();
         // Don't do any more checking.  We will just emit
         // spurious errors.
@@ -1375,20 +1403,21 @@ CheckPrintfHandler::HandleAmount(const analyze_printf::OptionalAmount &Amt,
 }
 
 void CheckPrintfHandler::HandleInvalidAmount(
-                                      const analyze_printf::FormatSpecifier &FS,
+                                      const analyze_printf::PrintfSpecifier &FS,
                                       const analyze_printf::OptionalAmount &Amt,
                                       unsigned type,
                                       const char *startSpecifier,
                                       unsigned specifierLen) {
-  const analyze_printf::ConversionSpecifier &CS = FS.getConversionSpecifier();
+  const analyze_printf::PrintfConversionSpecifier &CS =
+    FS.getConversionSpecifier();
   switch (Amt.getHowSpecified()) {
   case analyze_printf::OptionalAmount::Constant:
     S.Diag(getLocationOfByte(Amt.getStart()),
         diag::warn_printf_nonsensical_optional_amount)
       << type
       << CS.toString()
-      << getFormatSpecifierRange(startSpecifier, specifierLen)
-      << FixItHint::CreateRemoval(getFormatSpecifierRange(Amt.getStart(),
+      << getSpecifierRange(startSpecifier, specifierLen)
+      << FixItHint::CreateRemoval(getSpecifierRange(Amt.getStart(),
           Amt.getConstantLength()));
     break;
 
@@ -1397,26 +1426,27 @@ void CheckPrintfHandler::HandleInvalidAmount(
         diag::warn_printf_nonsensical_optional_amount)
       << type
       << CS.toString()
-      << getFormatSpecifierRange(startSpecifier, specifierLen);
+      << getSpecifierRange(startSpecifier, specifierLen);
     break;
   }
 }
 
-void CheckPrintfHandler::HandleFlag(const analyze_printf::FormatSpecifier &FS,
+void CheckPrintfHandler::HandleFlag(const analyze_printf::PrintfSpecifier &FS,
                                     const analyze_printf::OptionalFlag &flag,
                                     const char *startSpecifier,
                                     unsigned specifierLen) {
   // Warn about pointless flag with a fixit removal.
-  const analyze_printf::ConversionSpecifier &CS = FS.getConversionSpecifier();
+  const analyze_printf::PrintfConversionSpecifier &CS =
+    FS.getConversionSpecifier();
   S.Diag(getLocationOfByte(flag.getPosition()),
       diag::warn_printf_nonsensical_flag)
     << flag.toString() << CS.toString()
-    << getFormatSpecifierRange(startSpecifier, specifierLen)
-    << FixItHint::CreateRemoval(getFormatSpecifierRange(flag.getPosition(), 1));
+    << getSpecifierRange(startSpecifier, specifierLen)
+    << FixItHint::CreateRemoval(getSpecifierRange(flag.getPosition(), 1));
 }
 
 void CheckPrintfHandler::HandleIgnoredFlag(
-                                const analyze_printf::FormatSpecifier &FS,
+                                const analyze_printf::PrintfSpecifier &FS,
                                 const analyze_printf::OptionalFlag &ignoredFlag,
                                 const analyze_printf::OptionalFlag &flag,
                                 const char *startSpecifier,
@@ -1425,30 +1455,33 @@ void CheckPrintfHandler::HandleIgnoredFlag(
   S.Diag(getLocationOfByte(ignoredFlag.getPosition()),
       diag::warn_printf_ignored_flag)
     << ignoredFlag.toString() << flag.toString()
-    << getFormatSpecifierRange(startSpecifier, specifierLen)
-    << FixItHint::CreateRemoval(getFormatSpecifierRange(
+    << getSpecifierRange(startSpecifier, specifierLen)
+    << FixItHint::CreateRemoval(getSpecifierRange(
         ignoredFlag.getPosition(), 1));
 }
 
 bool
-CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
+CheckPrintfHandler::HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier
                                             &FS,
                                           const char *startSpecifier,
                                           unsigned specifierLen) {
 
+  using namespace analyze_format_string;
   using namespace analyze_printf;  
-  const ConversionSpecifier &CS = FS.getConversionSpecifier();
+  const PrintfConversionSpecifier &CS = FS.getConversionSpecifier();
 
-  if (atFirstArg) {
-    atFirstArg = false;
-    usesPositionalArgs = FS.usesPositionalArg();
-  }
-  else if (usesPositionalArgs != FS.usesPositionalArg()) {
-    // Cannot mix-and-match positional and non-positional arguments.
-    S.Diag(getLocationOfByte(CS.getStart()),
-           diag::warn_printf_mix_positional_nonpositional_args)
-      << getFormatSpecifierRange(startSpecifier, specifierLen);
-    return false;
+  if (FS.consumesDataArgument()) {
+    if (atFirstArg) {
+        atFirstArg = false;
+        usesPositionalArgs = FS.usesPositionalArg();
+    }
+    else if (usesPositionalArgs != FS.usesPositionalArg()) {
+      // Cannot mix-and-match positional and non-positional arguments.
+      S.Diag(getLocationOfByte(CS.getStart()),
+             diag::warn_format_mix_positional_nonpositional_args)
+        << getSpecifierRange(startSpecifier, specifierLen);
+      return false;
+    }
   }
 
   // First check if the field width, precision, and conversion specifier
@@ -1481,7 +1514,8 @@ CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
   // Check for using an Objective-C specific conversion specifier
   // in a non-ObjC literal.
   if (!IsObjCLiteral && CS.isObjCArg()) {
-    return HandleInvalidConversionSpecifier(FS, startSpecifier, specifierLen);
+    return HandleInvalidPrintfConversionSpecifier(FS, startSpecifier,
+                                                  specifierLen);
   }
 
   // Check for invalid use of field width
@@ -1520,17 +1554,17 @@ CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
   const LengthModifier &LM = FS.getLengthModifier();
   if (!FS.hasValidLengthModifier())
     S.Diag(getLocationOfByte(LM.getStart()),
-        diag::warn_printf_nonsensical_length)
+        diag::warn_format_nonsensical_length)
       << LM.toString() << CS.toString()
-      << getFormatSpecifierRange(startSpecifier, specifierLen)
-      << FixItHint::CreateRemoval(getFormatSpecifierRange(LM.getStart(),
+      << getSpecifierRange(startSpecifier, specifierLen)
+      << FixItHint::CreateRemoval(getSpecifierRange(LM.getStart(),
           LM.getLength()));
 
   // Are we using '%n'?
-  if (CS.getKind() == ConversionSpecifier::OutIntPtrArg) {
+  if (CS.getKind() == ConversionSpecifier::nArg) {
     // Issue a warning about this being a possible security issue.
     S.Diag(getLocationOfByte(CS.getStart()), diag::warn_printf_write_back)
-      << getFormatSpecifierRange(startSpecifier, specifierLen);
+      << getSpecifierRange(startSpecifier, specifierLen);
     // Continue checking the other format specifiers.
     return true;
   }
@@ -1539,22 +1573,8 @@ CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
   if (HasVAListArg)
     return true;
 
-  if (argIndex >= NumDataArgs) {
-    if (FS.usesPositionalArg())  {
-      S.Diag(getLocationOfByte(CS.getStart()),
-             diag::warn_printf_positional_arg_exceeds_data_args)
-        << (argIndex+1) << NumDataArgs
-        << getFormatSpecifierRange(startSpecifier, specifierLen);
-    }
-    else {
-      S.Diag(getLocationOfByte(CS.getStart()),
-             diag::warn_printf_insufficient_data_args)
-        << getFormatSpecifierRange(startSpecifier, specifierLen);
-    }
-
-    // Don't do any more checking.
+  if (!CheckNumArgs(FS, CS, startSpecifier, specifierLen, argIndex))
     return false;
-  }
 
   // Now type check the data expression that matches the
   // format specifier.
@@ -1570,7 +1590,7 @@ CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
           return true;
 
     // We may be able to offer a FixItHint if it is a supported type.
-    FormatSpecifier fixedFS = FS;
+    PrintfSpecifier fixedFS = FS;
     bool success = fixedFS.fixType(Ex->getType());
 
     if (success) {
@@ -1579,20 +1599,23 @@ CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
       llvm::raw_svector_ostream os(buf);
       fixedFS.toString(os);
 
+      // FIXME: getRepresentativeType() perhaps should return a string
+      // instead of a QualType to better handle when the representative
+      // type is 'wint_t' (which is defined in the system headers).
       S.Diag(getLocationOfByte(CS.getStart()),
           diag::warn_printf_conversion_argument_type_mismatch)
         << ATR.getRepresentativeType(S.Context) << Ex->getType()
-        << getFormatSpecifierRange(startSpecifier, specifierLen)
+        << getSpecifierRange(startSpecifier, specifierLen)
         << Ex->getSourceRange()
         << FixItHint::CreateReplacement(
-            getFormatSpecifierRange(startSpecifier, specifierLen),
+            getSpecifierRange(startSpecifier, specifierLen),
             os.str());
     }
     else {
       S.Diag(getLocationOfByte(CS.getStart()),
              diag::warn_printf_conversion_argument_type_mismatch)
         << ATR.getRepresentativeType(S.Context) << Ex->getType()
-        << getFormatSpecifierRange(startSpecifier, specifierLen)
+        << getSpecifierRange(startSpecifier, specifierLen)
         << Ex->getSourceRange();
     }
   }
@@ -1600,54 +1623,173 @@ CheckPrintfHandler::HandleFormatSpecifier(const analyze_printf::FormatSpecifier
   return true;
 }
 
-void CheckPrintfHandler::DoneProcessing() {
-  // Does the number of data arguments exceed the number of
-  // format conversions in the format string?
-  if (!HasVAListArg) {
-    // Find any arguments that weren't covered.
-    CoveredArgs.flip();
-    signed notCoveredArg = CoveredArgs.find_first();
-    if (notCoveredArg >= 0) {
-      assert((unsigned)notCoveredArg < NumDataArgs);
-      S.Diag(getDataArg((unsigned) notCoveredArg)->getLocStart(),
-             diag::warn_printf_data_arg_not_used)
-        << getFormatStringRange();
+//===--- CHECK: Scanf format string checking ------------------------------===//
+
+namespace {  
+class CheckScanfHandler : public CheckFormatHandler {
+public:
+  CheckScanfHandler(Sema &s, const StringLiteral *fexpr,
+                    const Expr *origFormatExpr, unsigned firstDataArg,
+                    unsigned numDataArgs, bool isObjCLiteral,
+                    const char *beg, bool hasVAListArg,
+                    const CallExpr *theCall, unsigned formatIdx)
+  : CheckFormatHandler(s, fexpr, origFormatExpr, firstDataArg,
+                       numDataArgs, isObjCLiteral, beg, hasVAListArg,
+                       theCall, formatIdx) {}
+  
+  bool HandleScanfSpecifier(const analyze_scanf::ScanfSpecifier &FS,
+                            const char *startSpecifier,
+                            unsigned specifierLen);
+  
+  bool HandleInvalidScanfConversionSpecifier(
+          const analyze_scanf::ScanfSpecifier &FS,
+          const char *startSpecifier,
+          unsigned specifierLen);
+
+  void HandleIncompleteScanList(const char *start, const char *end);
+};
+}
+
+void CheckScanfHandler::HandleIncompleteScanList(const char *start,
+                                                 const char *end) {
+  S.Diag(getLocationOfByte(end), diag::warn_scanf_scanlist_incomplete)
+    << getSpecifierRange(start, end - start);
+}
+
+bool CheckScanfHandler::HandleInvalidScanfConversionSpecifier(
+                                        const analyze_scanf::ScanfSpecifier &FS,
+                                        const char *startSpecifier,
+                                        unsigned specifierLen) {
+
+  const analyze_scanf::ScanfConversionSpecifier &CS =
+    FS.getConversionSpecifier();
+
+  return HandleInvalidConversionSpecifier(FS.getArgIndex(),
+                                          getLocationOfByte(CS.getStart()),
+                                          startSpecifier, specifierLen,
+                                          CS.getStart(), CS.getLength());
+}
+
+bool CheckScanfHandler::HandleScanfSpecifier(
+                                       const analyze_scanf::ScanfSpecifier &FS,
+                                       const char *startSpecifier,
+                                       unsigned specifierLen) {
+  
+  using namespace analyze_scanf;
+  using namespace analyze_format_string;  
+
+  const ScanfConversionSpecifier &CS = FS.getConversionSpecifier();
+
+  // Handle case where '%' and '*' don't consume an argument.  These shouldn't
+  // be used to decide if we are using positional arguments consistently.
+  if (FS.consumesDataArgument()) {
+    if (atFirstArg) {
+      atFirstArg = false;
+      usesPositionalArgs = FS.usesPositionalArg();
+    }
+    else if (usesPositionalArgs != FS.usesPositionalArg()) {
+      // Cannot mix-and-match positional and non-positional arguments.
+      S.Diag(getLocationOfByte(CS.getStart()),
+             diag::warn_format_mix_positional_nonpositional_args)
+        << getSpecifierRange(startSpecifier, specifierLen);
+      return false;
+    }
+  }
+  
+  // Check if the field with is non-zero.
+  const OptionalAmount &Amt = FS.getFieldWidth();
+  if (Amt.getHowSpecified() == OptionalAmount::Constant) {
+    if (Amt.getConstantAmount() == 0) {
+      const CharSourceRange &R = getSpecifierRange(Amt.getStart(),
+                                                   Amt.getConstantLength());
+      S.Diag(getLocationOfByte(Amt.getStart()),
+             diag::warn_scanf_nonzero_width)
+        << R << FixItHint::CreateRemoval(R);
     }
   }
+  
+  if (!FS.consumesDataArgument()) {
+    // FIXME: Technically specifying a precision or field width here
+    // makes no sense.  Worth issuing a warning at some point.
+    return true;
+  }
+  
+  // Consume the argument.
+  unsigned argIndex = FS.getArgIndex();
+  if (argIndex < NumDataArgs) {
+      // The check to see if the argIndex is valid will come later.
+      // We set the bit here because we may exit early from this
+      // function if we encounter some other error.
+    CoveredArgs.set(argIndex);
+  }
+  
+  // Check the length modifier is valid with the given conversion specifier.
+  const LengthModifier &LM = FS.getLengthModifier();
+  if (!FS.hasValidLengthModifier()) {
+    S.Diag(getLocationOfByte(LM.getStart()),
+           diag::warn_format_nonsensical_length)
+      << LM.toString() << CS.toString()
+      << getSpecifierRange(startSpecifier, specifierLen)
+      << FixItHint::CreateRemoval(getSpecifierRange(LM.getStart(),
+                                                    LM.getLength()));
+  }
+
+  // The remaining checks depend on the data arguments.
+  if (HasVAListArg)
+    return true;
+  
+  if (!CheckNumArgs(FS, CS, startSpecifier, specifierLen, argIndex))
+    return false;
+  
+  // FIXME: Check that the argument type matches the format specifier.
+  
+  return true;
 }
 
-void Sema::CheckPrintfString(const StringLiteral *FExpr,
+void Sema::CheckFormatString(const StringLiteral *FExpr,
                              const Expr *OrigFormatExpr,
                              const CallExpr *TheCall, bool HasVAListArg,
-                             unsigned format_idx, unsigned firstDataArg) {
-
+                             unsigned format_idx, unsigned firstDataArg,
+                             bool isPrintf) {
+  
   // CHECK: is the format string a wide literal?
   if (FExpr->isWide()) {
     Diag(FExpr->getLocStart(),
-         diag::warn_printf_format_string_is_wide_literal)
+         diag::warn_format_string_is_wide_literal)
     << OrigFormatExpr->getSourceRange();
     return;
   }
-
+  
   // Str - The format string.  NOTE: this is NOT null-terminated!
-  const char *Str = FExpr->getStrData();
-
+  llvm::StringRef StrRef = FExpr->getString();
+  const char *Str = StrRef.data();
+  unsigned StrLen = StrRef.size();
+  
   // CHECK: empty format string?
-  unsigned StrLen = FExpr->getByteLength();
-
   if (StrLen == 0) {
-    Diag(FExpr->getLocStart(), diag::warn_printf_empty_format_string)
+    Diag(FExpr->getLocStart(), diag::warn_empty_format_string)
     << OrigFormatExpr->getSourceRange();
     return;
   }
-
-  CheckPrintfHandler H(*this, FExpr, OrigFormatExpr, firstDataArg,
-                       TheCall->getNumArgs() - firstDataArg,
-                       isa<ObjCStringLiteral>(OrigFormatExpr), Str,
-                       HasVAListArg, TheCall, format_idx);
-
-  if (!analyze_printf::ParseFormatString(H, Str, Str + StrLen))
-    H.DoneProcessing();
+  
+  if (isPrintf) {
+    CheckPrintfHandler H(*this, FExpr, OrigFormatExpr, firstDataArg,
+                         TheCall->getNumArgs() - firstDataArg,
+                         isa<ObjCStringLiteral>(OrigFormatExpr), Str,
+                         HasVAListArg, TheCall, format_idx);
+  
+    if (!analyze_format_string::ParsePrintfString(H, Str, Str + StrLen))
+      H.DoneProcessing();
+  }
+  else {
+    CheckScanfHandler H(*this, FExpr, OrigFormatExpr, firstDataArg,
+                        TheCall->getNumArgs() - firstDataArg,
+                        isa<ObjCStringLiteral>(OrigFormatExpr), Str,
+                        HasVAListArg, TheCall, format_idx);
+    
+    if (!analyze_format_string::ParseScanfString(H, Str, Str + StrLen))
+      H.DoneProcessing();
+  }
 }
 
 //===--- CHECK: Return Address of Stack Variable --------------------------===//
@@ -1729,7 +1871,7 @@ static DeclRefExpr* EvalAddr(Expr *E) {
     // is AddrOf.  All others don't make sense as pointers.
     UnaryOperator *U = cast<UnaryOperator>(E);
 
-    if (U->getOpcode() == UnaryOperator::AddrOf)
+    if (U->getOpcode() == UO_AddrOf)
       return EvalVal(U->getSubExpr());
     else
       return NULL;
@@ -1739,9 +1881,9 @@ static DeclRefExpr* EvalAddr(Expr *E) {
     // Handle pointer arithmetic.  All other binary operators are not valid
     // in this context.
     BinaryOperator *B = cast<BinaryOperator>(E);
-    BinaryOperator::Opcode op = B->getOpcode();
+    BinaryOperatorKind op = B->getOpcode();
 
-    if (op != BinaryOperator::Add && op != BinaryOperator::Sub)
+    if (op != BO_Add && op != BO_Sub)
       return NULL;
 
     Expr *Base = B->getLHS();
@@ -1814,7 +1956,7 @@ static DeclRefExpr* EvalAddr(Expr *E) {
 ///  EvalVal - This function is complements EvalAddr in the mutual recursion.
 ///   See the comments for EvalAddr for more details.
 static DeclRefExpr* EvalVal(Expr *E) {
-
+do {
   // We should only be called for evaluating non-pointer expressions, or
   // expressions with a pointer type that are not used as references but instead
   // are l-values (e.g., DeclRefExpr with a pointer type).
@@ -1823,6 +1965,15 @@ static DeclRefExpr* EvalVal(Expr *E) {
   // viewed AST node.  We then recursively traverse the AST by calling
   // EvalAddr and EvalVal appropriately.
   switch (E->getStmtClass()) {
+  case Stmt::ImplicitCastExprClass: {
+    ImplicitCastExpr *IE = cast<ImplicitCastExpr>(E);
+    if (IE->getValueKind() == VK_LValue) {
+      E = IE->getSubExpr();
+      continue;
+    }
+    return NULL;
+  }
+
   case Stmt::DeclRefExprClass: {
     // DeclRefExpr: the base case.  When we hit a DeclRefExpr we are looking
     //  at code that refers to a variable's name.  We check if it has local
@@ -1835,9 +1986,11 @@ static DeclRefExpr* EvalVal(Expr *E) {
     return NULL;
   }
 
-  case Stmt::ParenExprClass:
+  case Stmt::ParenExprClass: {
     // Ignore parentheses.
-    return EvalVal(cast<ParenExpr>(E)->getSubExpr());
+    E = cast<ParenExpr>(E)->getSubExpr();
+    continue;
+  }
 
   case Stmt::UnaryOperatorClass: {
     // The only unary operator that make sense to handle here
@@ -1845,7 +1998,7 @@ static DeclRefExpr* EvalVal(Expr *E) {
     // handling all sorts of rvalues passed to a unary operator.
     UnaryOperator *U = cast<UnaryOperator>(E);
 
-    if (U->getOpcode() == UnaryOperator::Deref)
+    if (U->getOpcode() == UO_Deref)
       return EvalAddr(U->getSubExpr());
 
     return NULL;
@@ -1876,16 +2029,22 @@ static DeclRefExpr* EvalVal(Expr *E) {
     MemberExpr *M = cast<MemberExpr>(E);
 
     // Check for indirect access.  We only want direct field accesses.
-    if (!M->isArrow())
-      return EvalVal(M->getBase());
-    else
+    if (M->isArrow())
       return NULL;
+
+    // Check whether the member type is itself a reference, in which case
+    // we're not going to refer to the member, but to what the member refers to.
+    if (M->getMemberDecl()->getType()->isReferenceType())
+      return NULL;
+
+    return EvalVal(M->getBase());
   }
 
   // Everything else: we simply don't reason about them.
   default:
     return NULL;
   }
+} while (true);
 }
 
 //===--- CHECK: Floating-Point comparisons (-Wfloat-equal) ---------------===//
@@ -1954,7 +2113,6 @@ struct IntRange {
   /// True if the int is known not to have negative values.
   bool NonNegative;
 
-  IntRange() {}
   IntRange(unsigned Width, bool NonNegative)
     : Width(Width), NonNegative(NonNegative)
   {}
@@ -2063,13 +2221,13 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
   // user has an explicit widening cast, we should treat the value as
   // being of the new, wider type.
   if (ImplicitCastExpr *CE = dyn_cast<ImplicitCastExpr>(E)) {
-    if (CE->getCastKind() == CastExpr::CK_NoOp)
+    if (CE->getCastKind() == CK_NoOp)
       return GetExprRange(C, CE->getSubExpr(), MaxWidth);
 
     IntRange OutputTypeRange = IntRange::forType(C, CE->getType());
 
-    bool isIntegerCast = (CE->getCastKind() == CastExpr::CK_IntegralCast);
-    if (!isIntegerCast && CE->getCastKind() == CastExpr::CK_Unknown)
+    bool isIntegerCast = (CE->getCastKind() == CK_IntegralCast);
+    if (!isIntegerCast && CE->getCastKind() == CK_Unknown)
       isIntegerCast = CE->getSubExpr()->getType()->isIntegerType();
 
     // Assume that non-integer casts can span the full range of the type.
@@ -2108,38 +2266,38 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     switch (BO->getOpcode()) {
 
     // Boolean-valued operations are single-bit and positive.
-    case BinaryOperator::LAnd:
-    case BinaryOperator::LOr:
-    case BinaryOperator::LT:
-    case BinaryOperator::GT:
-    case BinaryOperator::LE:
-    case BinaryOperator::GE:
-    case BinaryOperator::EQ:
-    case BinaryOperator::NE:
+    case BO_LAnd:
+    case BO_LOr:
+    case BO_LT:
+    case BO_GT:
+    case BO_LE:
+    case BO_GE:
+    case BO_EQ:
+    case BO_NE:
       return IntRange::forBoolType();
 
     // The type of these compound assignments is the type of the LHS,
     // so the RHS is not necessarily an integer.
-    case BinaryOperator::MulAssign:
-    case BinaryOperator::DivAssign:
-    case BinaryOperator::RemAssign:
-    case BinaryOperator::AddAssign:
-    case BinaryOperator::SubAssign:
+    case BO_MulAssign:
+    case BO_DivAssign:
+    case BO_RemAssign:
+    case BO_AddAssign:
+    case BO_SubAssign:
       return IntRange::forType(C, E->getType());
 
     // Operations with opaque sources are black-listed.
-    case BinaryOperator::PtrMemD:
-    case BinaryOperator::PtrMemI:
+    case BO_PtrMemD:
+    case BO_PtrMemI:
       return IntRange::forType(C, E->getType());
 
     // Bitwise-and uses the *infinum* of the two source ranges.
-    case BinaryOperator::And:
-    case BinaryOperator::AndAssign:
+    case BO_And:
+    case BO_AndAssign:
       return IntRange::meet(GetExprRange(C, BO->getLHS(), MaxWidth),
                             GetExprRange(C, BO->getRHS(), MaxWidth));
 
     // Left shift gets black-listed based on a judgement call.
-    case BinaryOperator::Shl:
+    case BO_Shl:
       // ...except that we want to treat '1 << (blah)' as logically
       // positive.  It's an important idiom.
       if (IntegerLiteral *I
@@ -2151,12 +2309,12 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
       }
       // fallthrough
 
-    case BinaryOperator::ShlAssign:
+    case BO_ShlAssign:
       return IntRange::forType(C, E->getType());
 
     // Right shift by a constant can narrow its left argument.
-    case BinaryOperator::Shr:
-    case BinaryOperator::ShrAssign: {
+    case BO_Shr:
+    case BO_ShrAssign: {
       IntRange L = GetExprRange(C, BO->getLHS(), MaxWidth);
 
       // If the shift amount is a positive constant, drop the width by
@@ -2175,11 +2333,11 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
     }
 
     // Comma acts as its right operand.
-    case BinaryOperator::Comma:
+    case BO_Comma:
       return GetExprRange(C, BO->getRHS(), MaxWidth);
 
     // Black-list pointer subtractions.
-    case BinaryOperator::Sub:
+    case BO_Sub:
       if (BO->getLHS()->getType()->isPointerType())
         return IntRange::forType(C, E->getType());
       // fallthrough
@@ -2198,13 +2356,12 @@ IntRange GetExprRange(ASTContext &C, Expr *E, unsigned MaxWidth) {
   if (UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
     switch (UO->getOpcode()) {
     // Boolean-valued operations are white-listed.
-    case UnaryOperator::LNot:
+    case UO_LNot:
       return IntRange::forBoolType();
 
     // Operations with opaque sources are black-listed.
-    case UnaryOperator::Deref:
-    case UnaryOperator::AddrOf: // should be impossible
-    case UnaryOperator::OffsetOf:
+    case UO_Deref:
+    case UO_AddrOf: // should be impossible
       return IntRange::forType(C, E->getType());
 
     default:
@@ -2277,20 +2434,20 @@ bool IsZero(Sema &S, Expr *E) {
 }
 
 void CheckTrivialUnsignedComparison(Sema &S, BinaryOperator *E) {
-  BinaryOperator::Opcode op = E->getOpcode();
-  if (op == BinaryOperator::LT && IsZero(S, E->getRHS())) {
+  BinaryOperatorKind op = E->getOpcode();
+  if (op == BO_LT && IsZero(S, E->getRHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_lunsigned_always_true_comparison)
       << "< 0" << "false"
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
-  } else if (op == BinaryOperator::GE && IsZero(S, E->getRHS())) {
+  } else if (op == BO_GE && IsZero(S, E->getRHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_lunsigned_always_true_comparison)
       << ">= 0" << "true"
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
-  } else if (op == BinaryOperator::GT && IsZero(S, E->getLHS())) {
+  } else if (op == BO_GT && IsZero(S, E->getLHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_runsigned_always_true_comparison)
       << "0 >" << "false" 
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
-  } else if (op == BinaryOperator::LE && IsZero(S, E->getLHS())) {
+  } else if (op == BO_LE && IsZero(S, E->getLHS())) {
     S.Diag(E->getOperatorLoc(), diag::warn_runsigned_always_true_comparison)
       << "0 <=" << "true" 
       << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
@@ -2319,7 +2476,7 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   // We don't do anything special if this isn't an unsigned integral
   // comparison:  we're only interested in integral comparisons, and
   // signed comparisons only happen in cases we don't care to warn about.
-  if (!T->isUnsignedIntegerType())
+  if (!T->hasUnsignedIntegerRepresentation())
     return AnalyzeImpConvsInComparison(S, E);
 
   Expr *lex = E->getLHS()->IgnoreParenImpCasts();
@@ -2328,12 +2485,12 @@ void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   // Check to see if one of the (unmodified) operands is of different
   // signedness.
   Expr *signedOperand, *unsignedOperand;
-  if (lex->getType()->isSignedIntegerType()) {
-    assert(!rex->getType()->isSignedIntegerType() &&
+  if (lex->getType()->hasSignedIntegerRepresentation()) {
+    assert(!rex->getType()->hasSignedIntegerRepresentation() &&
            "unsigned comparison between two signed integer expressions?");
     signedOperand = lex;
     unsignedOperand = rex;
-  } else if (rex->getType()->isSignedIntegerType()) {
+  } else if (rex->getType()->hasSignedIntegerRepresentation()) {
     signedOperand = rex;
     unsignedOperand = lex;
   } else {
@@ -2648,3 +2805,48 @@ bool Sema::CheckParmsForFunctionDef(FunctionDecl *FD) {
 
   return HasInvalidParm;
 }
+
+/// CheckCastAlign - Implements -Wcast-align, which warns when a
+/// pointer cast increases the alignment requirements.
+void Sema::CheckCastAlign(Expr *Op, QualType T, SourceRange TRange) {
+  // This is actually a lot of work to potentially be doing on every
+  // cast; don't do it if we're ignoring -Wcast_align (as is the default).
+  if (getDiagnostics().getDiagnosticLevel(diag::warn_cast_align)
+        == Diagnostic::Ignored)
+    return;
+
+  // Ignore dependent types.
+  if (T->isDependentType() || Op->getType()->isDependentType())
+    return;
+
+  // Require that the destination be a pointer type.
+  const PointerType *DestPtr = T->getAs<PointerType>();
+  if (!DestPtr) return;
+
+  // If the destination has alignment 1, we're done.
+  QualType DestPointee = DestPtr->getPointeeType();
+  if (DestPointee->isIncompleteType()) return;
+  CharUnits DestAlign = Context.getTypeAlignInChars(DestPointee);
+  if (DestAlign.isOne()) return;
+
+  // Require that the source be a pointer type.
+  const PointerType *SrcPtr = Op->getType()->getAs<PointerType>();
+  if (!SrcPtr) return;
+  QualType SrcPointee = SrcPtr->getPointeeType();
+
+  // Whitelist casts from cv void*.  We already implicitly
+  // whitelisted casts to cv void*, since they have alignment 1.
+  // Also whitelist casts involving incomplete types, which implicitly
+  // includes 'void'.
+  if (SrcPointee->isIncompleteType()) return;
+
+  CharUnits SrcAlign = Context.getTypeAlignInChars(SrcPointee);
+  if (SrcAlign >= DestAlign) return;
+
+  Diag(TRange.getBegin(), diag::warn_cast_align)
+    << Op->getType() << T
+    << static_cast<unsigned>(SrcAlign.getQuantity())
+    << static_cast<unsigned>(DestAlign.getQuantity())
+    << TRange << Op->getSourceRange();
+}
+
diff --git a/lib/Sema/SemaCodeComplete.cpp b/lib/Sema/SemaCodeComplete.cpp
index 55288750fd5f..f00d1cd20f1e 100644
--- a/lib/Sema/SemaCodeComplete.cpp
+++ b/lib/Sema/SemaCodeComplete.cpp
@@ -10,10 +10,14 @@
 //  This file defines the code-completion semantic actions.
 //
 //===----------------------------------------------------------------------===//
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Overload.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/ExternalSemaSource.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/Lex/MacroInfo.h"
@@ -21,11 +25,13 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Twine.h"
 #include <list>
 #include <map>
 #include <vector>
 
 using namespace clang;
+using namespace sema;
 
 namespace {
   /// \brief A container of code-completion results.
@@ -37,7 +43,7 @@ namespace {
     /// filtered out (returns false).
     typedef bool (ResultBuilder::*LookupFilter)(NamedDecl *) const;
     
-    typedef CodeCompleteConsumer::Result Result;
+    typedef CodeCompletionResult Result;
     
   private:
     /// \brief The actual results we have found.
@@ -130,17 +136,28 @@ namespace {
     /// different levels of, e.g., the inheritance hierarchy.
     std::list<ShadowMap> ShadowMaps;
     
+    /// \brief If we're potentially referring to a C++ member function, the set
+    /// of qualifiers applied to the object type.
+    Qualifiers ObjectTypeQualifiers;
+    
+    /// \brief Whether the \p ObjectTypeQualifiers field is active.
+    bool HasObjectTypeQualifiers;
+    
+    /// \brief The selector that we prefer.
+    Selector PreferredSelector;
+    
     void AdjustResultPriorityForPreferredType(Result &R);
 
   public:
     explicit ResultBuilder(Sema &SemaRef, LookupFilter Filter = 0)
-      : SemaRef(SemaRef), Filter(Filter), AllowNestedNameSpecifiers(false) { }
+      : SemaRef(SemaRef), Filter(Filter), AllowNestedNameSpecifiers(false),
+        HasObjectTypeQualifiers(false) { }
     
     /// \brief Whether we should include code patterns in the completion
     /// results.
     bool includeCodePatterns() const {
       return SemaRef.CodeCompleter && 
-      SemaRef.CodeCompleter->includeCodePatterns();
+             SemaRef.CodeCompleter->includeCodePatterns();
     }
     
     /// \brief Set the filter used for code-completion results.
@@ -161,6 +178,27 @@ namespace {
       PreferredType = SemaRef.Context.getCanonicalType(T); 
     }
     
+    /// \brief Set the cv-qualifiers on the object type, for us in filtering
+    /// calls to member functions.
+    ///
+    /// When there are qualifiers in this set, they will be used to filter
+    /// out member functions that aren't available (because there will be a 
+    /// cv-qualifier mismatch) or prefer functions with an exact qualifier
+    /// match.
+    void setObjectTypeQualifiers(Qualifiers Quals) {
+      ObjectTypeQualifiers = Quals;
+      HasObjectTypeQualifiers = true;
+    }
+    
+    /// \brief Set the preferred selector.
+    ///
+    /// When an Objective-C method declaration result is added, and that
+    /// method's selector matches this preferred selector, we give that method
+    /// a slight priority boost.
+    void setPreferredSelector(Selector Sel) {
+      PreferredSelector = Sel;
+    }
+    
     /// \brief Specify whether nested-name-specifiers are allowed.
     void allowNestedNameSpecifiers(bool Allow = true) {
       AllowNestedNameSpecifiers = Allow;
@@ -227,6 +265,7 @@ namespace {
     //@{
     bool IsOrdinaryName(NamedDecl *ND) const;
     bool IsOrdinaryNonTypeName(NamedDecl *ND) const;
+    bool IsIntegralConstantValue(NamedDecl *ND) const;
     bool IsOrdinaryNonValueName(NamedDecl *ND) const;
     bool IsNestedNameSpecifier(NamedDecl *ND) const;
     bool IsEnum(NamedDecl *ND) const;
@@ -238,6 +277,7 @@ namespace {
     bool IsMember(NamedDecl *ND) const;
     bool IsObjCIvar(NamedDecl *ND) const;
     bool IsObjCMessageReceiver(NamedDecl *ND) const;
+    bool IsObjCCollection(NamedDecl *ND) const;
     //@}    
   };  
 }
@@ -365,8 +405,12 @@ getRequiredQualification(ASTContext &Context,
     DeclContext *Parent = TargetParents.back();
     TargetParents.pop_back();
     
-    if (NamespaceDecl *Namespace = dyn_cast<NamespaceDecl>(Parent))
+    if (NamespaceDecl *Namespace = dyn_cast<NamespaceDecl>(Parent)) {
+      if (!Namespace->getIdentifier())
+        continue;
+
       Result = NestedNameSpecifier::Create(Context, Result, Namespace);
+    }
     else if (TagDecl *TD = dyn_cast<TagDecl>(Parent))
       Result = NestedNameSpecifier::Create(Context, Result,
                                            false,
@@ -425,6 +469,12 @@ bool ResultBuilder::isInterestingDecl(NamedDecl *ND,
   if (isa<CXXConstructorDecl>(ND))
     return false;
   
+  if (Filter == &ResultBuilder::IsNestedNameSpecifier ||
+      ((isa<NamespaceDecl>(ND) || isa<NamespaceAliasDecl>(ND)) &&
+       Filter != &ResultBuilder::IsNamespace &&
+       Filter != &ResultBuilder::IsNamespaceOrAlias))
+    AsNestedNameSpecifier = true;
+
   // Filter out any unwanted results.
   if (Filter && !(this->*Filter)(ND)) {
     // Check whether it is interesting as a nested-name-specifier.
@@ -438,11 +488,7 @@ bool ResultBuilder::isInterestingDecl(NamedDecl *ND,
     }
 
     return false;
-  }
-
-  if (Filter == &ResultBuilder::IsNestedNameSpecifier)
-    AsNestedNameSpecifier = true;
-  
+  }  
   // ... then it must be interesting!
   return true;
 }
@@ -455,13 +501,13 @@ bool ResultBuilder::CheckHiddenResult(Result &R, DeclContext *CurContext,
   if (!SemaRef.getLangOptions().CPlusPlus)
     return true;
   
-  DeclContext *HiddenCtx = R.Declaration->getDeclContext()->getLookupContext();
+  DeclContext *HiddenCtx = R.Declaration->getDeclContext()->getRedeclContext();
   
   // There is no way to qualify a name declared in a function or method.
   if (HiddenCtx->isFunctionOrMethod())
     return true;
   
-  if (HiddenCtx == Hiding->getDeclContext()->getLookupContext())
+  if (HiddenCtx == Hiding->getDeclContext()->getRedeclContext())
     return true;
   
   // We can refer to the result with the appropriate qualification. Do it.
@@ -475,21 +521,9 @@ bool ResultBuilder::CheckHiddenResult(Result &R, DeclContext *CurContext,
   return false;
 }
 
-enum SimplifiedTypeClass {
-  STC_Arithmetic,
-  STC_Array,
-  STC_Block,
-  STC_Function,
-  STC_ObjectiveC,
-  STC_Other,
-  STC_Pointer,
-  STC_Record,
-  STC_Void
-};
-
 /// \brief A simplified classification of types used to determine whether two
 /// types are "similar enough" when adjusting priorities.
-static SimplifiedTypeClass getSimplifiedTypeClass(CanQualType T) {
+SimplifiedTypeClass clang::getSimplifiedTypeClass(CanQualType T) {
   switch (T->getTypeClass()) {
   case Type::Builtin:
     switch (cast<BuiltinType>(T)->getKind()) {
@@ -560,7 +594,7 @@ static SimplifiedTypeClass getSimplifiedTypeClass(CanQualType T) {
 
 /// \brief Get the type that a given expression will have if this declaration
 /// is used as an expression in its "typical" code-completion form.
-static QualType getDeclUsageType(ASTContext &C, NamedDecl *ND) {
+QualType clang::getDeclUsageType(ASTContext &C, NamedDecl *ND) {
   ND = cast<NamedDecl>(ND->getUnderlyingDecl());
   
   if (TypeDecl *Type = dyn_cast<TypeDecl>(ND))
@@ -594,9 +628,12 @@ void ResultBuilder::AdjustResultPriorityForPreferredType(Result &R) {
   
   CanQualType TC = SemaRef.Context.getCanonicalType(T);
   // Check for exactly-matching types (modulo qualifiers).
-  if (SemaRef.Context.hasSameUnqualifiedType(PreferredType, TC))
-    R.Priority /= CCF_ExactTypeMatch;
-  // Check for nearly-matching types, based on classification of each.
+  if (SemaRef.Context.hasSameUnqualifiedType(PreferredType, TC)) {
+    if (PreferredType->isVoidType())
+      R.Priority += CCD_VoidMatch;
+    else
+      R.Priority /= CCF_ExactTypeMatch;
+  } // Check for nearly-matching types, based on classification of each.
   else if ((getSimplifiedTypeClass(PreferredType)
                                                == getSimplifiedTypeClass(TC)) &&
            !(PreferredType->isEnumeralType() && TC->isEnumeralType()))
@@ -681,7 +718,14 @@ void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
   // Make sure that any given declaration only shows up in the result set once.
   if (!AllDeclsFound.insert(CanonDecl))
     return;
-  
+
+  // If this is an Objective-C method declaration whose selector matches our
+  // preferred selector, give it a priority boost.
+  if (!PreferredSelector.isNull())
+    if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(R.Declaration))
+      if (PreferredSelector == Method->getSelector())
+        R.Priority += CCD_SelectorMatch;
+
   // If the filter is for nested-name-specifiers, then this result starts a
   // nested-name-specifier.
   if (AsNestedNameSpecifier) {
@@ -689,7 +733,7 @@ void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
     R.Priority = CCP_NestedNameSpecifier;
   } else if (!PreferredType.isNull())
       AdjustResultPriorityForPreferredType(R);
-
+      
   // If this result is supposed to have an informative qualifier, add one.
   if (R.QualifierIsInformative && !R.Qualifier &&
       !R.StartsNestedNameSpecifier) {
@@ -742,7 +786,7 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
   }
   else if (Filter == &ResultBuilder::IsMember && !R.Qualifier && InBaseClass &&
            isa<CXXRecordDecl>(R.Declaration->getDeclContext()
-                                                  ->getLookupContext()))
+                                                  ->getRedeclContext()))
     R.QualifierIsInformative = true;
 
   // If this result is supposed to have an informative qualifier, add one.
@@ -762,9 +806,30 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
   if (InBaseClass)
     R.Priority += CCD_InBaseClass;
   
+  // If this is an Objective-C method declaration whose selector matches our
+  // preferred selector, give it a priority boost.
+  if (!PreferredSelector.isNull())
+    if (ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(R.Declaration))
+      if (PreferredSelector == Method->getSelector())
+        R.Priority += CCD_SelectorMatch;
+
   if (!PreferredType.isNull())
     AdjustResultPriorityForPreferredType(R);
   
+  if (HasObjectTypeQualifiers)
+    if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(R.Declaration))
+      if (Method->isInstance()) {
+        Qualifiers MethodQuals
+                        = Qualifiers::fromCVRMask(Method->getTypeQualifiers());
+        if (ObjectTypeQualifiers == MethodQuals)
+          R.Priority += CCD_ObjectQualifierMatch;
+        else if (ObjectTypeQualifiers - MethodQuals) {
+          // The method cannot be invoked, because doing so would drop 
+          // qualifiers.
+          return;
+        }
+      }
+  
   // Insert this result into the set of results.
   Results.push_back(R);
 }
@@ -821,6 +886,17 @@ bool ResultBuilder::IsOrdinaryNonTypeName(NamedDecl *ND) const {
   return ND->getIdentifierNamespace() & IDNS;
 }
 
+bool ResultBuilder::IsIntegralConstantValue(NamedDecl *ND) const {
+  if (!IsOrdinaryNonTypeName(ND))
+    return 0;
+  
+  if (ValueDecl *VD = dyn_cast<ValueDecl>(ND->getUnderlyingDecl()))
+    if (VD->getType()->isIntegralOrEnumerationType())
+      return true;
+        
+  return false;
+}
+
 /// \brief Determines whether this given declaration will be found by
 /// ordinary name lookup.
 bool ResultBuilder::IsOrdinaryNonValueName(NamedDecl *ND) const {
@@ -888,7 +964,10 @@ bool ResultBuilder::IsNamespaceOrAlias(NamedDecl *ND) const {
 
 /// \brief Determines whether the given declaration is a type.
 bool ResultBuilder::IsType(NamedDecl *ND) const {
-  return isa<TypeDecl>(ND);
+  if (UsingShadowDecl *Using = dyn_cast<UsingShadowDecl>(ND))
+    ND = Using->getTargetDecl();
+  
+  return isa<TypeDecl>(ND) || isa<ObjCInterfaceDecl>(ND);
 }
 
 /// \brief Determines which members of a class should be visible via
@@ -944,6 +1023,20 @@ bool ResultBuilder::IsObjCMessageReceiver(NamedDecl *ND) const {
   return isObjCReceiverType(SemaRef.Context, T);
 }
 
+bool ResultBuilder::IsObjCCollection(NamedDecl *ND) const {
+  if ((SemaRef.getLangOptions().CPlusPlus && !IsOrdinaryName(ND)) ||
+      (!SemaRef.getLangOptions().CPlusPlus && !IsOrdinaryNonTypeName(ND)))
+    return false;
+  
+  QualType T = getDeclUsageType(SemaRef.Context, ND);
+  if (T.isNull())
+    return false;
+  
+  T = SemaRef.Context.getBaseElementType(T);
+  return T->isObjCObjectType() || T->isObjCObjectPointerType() ||
+         T->isObjCIdType() || 
+         (SemaRef.getLangOptions().CPlusPlus && T->isRecordType());
+}
 
 /// \rief Determines whether the given declaration is an Objective-C
 /// instance variable.
@@ -971,7 +1064,7 @@ namespace {
 /// \brief Add type specifiers for the current language as keyword results.
 static void AddTypeSpecifierResults(const LangOptions &LangOpts,
                                     ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   Results.AddResult(Result("short", CCP_Type));
   Results.AddResult(Result("long", CCP_Type));
   Results.AddResult(Result("signed", CCP_Type));
@@ -1046,10 +1139,10 @@ static void AddTypeSpecifierResults(const LangOptions &LangOpts,
   }
 }
 
-static void AddStorageSpecifiers(Action::CodeCompletionContext CCC,
+static void AddStorageSpecifiers(Sema::ParserCompletionContext CCC,
                                  const LangOptions &LangOpts, 
                                  ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   // Note: we don't suggest either "auto" or "register", because both
   // are pointless as storage specifiers. Elsewhere, we suggest "auto"
   // in C++0x as a type specifier.
@@ -1057,13 +1150,13 @@ static void AddStorageSpecifiers(Action::CodeCompletionContext CCC,
   Results.AddResult(Result("static"));
 }
 
-static void AddFunctionSpecifiers(Action::CodeCompletionContext CCC,
+static void AddFunctionSpecifiers(Sema::ParserCompletionContext CCC,
                                   const LangOptions &LangOpts, 
                                   ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   switch (CCC) {
-  case Action::CCC_Class:
-  case Action::CCC_MemberTemplate:
+  case Sema::PCC_Class:
+  case Sema::PCC_MemberTemplate:
     if (LangOpts.CPlusPlus) {
       Results.AddResult(Result("explicit"));
       Results.AddResult(Result("friend"));
@@ -1072,20 +1165,21 @@ static void AddFunctionSpecifiers(Action::CodeCompletionContext CCC,
     }    
     // Fall through
 
-  case Action::CCC_ObjCInterface:
-  case Action::CCC_ObjCImplementation:
-  case Action::CCC_Namespace:
-  case Action::CCC_Template:
+  case Sema::PCC_ObjCInterface:
+  case Sema::PCC_ObjCImplementation:
+  case Sema::PCC_Namespace:
+  case Sema::PCC_Template:
     if (LangOpts.CPlusPlus || LangOpts.C99)
       Results.AddResult(Result("inline"));
     break;
 
-  case Action::CCC_ObjCInstanceVariableList:
-  case Action::CCC_Expression:
-  case Action::CCC_Statement:
-  case Action::CCC_ForInit:
-  case Action::CCC_Condition:
-  case Action::CCC_RecoveryInFunction:
+  case Sema::PCC_ObjCInstanceVariableList:
+  case Sema::PCC_Expression:
+  case Sema::PCC_Statement:
+  case Sema::PCC_ForInit:
+  case Sema::PCC_Condition:
+  case Sema::PCC_RecoveryInFunction:
+  case Sema::PCC_Type:
     break;
   }
 }
@@ -1110,31 +1204,32 @@ static void AddTypedefResult(ResultBuilder &Results) {
   Pattern->AddPlaceholderChunk("type");
   Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
   Pattern->AddPlaceholderChunk("name");
-  Results.AddResult(CodeCompleteConsumer::Result(Pattern));        
+  Results.AddResult(CodeCompletionResult(Pattern));        
 }
 
-static bool WantTypesInContext(Action::CodeCompletionContext CCC,
+static bool WantTypesInContext(Sema::ParserCompletionContext CCC,
                                const LangOptions &LangOpts) {
   if (LangOpts.CPlusPlus)
     return true;
   
   switch (CCC) {
-  case Action::CCC_Namespace:
-  case Action::CCC_Class:
-  case Action::CCC_ObjCInstanceVariableList:
-  case Action::CCC_Template:
-  case Action::CCC_MemberTemplate:
-  case Action::CCC_Statement:
-  case Action::CCC_RecoveryInFunction:
+  case Sema::PCC_Namespace:
+  case Sema::PCC_Class:
+  case Sema::PCC_ObjCInstanceVariableList:
+  case Sema::PCC_Template:
+  case Sema::PCC_MemberTemplate:
+  case Sema::PCC_Statement:
+  case Sema::PCC_RecoveryInFunction:
+  case Sema::PCC_Type:
     return true;
     
-  case Action::CCC_ObjCInterface:
-  case Action::CCC_ObjCImplementation:
-  case Action::CCC_Expression:
-  case Action::CCC_Condition:
+  case Sema::PCC_ObjCInterface:
+  case Sema::PCC_ObjCImplementation:
+  case Sema::PCC_Expression:
+  case Sema::PCC_Condition:
     return false;
     
-  case Action::CCC_ForInit:
+  case Sema::PCC_ForInit:
     return LangOpts.ObjC1 || LangOpts.C99;
   }
   
@@ -1142,13 +1237,13 @@ static bool WantTypesInContext(Action::CodeCompletionContext CCC,
 }
 
 /// \brief Add language constructs that show up for "ordinary" names.
-static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
+static void AddOrdinaryNameResults(Sema::ParserCompletionContext CCC,
                                    Scope *S,
                                    Sema &SemaRef,
                                    ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   switch (CCC) {
-  case Action::CCC_Namespace:
+  case Sema::PCC_Namespace:
     if (SemaRef.getLangOptions().CPlusPlus) {
       CodeCompletionString *Pattern = 0;
       
@@ -1207,7 +1302,7 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
     AddTypedefResult(Results);
     // Fall through
 
-  case Action::CCC_Class:
+  case Sema::PCC_Class:
     if (SemaRef.getLangOptions().CPlusPlus) {
       // Using declaration
       CodeCompletionString *Pattern = new CodeCompletionString;
@@ -1231,7 +1326,7 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
         Results.AddResult(Result(Pattern));
       }
 
-      if (CCC == Action::CCC_Class) {
+      if (CCC == Sema::PCC_Class) {
         AddTypedefResult(Results);
 
         // public:
@@ -1255,8 +1350,8 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
     }
     // Fall through
 
-  case Action::CCC_Template:
-  case Action::CCC_MemberTemplate:
+  case Sema::PCC_Template:
+  case Sema::PCC_MemberTemplate:
     if (SemaRef.getLangOptions().CPlusPlus && Results.includeCodePatterns()) {
       // template < parameters >
       CodeCompletionString *Pattern = new CodeCompletionString;
@@ -1271,24 +1366,24 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
     AddFunctionSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     break;
 
-  case Action::CCC_ObjCInterface:
+  case Sema::PCC_ObjCInterface:
     AddObjCInterfaceResults(SemaRef.getLangOptions(), Results, true);
     AddStorageSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     AddFunctionSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     break;
       
-  case Action::CCC_ObjCImplementation:
+  case Sema::PCC_ObjCImplementation:
     AddObjCImplementationResults(SemaRef.getLangOptions(), Results, true);
     AddStorageSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     AddFunctionSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     break;
       
-  case Action::CCC_ObjCInstanceVariableList:
+  case Sema::PCC_ObjCInstanceVariableList:
     AddObjCVisibilityResults(SemaRef.getLangOptions(), Results, true);
     break;
       
-  case Action::CCC_RecoveryInFunction:
-  case Action::CCC_Statement: {
+  case Sema::PCC_RecoveryInFunction:
+  case Sema::PCC_Statement: {
     AddTypedefResult(Results);
 
     CodeCompletionString *Pattern = 0;
@@ -1344,7 +1439,7 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
     }
     
     // Switch-specific statements.
-    if (!SemaRef.getSwitchStack().empty()) {
+    if (!SemaRef.getCurFunction()->SwitchStack.empty()) {
       // case expression:
       Pattern = new CodeCompletionString;
       Pattern->AddTypedTextChunk("case");
@@ -1460,12 +1555,12 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
   }
 
   // Fall through (for statement expressions).
-  case Action::CCC_ForInit:
-  case Action::CCC_Condition:
+  case Sema::PCC_ForInit:
+  case Sema::PCC_Condition:
     AddStorageSpecifiers(CCC, SemaRef.getLangOptions(), Results);
     // Fall through: conditions and statements can have expressions.
 
-  case Action::CCC_Expression: {
+  case Sema::PCC_Expression: {
     CodeCompletionString *Pattern = 0;
     if (SemaRef.getLangOptions().CPlusPlus) {
       // 'this', if we're in a non-static member function.
@@ -1600,12 +1695,15 @@ static void AddOrdinaryNameResults(Action::CodeCompletionContext CCC,
     Results.AddResult(Result(Pattern));
     break;
   }
+      
+  case Sema::PCC_Type:
+    break;
   }
 
   if (WantTypesInContext(CCC, SemaRef.getLangOptions()))
     AddTypeSpecifierResults(SemaRef.getLangOptions(), Results);
 
-  if (SemaRef.getLangOptions().CPlusPlus)
+  if (SemaRef.getLangOptions().CPlusPlus && CCC != Sema::PCC_Type)
     Results.AddResult(Result("operator"));
 }
 
@@ -1645,6 +1743,117 @@ static void AddResultTypeChunk(ASTContext &Context,
   Result->AddResultTypeChunk(TypeStr);
 }
 
+static void MaybeAddSentinel(ASTContext &Context, NamedDecl *FunctionOrMethod,
+                             CodeCompletionString *Result) {
+  if (SentinelAttr *Sentinel = FunctionOrMethod->getAttr<SentinelAttr>())
+    if (Sentinel->getSentinel() == 0) {
+      if (Context.getLangOptions().ObjC1 &&
+          Context.Idents.get("nil").hasMacroDefinition())
+        Result->AddTextChunk(", nil");
+      else if (Context.Idents.get("NULL").hasMacroDefinition())
+        Result->AddTextChunk(", NULL");
+      else
+        Result->AddTextChunk(", (void*)0");
+    }
+}
+
+static std::string FormatFunctionParameter(ASTContext &Context,
+                                           ParmVarDecl *Param,
+                                           bool SuppressName = false) {
+  bool ObjCMethodParam = isa<ObjCMethodDecl>(Param->getDeclContext());
+  if (Param->getType()->isDependentType() ||
+      !Param->getType()->isBlockPointerType()) {
+    // The argument for a dependent or non-block parameter is a placeholder 
+    // containing that parameter's type.
+    std::string Result;
+    
+    if (Param->getIdentifier() && !ObjCMethodParam && !SuppressName)
+      Result = Param->getIdentifier()->getName();
+    
+    Param->getType().getAsStringInternal(Result, 
+                                         Context.PrintingPolicy);
+    
+    if (ObjCMethodParam) {
+      Result = "(" + Result;
+      Result += ")";
+      if (Param->getIdentifier() && !SuppressName)
+        Result += Param->getIdentifier()->getName();
+    }
+    return Result;
+  }
+  
+  // The argument for a block pointer parameter is a block literal with
+  // the appropriate type.
+  FunctionProtoTypeLoc *Block = 0;
+  TypeLoc TL;
+  if (TypeSourceInfo *TSInfo = Param->getTypeSourceInfo()) {
+    TL = TSInfo->getTypeLoc().getUnqualifiedLoc();
+    while (true) {
+      // Look through typedefs.
+      if (TypedefTypeLoc *TypedefTL = dyn_cast<TypedefTypeLoc>(&TL)) {
+        if (TypeSourceInfo *InnerTSInfo
+            = TypedefTL->getTypedefDecl()->getTypeSourceInfo()) {
+          TL = InnerTSInfo->getTypeLoc().getUnqualifiedLoc();
+          continue;
+        }
+      }
+      
+      // Look through qualified types
+      if (QualifiedTypeLoc *QualifiedTL = dyn_cast<QualifiedTypeLoc>(&TL)) {
+        TL = QualifiedTL->getUnqualifiedLoc();
+        continue;
+      }
+      
+      // Try to get the function prototype behind the block pointer type,
+      // then we're done.
+      if (BlockPointerTypeLoc *BlockPtr
+          = dyn_cast<BlockPointerTypeLoc>(&TL)) {
+        TL = BlockPtr->getPointeeLoc();
+        Block = dyn_cast<FunctionProtoTypeLoc>(&TL);
+      }
+      break;
+    }
+  }
+
+  if (!Block) {
+    // We were unable to find a FunctionProtoTypeLoc with parameter names
+    // for the block; just use the parameter type as a placeholder.
+    std::string Result;
+    Param->getType().getUnqualifiedType().
+                            getAsStringInternal(Result, Context.PrintingPolicy);
+    
+    if (ObjCMethodParam) {
+      Result = "(" + Result;
+      Result += ")";
+      if (Param->getIdentifier())
+        Result += Param->getIdentifier()->getName();
+    }
+      
+    return Result;
+  }
+  
+  // We have the function prototype behind the block pointer type, as it was
+  // written in the source.
+  std::string Result = "(^)(";
+  for (unsigned I = 0, N = Block->getNumArgs(); I != N; ++I) {
+    if (I)
+      Result += ", ";
+    Result += FormatFunctionParameter(Context, Block->getArg(I));
+    
+    if (I == N - 1 && Block->getTypePtr()->isVariadic())
+      Result += ", ...";
+  }
+  if (Block->getTypePtr()->isVariadic() && Block->getNumArgs() == 0)
+    Result += "...";
+  else if (Block->getNumArgs() == 0 && !Context.getLangOptions().CPlusPlus)
+    Result += "void";
+             
+  Result += ")";
+  Block->getTypePtr()->getResultType().getAsStringInternal(Result, 
+                                                        Context.PrintingPolicy);
+  return Result;
+}
+
 /// \brief Add function parameter chunks to the given code completion string.
 static void AddFunctionParameterChunks(ASTContext &Context,
                                        FunctionDecl *Function,
@@ -1668,21 +1877,23 @@ static void AddFunctionParameterChunks(ASTContext &Context,
       CCStr->AddChunk(Chunk(CodeCompletionString::CK_Comma));
     
     // Format the placeholder string.
-    std::string PlaceholderStr;
-    if (Param->getIdentifier())
-      PlaceholderStr = Param->getIdentifier()->getName();
-    
-    Param->getType().getAsStringInternal(PlaceholderStr, 
-                                         Context.PrintingPolicy);
-    
+    std::string PlaceholderStr = FormatFunctionParameter(Context, Param);
+        
+    if (Function->isVariadic() && P == N - 1)
+      PlaceholderStr += ", ...";
+
     // Add the placeholder string.
     CCStr->AddPlaceholderChunk(PlaceholderStr);
   }
   
   if (const FunctionProtoType *Proto 
         = Function->getType()->getAs<FunctionProtoType>())
-    if (Proto->isVariadic())
-      CCStr->AddPlaceholderChunk(", ...");
+    if (Proto->isVariadic()) {
+      if (Proto->getNumArgs() == 0)
+        CCStr->AddPlaceholderChunk("...");
+
+      MaybeAddSentinel(Context, Function, CCStr);
+    }
 }
 
 /// \brief Add template parameter chunks to the given code completion string.
@@ -1799,13 +2010,15 @@ static void AddFunctionTypeQualsToCompletionString(CodeCompletionString *Result,
 /// how to use this result, or NULL to indicate that the string or name of the
 /// result is all that is needed.
 CodeCompletionString *
-CodeCompleteConsumer::Result::CreateCodeCompletionString(Sema &S) {
+CodeCompletionResult::CreateCodeCompletionString(Sema &S,
+                                               CodeCompletionString *Result) {
   typedef CodeCompletionString::Chunk Chunk;
   
   if (Kind == RK_Pattern)
-    return Pattern->Clone();
+    return Pattern->Clone(Result);
   
-  CodeCompletionString *Result = new CodeCompletionString;
+  if (!Result)
+    Result = new CodeCompletionString;
 
   if (Kind == RK_Keyword) {
     Result->AddTypedTextChunk(Keyword);
@@ -1978,10 +2191,20 @@ CodeCompleteConsumer::Result::CreateCodeCompletionString(Sema &S) {
         continue;
 
       std::string Arg;
-      (*P)->getType().getAsStringInternal(Arg, S.Context.PrintingPolicy);
-      Arg = "(" + Arg + ")";
-      if (IdentifierInfo *II = (*P)->getIdentifier())
-        Arg += II->getName().str();
+      
+      if ((*P)->getType()->isBlockPointerType() && !DeclaringEntity)
+        Arg = FormatFunctionParameter(S.Context, *P, true);
+      else {
+        (*P)->getType().getAsStringInternal(Arg, S.Context.PrintingPolicy);
+        Arg = "(" + Arg + ")";
+        if (IdentifierInfo *II = (*P)->getIdentifier())
+          if (DeclaringEntity || AllParametersAreInformative)
+            Arg += II->getName().str();
+      }
+      
+      if (Method->isVariadic() && (P + 1) == PEnd)
+        Arg += ", ...";
+      
       if (DeclaringEntity)
         Result->AddTextChunk(Arg);
       else if (AllParametersAreInformative)
@@ -1991,12 +2214,16 @@ CodeCompleteConsumer::Result::CreateCodeCompletionString(Sema &S) {
     }
 
     if (Method->isVariadic()) {
-      if (DeclaringEntity)
-        Result->AddTextChunk(", ...");
-      else if (AllParametersAreInformative)
-        Result->AddInformativeChunk(", ...");
-      else
-        Result->AddPlaceholderChunk(", ...");
+      if (Method->param_size() == 0) {
+        if (DeclaringEntity)
+          Result->AddTextChunk(", ...");
+        else if (AllParametersAreInformative)
+          Result->AddInformativeChunk(", ...");
+        else
+          Result->AddPlaceholderChunk(", ...");
+      }
+      
+      MaybeAddSentinel(S.Context, Method, Result);
     }
     
     return Result;
@@ -2076,226 +2303,421 @@ CodeCompleteConsumer::OverloadCandidate::CreateSignatureString(
   return Result;
 }
 
-namespace {
-  struct SortCodeCompleteResult {
-    typedef CodeCompleteConsumer::Result Result;
-    
-    bool isEarlierDeclarationName(DeclarationName X, DeclarationName Y) const {
-      Selector XSel = X.getObjCSelector();
-      Selector YSel = Y.getObjCSelector();
-      if (!XSel.isNull() && !YSel.isNull()) {
-        // We are comparing two selectors.
-        unsigned N = std::min(XSel.getNumArgs(), YSel.getNumArgs());
-        if (N == 0)
-          ++N;
-        for (unsigned I = 0; I != N; ++I) {
-          IdentifierInfo *XId = XSel.getIdentifierInfoForSlot(I);
-          IdentifierInfo *YId = YSel.getIdentifierInfoForSlot(I);
-          if (!XId || !YId)
-            return XId && !YId;
-          
-          switch (XId->getName().compare_lower(YId->getName())) {
-          case -1: return true;
-          case 1: return false;
-          default: break;
-          }
-        }
-    
-        return XSel.getNumArgs() < YSel.getNumArgs();
-      }
+unsigned clang::getMacroUsagePriority(llvm::StringRef MacroName, 
+                                      bool PreferredTypeIsPointer) {
+  unsigned Priority = CCP_Macro;
+  
+  // Treat the "nil" and "NULL" macros as null pointer constants.
+  if (MacroName.equals("nil") || MacroName.equals("NULL")) {
+    Priority = CCP_Constant;
+    if (PreferredTypeIsPointer)
+      Priority = Priority / CCF_SimilarTypeMatch;
+  }
+  
+  return Priority;
+}
 
-      // For non-selectors, order by kind.
-      if (X.getNameKind() != Y.getNameKind())
-        return X.getNameKind() < Y.getNameKind();
-      
-      // Order identifiers by comparison of their lowercased names.
-      if (IdentifierInfo *XId = X.getAsIdentifierInfo())
-        return XId->getName().compare_lower(
-                                     Y.getAsIdentifierInfo()->getName()) < 0;
-
-      // Order overloaded operators by the order in which they appear
-      // in our list of operators.
-      if (OverloadedOperatorKind XOp = X.getCXXOverloadedOperator())
-        return XOp < Y.getCXXOverloadedOperator();
-
-      // Order C++0x user-defined literal operators lexically by their
-      // lowercased suffixes.
-      if (IdentifierInfo *XLit = X.getCXXLiteralIdentifier())
-        return XLit->getName().compare_lower(
-                                  Y.getCXXLiteralIdentifier()->getName()) < 0;
-
-      // The only stable ordering we have is to turn the name into a
-      // string and then compare the lower-case strings. This is
-      // inefficient, but thankfully does not happen too often.
-      return llvm::StringRef(X.getAsString()).compare_lower(
-                                                 Y.getAsString()) < 0;
-    }
-    
-    /// \brief Retrieve the name that should be used to order a result.
-    ///
-    /// If the name needs to be constructed as a string, that string will be
-    /// saved into Saved and the returned StringRef will refer to it.
-    static llvm::StringRef getOrderedName(const Result &R,
-                                          std::string &Saved) {
-      switch (R.Kind) {
-      case Result::RK_Keyword:
-        return R.Keyword;
-          
-      case Result::RK_Pattern:
-        return R.Pattern->getTypedText();
-          
-      case Result::RK_Macro:
-        return R.Macro->getName();
-          
-      case Result::RK_Declaration:
-        // Handle declarations below.
-        break;
-      }
-            
-      DeclarationName Name = R.Declaration->getDeclName();
-      
-      // If the name is a simple identifier (by far the common case), or a
-      // zero-argument selector, just return a reference to that identifier.
-      if (IdentifierInfo *Id = Name.getAsIdentifierInfo())
-        return Id->getName();
-      if (Name.isObjCZeroArgSelector())
-        if (IdentifierInfo *Id
-                          = Name.getObjCSelector().getIdentifierInfoForSlot(0))
-          return Id->getName();
-      
-      Saved = Name.getAsString();
-      return Saved;
-    }
-    
-    bool operator()(const Result &X, const Result &Y) const {
-      std::string XSaved, YSaved;
-      llvm::StringRef XStr = getOrderedName(X, XSaved);
-      llvm::StringRef YStr = getOrderedName(Y, YSaved);
-      int cmp = XStr.compare_lower(YStr);
-      if (cmp)
-        return cmp < 0;
-      
-      // Non-hidden names precede hidden names.
-      if (X.Hidden != Y.Hidden)
-        return !X.Hidden;
+CXCursorKind clang::getCursorKindForDecl(Decl *D) {
+  if (!D)
+    return CXCursor_UnexposedDecl;
+  
+  switch (D->getKind()) {
+    case Decl::Enum:               return CXCursor_EnumDecl;
+    case Decl::EnumConstant:       return CXCursor_EnumConstantDecl;
+    case Decl::Field:              return CXCursor_FieldDecl;
+    case Decl::Function:  
+      return CXCursor_FunctionDecl;
+    case Decl::ObjCCategory:       return CXCursor_ObjCCategoryDecl;
+    case Decl::ObjCCategoryImpl:   return CXCursor_ObjCCategoryImplDecl;
+    case Decl::ObjCClass:
+      // FIXME
+      return CXCursor_UnexposedDecl;
+    case Decl::ObjCForwardProtocol:
+      // FIXME
+      return CXCursor_UnexposedDecl;      
+    case Decl::ObjCImplementation: return CXCursor_ObjCImplementationDecl;
+    case Decl::ObjCInterface:      return CXCursor_ObjCInterfaceDecl;
+    case Decl::ObjCIvar:           return CXCursor_ObjCIvarDecl; 
+    case Decl::ObjCMethod:
+      return cast<ObjCMethodDecl>(D)->isInstanceMethod()
+      ? CXCursor_ObjCInstanceMethodDecl : CXCursor_ObjCClassMethodDecl;
+    case Decl::CXXMethod:          return CXCursor_CXXMethod;
+    case Decl::CXXConstructor:     return CXCursor_Constructor;
+    case Decl::CXXDestructor:      return CXCursor_Destructor;
+    case Decl::CXXConversion:      return CXCursor_ConversionFunction;
+    case Decl::ObjCProperty:       return CXCursor_ObjCPropertyDecl;
+    case Decl::ObjCProtocol:       return CXCursor_ObjCProtocolDecl;
+    case Decl::ParmVar:            return CXCursor_ParmDecl;
+    case Decl::Typedef:            return CXCursor_TypedefDecl;
+    case Decl::Var:                return CXCursor_VarDecl;
+    case Decl::Namespace:          return CXCursor_Namespace;
+    case Decl::NamespaceAlias:     return CXCursor_NamespaceAlias;
+    case Decl::TemplateTypeParm:   return CXCursor_TemplateTypeParameter;
+    case Decl::NonTypeTemplateParm:return CXCursor_NonTypeTemplateParameter;
+    case Decl::TemplateTemplateParm:return CXCursor_TemplateTemplateParameter;
+    case Decl::FunctionTemplate:   return CXCursor_FunctionTemplate;
+    case Decl::ClassTemplate:      return CXCursor_ClassTemplate;
+    case Decl::ClassTemplatePartialSpecialization:
+      return CXCursor_ClassTemplatePartialSpecialization;
+    case Decl::UsingDirective:     return CXCursor_UsingDirective;
       
-      // Non-nested-name-specifiers precede nested-name-specifiers.
-      if (X.StartsNestedNameSpecifier != Y.StartsNestedNameSpecifier)
-        return !X.StartsNestedNameSpecifier;
+    case Decl::Using:
+    case Decl::UnresolvedUsingValue:
+    case Decl::UnresolvedUsingTypename: 
+      return CXCursor_UsingDeclaration;
       
-      return false;
-    }
-  };
+    default:
+      if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
+        switch (TD->getTagKind()) {
+          case TTK_Struct: return CXCursor_StructDecl;
+          case TTK_Class:  return CXCursor_ClassDecl;
+          case TTK_Union:  return CXCursor_UnionDecl;
+          case TTK_Enum:   return CXCursor_EnumDecl;
+        }
+      }
+  }
+  
+  return CXCursor_UnexposedDecl;
 }
 
 static void AddMacroResults(Preprocessor &PP, ResultBuilder &Results,
                             bool TargetTypeIsPointer = false) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   Results.EnterNewScope();
   for (Preprocessor::macro_iterator M = PP.macro_begin(), 
                                  MEnd = PP.macro_end();
        M != MEnd; ++M) {
-    unsigned Priority = CCP_Macro;
-    
-    // Treat the "nil" and "NULL" macros as null pointer constants.
-    if (M->first->isStr("nil") || M->first->isStr("NULL")) {
-      Priority = CCP_Constant;
-      if (TargetTypeIsPointer)
-        Priority = Priority / CCF_SimilarTypeMatch;
-    }
-      
-    Results.AddResult(Result(M->first, Priority));
+    Results.AddResult(Result(M->first, 
+                             getMacroUsagePriority(M->first->getName(),
+                                                   TargetTypeIsPointer)));
   }
   Results.ExitScope();
 }
 
+static void AddPrettyFunctionResults(const LangOptions &LangOpts, 
+                                     ResultBuilder &Results) {
+  typedef CodeCompletionResult Result;
+  
+  Results.EnterNewScope();
+  Results.AddResult(Result("__PRETTY_FUNCTION__", CCP_Constant));
+  Results.AddResult(Result("__FUNCTION__", CCP_Constant));
+  if (LangOpts.C99 || LangOpts.CPlusPlus0x)
+    Results.AddResult(Result("__func__", CCP_Constant));
+  Results.ExitScope();
+}
+
 static void HandleCodeCompleteResults(Sema *S,
                                       CodeCompleteConsumer *CodeCompleter,
-                                     CodeCompleteConsumer::Result *Results,
-                                     unsigned NumResults) {
-  std::stable_sort(Results, Results + NumResults, SortCodeCompleteResult());
-
+                                      CodeCompletionContext Context,
+                                      CodeCompletionResult *Results,
+                                      unsigned NumResults) {
   if (CodeCompleter)
-    CodeCompleter->ProcessCodeCompleteResults(*S, Results, NumResults);
+    CodeCompleter->ProcessCodeCompleteResults(*S, Context, Results, NumResults);
   
   for (unsigned I = 0; I != NumResults; ++I)
     Results[I].Destroy();
 }
 
+static enum CodeCompletionContext::Kind mapCodeCompletionContext(Sema &S, 
+                                            Sema::ParserCompletionContext PCC) {
+  switch (PCC) {
+  case Sema::PCC_Namespace:
+    return CodeCompletionContext::CCC_TopLevel;
+      
+  case Sema::PCC_Class:
+    return CodeCompletionContext::CCC_ClassStructUnion;
+
+  case Sema::PCC_ObjCInterface:
+    return CodeCompletionContext::CCC_ObjCInterface;
+      
+  case Sema::PCC_ObjCImplementation:
+    return CodeCompletionContext::CCC_ObjCImplementation;
+
+  case Sema::PCC_ObjCInstanceVariableList:
+    return CodeCompletionContext::CCC_ObjCIvarList;
+      
+  case Sema::PCC_Template:
+  case Sema::PCC_MemberTemplate:
+  case Sema::PCC_RecoveryInFunction:
+    return CodeCompletionContext::CCC_Other;
+      
+  case Sema::PCC_Expression:
+  case Sema::PCC_ForInit:
+  case Sema::PCC_Condition:
+    return CodeCompletionContext::CCC_Expression;
+      
+  case Sema::PCC_Statement:
+    return CodeCompletionContext::CCC_Statement;
+
+  case Sema::PCC_Type:
+    return CodeCompletionContext::CCC_Type;
+  }
+  
+  return CodeCompletionContext::CCC_Other;
+}
+
+/// \brief If we're in a C++ virtual member function, add completion results
+/// that invoke the functions we override, since it's common to invoke the 
+/// overridden function as well as adding new functionality.
+///
+/// \param S The semantic analysis object for which we are generating results.
+///
+/// \param InContext This context in which the nested-name-specifier preceding
+/// the code-completion point 
+static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
+                                  ResultBuilder &Results) {
+  // Look through blocks.
+  DeclContext *CurContext = S.CurContext;
+  while (isa<BlockDecl>(CurContext))
+    CurContext = CurContext->getParent();
+  
+  
+  CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(CurContext);
+  if (!Method || !Method->isVirtual())
+    return;
+  
+  // We need to have names for all of the parameters, if we're going to 
+  // generate a forwarding call.
+  for (CXXMethodDecl::param_iterator P = Method->param_begin(),
+                                  PEnd = Method->param_end();
+       P != PEnd;
+       ++P) {
+    if (!(*P)->getDeclName())
+      return;
+  }
+
+  for (CXXMethodDecl::method_iterator M = Method->begin_overridden_methods(),
+                                   MEnd = Method->end_overridden_methods();
+       M != MEnd; ++M) {
+    CodeCompletionString *Pattern = new CodeCompletionString;
+    CXXMethodDecl *Overridden = const_cast<CXXMethodDecl *>(*M);
+    if (Overridden->getCanonicalDecl() == Method->getCanonicalDecl())
+      continue;
+        
+    // If we need a nested-name-specifier, add one now.
+    if (!InContext) {
+      NestedNameSpecifier *NNS
+        = getRequiredQualification(S.Context, CurContext,
+                                   Overridden->getDeclContext());
+      if (NNS) {
+        std::string Str;
+        llvm::raw_string_ostream OS(Str);
+        NNS->print(OS, S.Context.PrintingPolicy);
+        Pattern->AddTextChunk(OS.str());
+      }
+    } else if (!InContext->Equals(Overridden->getDeclContext()))
+      continue;
+    
+    Pattern->AddTypedTextChunk(Overridden->getNameAsString());
+    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+    bool FirstParam = true;
+    for (CXXMethodDecl::param_iterator P = Method->param_begin(),
+                                    PEnd = Method->param_end();
+         P != PEnd; ++P) {
+      if (FirstParam)
+        FirstParam = false;
+      else
+        Pattern->AddChunk(CodeCompletionString::CK_Comma);
+
+      Pattern->AddPlaceholderChunk((*P)->getIdentifier()->getName());
+    }
+    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Pattern,
+                                           CCP_SuperCompletion,
+                                           CXCursor_CXXMethod));
+    Results.Ignore(Overridden);
+  }
+}
+
 void Sema::CodeCompleteOrdinaryName(Scope *S, 
-                                    CodeCompletionContext CompletionContext) {
-  typedef CodeCompleteConsumer::Result Result;
-  ResultBuilder Results(*this);
+                                    ParserCompletionContext CompletionContext) {
+  typedef CodeCompletionResult Result;
+  ResultBuilder Results(*this);  
+  Results.EnterNewScope();
 
   // Determine how to filter results, e.g., so that the names of
   // values (functions, enumerators, function templates, etc.) are
   // only allowed where we can have an expression.
   switch (CompletionContext) {
-  case CCC_Namespace:
-  case CCC_Class:
-  case CCC_ObjCInterface:
-  case CCC_ObjCImplementation:
-  case CCC_ObjCInstanceVariableList:
-  case CCC_Template:
-  case CCC_MemberTemplate:
+  case PCC_Namespace:
+  case PCC_Class:
+  case PCC_ObjCInterface:
+  case PCC_ObjCImplementation:
+  case PCC_ObjCInstanceVariableList:
+  case PCC_Template:
+  case PCC_MemberTemplate:
+  case PCC_Type:
     Results.setFilter(&ResultBuilder::IsOrdinaryNonValueName);
     break;
 
-  case CCC_Expression:
-  case CCC_Statement:
-  case CCC_ForInit:
-  case CCC_Condition:
+  case PCC_Statement:
+    // For statements that are expressions, we prefer to call 'void' functions 
+    // rather than functions that return a result, since then the result would
+    // be ignored.
+    Results.setPreferredType(Context.VoidTy);
+    // Fall through
+      
+  case PCC_Expression:
+  case PCC_ForInit:
+  case PCC_Condition:
     if (WantTypesInContext(CompletionContext, getLangOptions()))
       Results.setFilter(&ResultBuilder::IsOrdinaryName);
     else
       Results.setFilter(&ResultBuilder::IsOrdinaryNonTypeName);
+      
+    if (getLangOptions().CPlusPlus)
+      MaybeAddOverrideCalls(*this, /*InContext=*/0, Results);
     break;
       
-  case CCC_RecoveryInFunction:
+  case PCC_RecoveryInFunction:
     // Unfiltered
     break;
   }
 
+  // If we are in a C++ non-static member function, check the qualifiers on
+  // the member function to filter/prioritize the results list.
+  if (CXXMethodDecl *CurMethod = dyn_cast<CXXMethodDecl>(CurContext))
+    if (CurMethod->isInstance())
+      Results.setObjectTypeQualifiers(
+                      Qualifiers::fromCVRMask(CurMethod->getTypeQualifiers()));
+  
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
 
-  Results.EnterNewScope();
   AddOrdinaryNameResults(CompletionContext, S, *this, Results);
   Results.ExitScope();
 
+  switch (CompletionContext) {
+  case PCC_Expression:
+  case PCC_Statement:
+  case PCC_RecoveryInFunction:
+    if (S->getFnParent())
+      AddPrettyFunctionResults(PP.getLangOptions(), Results);        
+    break;
+    
+  case PCC_Namespace:
+  case PCC_Class:
+  case PCC_ObjCInterface:
+  case PCC_ObjCImplementation:
+  case PCC_ObjCInstanceVariableList:
+  case PCC_Template:
+  case PCC_MemberTemplate:
+  case PCC_ForInit:
+  case PCC_Condition:
+  case PCC_Type:
+    break;
+  }
+  
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  
+  HandleCodeCompleteResults(this, CodeCompleter,
+                            mapCodeCompletionContext(*this, CompletionContext),
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompleteDeclarator(Scope *S,
+                                  bool AllowNonIdentifiers,
+                                  bool AllowNestedNameSpecifiers) {
+  typedef CodeCompletionResult Result;
+  ResultBuilder Results(*this);    
+  Results.EnterNewScope();
+  
+  // Type qualifiers can come after names.
+  Results.AddResult(Result("const"));
+  Results.AddResult(Result("volatile"));
+  if (getLangOptions().C99)
+    Results.AddResult(Result("restrict"));
+
+  if (getLangOptions().CPlusPlus) {
+    if (AllowNonIdentifiers) {
+      Results.AddResult(Result("operator")); 
+    }
+    
+    // Add nested-name-specifiers.
+    if (AllowNestedNameSpecifiers) {
+      Results.allowNestedNameSpecifiers();
+      CodeCompletionDeclConsumer Consumer(Results, CurContext);
+      LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer,
+                         CodeCompleter->includeGlobals());
+    }
+  }
+  Results.ExitScope();
+
+  // Note that we intentionally suppress macro results here, since we do not
+  // encourage using macros to produce the names of entities.
+
+  HandleCodeCompleteResults(this, CodeCompleter,
+                        AllowNestedNameSpecifiers
+                          ? CodeCompletionContext::CCC_PotentiallyQualifiedName
+                          : CodeCompletionContext::CCC_Name,
+                            Results.data(), Results.size());
 }
 
+struct Sema::CodeCompleteExpressionData {
+  CodeCompleteExpressionData(QualType PreferredType = QualType()) 
+    : PreferredType(PreferredType), IntegralConstantExpression(false),
+      ObjCCollection(false) { }
+  
+  QualType PreferredType;
+  bool IntegralConstantExpression;
+  bool ObjCCollection;
+  llvm::SmallVector<Decl *, 4> IgnoreDecls;
+};
+
 /// \brief Perform code-completion in an expression context when we know what
 /// type we're looking for.
-void Sema::CodeCompleteExpression(Scope *S, QualType T) {
-  typedef CodeCompleteConsumer::Result Result;
+///
+/// \param IntegralConstantExpression Only permit integral constant 
+/// expressions.
+void Sema::CodeCompleteExpression(Scope *S, 
+                                  const CodeCompleteExpressionData &Data) {
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
   
-  if (WantTypesInContext(CCC_Expression, getLangOptions()))
+  if (Data.ObjCCollection)
+    Results.setFilter(&ResultBuilder::IsObjCCollection);
+  else if (Data.IntegralConstantExpression)
+    Results.setFilter(&ResultBuilder::IsIntegralConstantValue);
+  else if (WantTypesInContext(PCC_Expression, getLangOptions()))
     Results.setFilter(&ResultBuilder::IsOrdinaryName);
   else
     Results.setFilter(&ResultBuilder::IsOrdinaryNonTypeName);
-  Results.setPreferredType(T.getNonReferenceType());
+
+  if (!Data.PreferredType.isNull())
+    Results.setPreferredType(Data.PreferredType.getNonReferenceType());
+  
+  // Ignore any declarations that we were told that we don't care about.
+  for (unsigned I = 0, N = Data.IgnoreDecls.size(); I != N; ++I)
+    Results.Ignore(Data.IgnoreDecls[I]);
   
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
   
   Results.EnterNewScope();
-  AddOrdinaryNameResults(CCC_Expression, S, *this, Results);
+  AddOrdinaryNameResults(PCC_Expression, S, *this, Results);
   Results.ExitScope();
   
   bool PreferredTypeIsPointer = false;
-  if (!T.isNull())
-    PreferredTypeIsPointer = T->isAnyPointerType() || 
-      T->isMemberPointerType() || T->isBlockPointerType();
+  if (!Data.PreferredType.isNull())
+    PreferredTypeIsPointer = Data.PreferredType->isAnyPointerType()
+      || Data.PreferredType->isMemberPointerType() 
+      || Data.PreferredType->isBlockPointerType();
   
+  if (S->getFnParent() && 
+      !Data.ObjCCollection && 
+      !Data.IntegralConstantExpression)
+    AddPrettyFunctionResults(PP.getLangOptions(), Results);        
+
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results, PreferredTypeIsPointer);
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                CodeCompletionContext(CodeCompletionContext::CCC_Expression, 
+                                      Data.PreferredType),
+                            Results.data(),Results.size());
 }
 
 
@@ -2303,7 +2725,7 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
                               bool AllowCategories,
                               DeclContext *CurContext,
                               ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
 
   // Add properties in this container.
   for (ObjCContainerDecl::prop_iterator P = Container->prop_begin(),
@@ -2327,9 +2749,9 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
     }
     
     // Look through protocols.
-    for (ObjCInterfaceDecl::protocol_iterator I = IFace->protocol_begin(),
-                                              E = IFace->protocol_end(); 
-         I != E; ++I)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+         I = IFace->all_referenced_protocol_begin(),
+         E = IFace->all_referenced_protocol_end(); I != E; ++I)
       AddObjCProperties(*I, AllowCategories, CurContext, Results);
     
     // Look in the superclass.
@@ -2339,8 +2761,8 @@ static void AddObjCProperties(ObjCContainerDecl *Container,
   } else if (const ObjCCategoryDecl *Category
                                     = dyn_cast<ObjCCategoryDecl>(Container)) {
     // Look through protocols.
-    for (ObjCInterfaceDecl::protocol_iterator P = Category->protocol_begin(),
-                                           PEnd = Category->protocol_end(); 
+    for (ObjCCategoryDecl::protocol_iterator P = Category->protocol_begin(),
+                                          PEnd = Category->protocol_end(); 
          P != PEnd; ++P)
       AddObjCProperties(*P, AllowCategories, CurContext, Results);
   }
@@ -2352,7 +2774,7 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
   if (!BaseE || !CodeCompleter)
     return;
   
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   Expr *Base = static_cast<Expr *>(BaseE);
   QualType BaseType = Base->getType();
@@ -2361,7 +2783,7 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
     if (const PointerType *Ptr = BaseType->getAs<PointerType>())
       BaseType = Ptr->getPointeeType();
     else if (BaseType->isObjCObjectPointerType())
-    /*Do nothing*/ ;
+      /*Do nothing*/ ;
     else
       return;
   }
@@ -2369,10 +2791,15 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
   ResultBuilder Results(*this, &ResultBuilder::IsMember);
   Results.EnterNewScope();
   if (const RecordType *Record = BaseType->getAs<RecordType>()) {
+    // Indicate that we are performing a member access, and the cv-qualifiers
+    // for the base object type.
+    Results.setObjectTypeQualifiers(BaseType.getQualifiers());
+    
     // Access to a C/C++ class, struct, or union.
     Results.allowNestedNameSpecifiers();
     CodeCompletionDeclConsumer Consumer(Results, CurContext);
-    LookupVisibleDecls(Record->getDecl(), LookupMemberName, Consumer);
+    LookupVisibleDecls(Record->getDecl(), LookupMemberName, Consumer,
+                       CodeCompleter->includeGlobals());
 
     if (getLangOptions().CPlusPlus) {
       if (!Results.empty()) {
@@ -2420,7 +2847,8 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
     if (Class) {
       CodeCompletionDeclConsumer Consumer(Results, CurContext);
       Results.setFilter(&ResultBuilder::IsObjCIvar);
-      LookupVisibleDecls(Class, LookupMemberName, Consumer);
+      LookupVisibleDecls(Class, LookupMemberName, Consumer,
+                         CodeCompleter->includeGlobals());
     }
   }
   
@@ -2429,27 +2857,35 @@ void Sema::CodeCompleteMemberReferenceExpr(Scope *S, ExprTy *BaseE,
   Results.ExitScope();
 
   // Hand off the results found for code completion.
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+            CodeCompletionContext(CodeCompletionContext::CCC_MemberAccess,
+                                              BaseType),
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
   if (!CodeCompleter)
     return;
   
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder::LookupFilter Filter = 0;
+  enum CodeCompletionContext::Kind ContextKind
+    = CodeCompletionContext::CCC_Other;
   switch ((DeclSpec::TST)TagSpec) {
   case DeclSpec::TST_enum:
     Filter = &ResultBuilder::IsEnum;
+    ContextKind = CodeCompletionContext::CCC_EnumTag;
     break;
     
   case DeclSpec::TST_union:
     Filter = &ResultBuilder::IsUnion;
+    ContextKind = CodeCompletionContext::CCC_UnionTag;
     break;
     
   case DeclSpec::TST_struct:
   case DeclSpec::TST_class:
     Filter = &ResultBuilder::IsClassOrStruct;
+    ContextKind = CodeCompletionContext::CCC_ClassOrStructTag;
     break;
     
   default:
@@ -2462,22 +2898,46 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
 
   // First pass: look for tags.
   Results.setFilter(Filter);
-  LookupVisibleDecls(S, LookupTagName, Consumer);
+  LookupVisibleDecls(S, LookupTagName, Consumer,
+                     CodeCompleter->includeGlobals());
 
-  // Second pass: look for nested name specifiers.
-  Results.setFilter(&ResultBuilder::IsNestedNameSpecifier);
-  LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer);
+  if (CodeCompleter->includeGlobals()) {
+    // Second pass: look for nested name specifiers.
+    Results.setFilter(&ResultBuilder::IsNestedNameSpecifier);
+    LookupVisibleDecls(S, LookupNestedNameSpecifierName, Consumer);
+  }
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, ContextKind,
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompleteTypeQualifiers(DeclSpec &DS) {
+  ResultBuilder Results(*this);
+  Results.EnterNewScope();
+  if (!(DS.getTypeQualifiers() & DeclSpec::TQ_const))
+    Results.AddResult("const");
+  if (!(DS.getTypeQualifiers() & DeclSpec::TQ_volatile))
+    Results.AddResult("volatile");
+  if (getLangOptions().C99 &&
+      !(DS.getTypeQualifiers() & DeclSpec::TQ_restrict))
+    Results.AddResult("restrict");
+  Results.ExitScope();
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_TypeQualifiers,
+                            Results.data(), Results.size());
 }
 
 void Sema::CodeCompleteCase(Scope *S) {
-  if (getSwitchStack().empty() || !CodeCompleter)
+  if (getCurFunction()->SwitchStack.empty() || !CodeCompleter)
     return;
   
-  SwitchStmt *Switch = getSwitchStack().back();
-  if (!Switch->getCond()->getType()->isEnumeralType())
+  SwitchStmt *Switch = getCurFunction()->SwitchStack.back();
+  if (!Switch->getCond()->getType()->isEnumeralType()) {
+    CodeCompleteExpressionData Data(Switch->getCond()->getType());
+    Data.IntegralConstantExpression = true;
+    CodeCompleteExpression(S, Data);
     return;
+  }
   
   // Code-complete the cases of a switch statement over an enumeration type
   // by providing the list of 
@@ -2541,14 +3001,16 @@ void Sema::CodeCompleteCase(Scope *S) {
     if (EnumeratorsSeen.count(*E))
       continue;
     
-    Results.AddResult(CodeCompleteConsumer::Result(*E, Qualifier),
+    Results.AddResult(CodeCompletionResult(*E, Qualifier),
                       CurContext, 0, false);
   }
   Results.ExitScope();
 
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Expression,
+                            Results.data(),Results.size());
 }
 
 namespace {
@@ -2562,7 +3024,7 @@ namespace {
     
     bool 
     operator()(const OverloadCandidate &X, const OverloadCandidate &Y) const {
-      return S.isBetterOverloadCandidate(X, Y, Loc);
+      return isBetterOverloadCandidate(S, X, Y, Loc);
     }
   };
 }
@@ -2594,7 +3056,7 @@ void Sema::CodeCompleteCall(Scope *S, ExprTy *FnIn,
   // Ignore type-dependent call expressions entirely.
   if (!Fn || Fn->isTypeDependent() || anyNullArguments(Args, NumArgs) ||
       Expr::hasAnyTypeDependentArguments(Args, NumArgs)) {
-    CodeCompleteOrdinaryName(S, CCC_Expression);
+    CodeCompleteOrdinaryName(S, PCC_Expression);
     return;
   }
 
@@ -2678,7 +3140,7 @@ void Sema::CodeCompleteCall(Scope *S, ExprTy *FnIn,
   }
 
   if (ParamType.isNull())
-    CodeCompleteOrdinaryName(S, CCC_Expression);
+    CodeCompleteOrdinaryName(S, PCC_Expression);
   else
     CodeCompleteExpression(S, ParamType);
   
@@ -2687,10 +3149,10 @@ void Sema::CodeCompleteCall(Scope *S, ExprTy *FnIn,
                                              Results.size());
 }
 
-void Sema::CodeCompleteInitializer(Scope *S, DeclPtrTy D) {
-  ValueDecl *VD = dyn_cast_or_null<ValueDecl>(D.getAs<Decl>());
+void Sema::CodeCompleteInitializer(Scope *S, Decl *D) {
+  ValueDecl *VD = dyn_cast_or_null<ValueDecl>(D);
   if (!VD) {
-    CodeCompleteOrdinaryName(S, CCC_Expression);
+    CodeCompleteOrdinaryName(S, PCC_Expression);
     return;
   }
   
@@ -2708,7 +3170,7 @@ void Sema::CodeCompleteReturn(Scope *S) {
     ResultType = Method->getResultType();
   
   if (ResultType.isNull())
-    CodeCompleteOrdinaryName(S, CCC_Expression);
+    CodeCompleteOrdinaryName(S, PCC_Expression);
   else
     CodeCompleteExpression(S, ResultType);
 }
@@ -2717,7 +3179,7 @@ void Sema::CodeCompleteAssignmentRHS(Scope *S, ExprTy *LHS) {
   if (LHS)
     CodeCompleteExpression(S, static_cast<Expr *>(LHS)->getType());
   else
-    CodeCompleteOrdinaryName(S, CCC_Expression);
+    CodeCompleteOrdinaryName(S, PCC_Expression);
 }
 
 void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
@@ -2735,16 +3197,29 @@ void Sema::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
     return;
 
   ResultBuilder Results(*this);
-  CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(Ctx, LookupOrdinaryName, Consumer);
   
+  Results.EnterNewScope();
   // The "template" keyword can follow "::" in the grammar, but only
   // put it into the grammar if the nested-name-specifier is dependent.
   NestedNameSpecifier *NNS = (NestedNameSpecifier *)SS.getScopeRep();
   if (!Results.empty() && NNS->isDependent())
     Results.AddResult("template");
+
+  // Add calls to overridden virtual functions, if there are any.
+  //
+  // FIXME: This isn't wonderful, because we don't know whether we're actually
+  // in a context that permits expressions. This is a general issue with
+  // qualified-id completions.
+  if (!EnteringContext)
+    MaybeAddOverrideCalls(*this, Ctx, Results);
+  Results.ExitScope();  
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  CodeCompletionDeclConsumer Consumer(Results, CurContext);
+  LookupVisibleDecls(Ctx, LookupOrdinaryName, Consumer);
+
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Name,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteUsing(Scope *S) {
@@ -2756,15 +3231,18 @@ void Sema::CodeCompleteUsing(Scope *S) {
   
   // If we aren't in class scope, we could see the "namespace" keyword.
   if (!S->isClassScope())
-    Results.AddResult(CodeCompleteConsumer::Result("namespace"));
+    Results.AddResult(CodeCompletionResult("namespace"));
   
   // After "using", we can see anything that would start a 
   // nested-name-specifier.
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteUsingDirective(Scope *S) {
@@ -2776,9 +3254,12 @@ void Sema::CodeCompleteUsingDirective(Scope *S) {
   ResultBuilder Results(*this, &ResultBuilder::IsNamespaceOrAlias);
   Results.EnterNewScope();
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Namespace,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteNamespaceDecl(Scope *S)  {
@@ -2807,12 +3288,14 @@ void Sema::CodeCompleteNamespaceDecl(Scope *S)  {
     for (std::map<NamespaceDecl *, NamespaceDecl *>::iterator 
          NS = OrigToLatest.begin(), NSEnd = OrigToLatest.end();
          NS != NSEnd; ++NS)
-      Results.AddResult(CodeCompleteConsumer::Result(NS->second, 0),
+      Results.AddResult(CodeCompletionResult(NS->second, 0),
                         CurContext, 0, false);
     Results.ExitScope();
   }
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteNamespaceAliasDecl(Scope *S)  {
@@ -2822,15 +3305,18 @@ void Sema::CodeCompleteNamespaceAliasDecl(Scope *S)  {
   // After "namespace", we expect to see a namespace or alias.
   ResultBuilder Results(*this, &ResultBuilder::IsNamespaceOrAlias);
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Namespace,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteOperatorName(Scope *S) {
   if (!CodeCompleter)
     return;
 
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, &ResultBuilder::IsType);
   Results.EnterNewScope();
   
@@ -2843,13 +3329,122 @@ void Sema::CodeCompleteOperatorName(Scope *S) {
   // Add any type names visible from the current scope
   Results.allowNestedNameSpecifiers();
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
   
   // Add any type specifiers
   AddTypeSpecifierResults(getLangOptions(), Results);
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Type,
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompleteConstructorInitializer(Decl *ConstructorD,
+                                    CXXBaseOrMemberInitializer** Initializers,
+                                              unsigned NumInitializers) {
+  CXXConstructorDecl *Constructor
+    = static_cast<CXXConstructorDecl *>(ConstructorD);
+  if (!Constructor)
+    return;
+  
+  ResultBuilder Results(*this);
+  Results.EnterNewScope();
+  
+  // Fill in any already-initialized fields or base classes.
+  llvm::SmallPtrSet<FieldDecl *, 4> InitializedFields;
+  llvm::SmallPtrSet<CanQualType, 4> InitializedBases;
+  for (unsigned I = 0; I != NumInitializers; ++I) {
+    if (Initializers[I]->isBaseInitializer())
+      InitializedBases.insert(
+        Context.getCanonicalType(QualType(Initializers[I]->getBaseClass(), 0)));
+    else
+      InitializedFields.insert(cast<FieldDecl>(Initializers[I]->getMember()));
+  }
+  
+  // Add completions for base classes.
+  bool SawLastInitializer = (NumInitializers == 0);
+  CXXRecordDecl *ClassDecl = Constructor->getParent();
+  for (CXXRecordDecl::base_class_iterator Base = ClassDecl->bases_begin(),
+                                       BaseEnd = ClassDecl->bases_end();
+       Base != BaseEnd; ++Base) {
+    if (!InitializedBases.insert(Context.getCanonicalType(Base->getType()))) {
+      SawLastInitializer
+        = NumInitializers > 0 && 
+          Initializers[NumInitializers - 1]->isBaseInitializer() &&
+          Context.hasSameUnqualifiedType(Base->getType(),
+               QualType(Initializers[NumInitializers - 1]->getBaseClass(), 0));
+      continue;
+    }
+    
+    CodeCompletionString *Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk(
+                           Base->getType().getAsString(Context.PrintingPolicy));
+    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+    Pattern->AddPlaceholderChunk("args");
+    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Pattern, 
+                                   SawLastInitializer? CCP_NextInitializer
+                                                     : CCP_MemberDeclaration));
+    SawLastInitializer = false;
+  }
+  
+  // Add completions for virtual base classes.
+  for (CXXRecordDecl::base_class_iterator Base = ClassDecl->vbases_begin(),
+                                       BaseEnd = ClassDecl->vbases_end();
+       Base != BaseEnd; ++Base) {
+    if (!InitializedBases.insert(Context.getCanonicalType(Base->getType()))) {
+      SawLastInitializer
+        = NumInitializers > 0 && 
+          Initializers[NumInitializers - 1]->isBaseInitializer() &&
+          Context.hasSameUnqualifiedType(Base->getType(),
+               QualType(Initializers[NumInitializers - 1]->getBaseClass(), 0));
+      continue;
+    }
+    
+    CodeCompletionString *Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk(
+                           Base->getType().getAsString(Context.PrintingPolicy));
+    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+    Pattern->AddPlaceholderChunk("args");
+    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Pattern, 
+                                   SawLastInitializer? CCP_NextInitializer
+                                                     : CCP_MemberDeclaration));
+    SawLastInitializer = false;
+  }
+  
+  // Add completions for members.
+  for (CXXRecordDecl::field_iterator Field = ClassDecl->field_begin(),
+                                  FieldEnd = ClassDecl->field_end();
+       Field != FieldEnd; ++Field) {
+    if (!InitializedFields.insert(cast<FieldDecl>(Field->getCanonicalDecl()))) {
+      SawLastInitializer
+        = NumInitializers > 0 && 
+          Initializers[NumInitializers - 1]->isMemberInitializer() &&
+          Initializers[NumInitializers - 1]->getMember() == *Field;
+      continue;
+    }
+    
+    if (!Field->getDeclName())
+      continue;
+    
+    CodeCompletionString *Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk(Field->getIdentifier()->getName());
+    Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+    Pattern->AddPlaceholderChunk("args");
+    Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+    Results.AddResult(CodeCompletionResult(Pattern, 
+                                   SawLastInitializer? CCP_NextInitializer
+                                                     : CCP_MemberDeclaration));
+    SawLastInitializer = false;
+  }
+  Results.ExitScope();
+  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Name,
+                            Results.data(), Results.size());
 }
 
 // Macro that expands to @Keyword or Keyword, depending on whether NeedAt is
@@ -2858,7 +3453,7 @@ void Sema::CodeCompleteOperatorName(Scope *S) {
 static void AddObjCImplementationResults(const LangOptions &LangOpts,
                                          ResultBuilder &Results,
                                          bool NeedAt) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   // Since we have an implementation, we can end it.
   Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt,end)));
   
@@ -2883,7 +3478,7 @@ static void AddObjCImplementationResults(const LangOptions &LangOpts,
 static void AddObjCInterfaceResults(const LangOptions &LangOpts,
                                     ResultBuilder &Results,
                                     bool NeedAt) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   // Since we have an interface or protocol, we can end it.
   Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt,end)));
@@ -2901,7 +3496,7 @@ static void AddObjCInterfaceResults(const LangOptions &LangOpts,
 }
 
 static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   CodeCompletionString *Pattern = 0;
   
   // @class name ;
@@ -2946,9 +3541,9 @@ static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt) {
   Results.AddResult(Result(Pattern));
 }
 
-void Sema::CodeCompleteObjCAtDirective(Scope *S, DeclPtrTy ObjCImpDecl,
+void Sema::CodeCompleteObjCAtDirective(Scope *S, Decl *ObjCImpDecl,
                                        bool InInterface) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
   Results.EnterNewScope();
   if (ObjCImpDecl)
@@ -2958,11 +3553,13 @@ void Sema::CodeCompleteObjCAtDirective(Scope *S, DeclPtrTy ObjCImpDecl,
   else
     AddObjCTopLevelResults(Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 static void AddObjCExpressionResults(ResultBuilder &Results, bool NeedAt) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   CodeCompletionString *Pattern = 0;
 
   // @encode ( type-name )
@@ -2991,7 +3588,7 @@ static void AddObjCExpressionResults(ResultBuilder &Results, bool NeedAt) {
 }
 
 static void AddObjCStatementResults(ResultBuilder &Results, bool NeedAt) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   CodeCompletionString *Pattern = 0;
   
   if (Results.includeCodePatterns()) {
@@ -3041,7 +3638,7 @@ static void AddObjCStatementResults(ResultBuilder &Results, bool NeedAt) {
 static void AddObjCVisibilityResults(const LangOptions &LangOpts,
                                      ResultBuilder &Results,
                                      bool NeedAt) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt,private)));
   Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt,protected)));
   Results.AddResult(Result(OBJC_AT_KEYWORD_NAME(NeedAt,public)));
@@ -3054,7 +3651,9 @@ void Sema::CodeCompleteObjCAtVisibility(Scope *S) {
   Results.EnterNewScope();
   AddObjCVisibilityResults(getLangOptions(), Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCAtStatement(Scope *S) {
@@ -3063,7 +3662,9 @@ void Sema::CodeCompleteObjCAtStatement(Scope *S) {
   AddObjCStatementResults(Results, false);
   AddObjCExpressionResults(Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCAtExpression(Scope *S) {
@@ -3071,7 +3672,9 @@ void Sema::CodeCompleteObjCAtExpression(Scope *S) {
   Results.EnterNewScope();
   AddObjCExpressionResults(Results, false);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 /// \brief Determine whether the addition of the given flag to an Objective-C
@@ -3110,37 +3713,39 @@ void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
   
   unsigned Attributes = ODS.getPropertyAttributes();
   
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
   Results.EnterNewScope();
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_readonly))
-    Results.AddResult(CodeCompleteConsumer::Result("readonly"));
+    Results.AddResult(CodeCompletionResult("readonly"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_assign))
-    Results.AddResult(CodeCompleteConsumer::Result("assign"));
+    Results.AddResult(CodeCompletionResult("assign"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_readwrite))
-    Results.AddResult(CodeCompleteConsumer::Result("readwrite"));
+    Results.AddResult(CodeCompletionResult("readwrite"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_retain))
-    Results.AddResult(CodeCompleteConsumer::Result("retain"));
+    Results.AddResult(CodeCompletionResult("retain"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_copy))
-    Results.AddResult(CodeCompleteConsumer::Result("copy"));
+    Results.AddResult(CodeCompletionResult("copy"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_nonatomic))
-    Results.AddResult(CodeCompleteConsumer::Result("nonatomic"));
+    Results.AddResult(CodeCompletionResult("nonatomic"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_setter)) {
     CodeCompletionString *Setter = new CodeCompletionString;
     Setter->AddTypedTextChunk("setter");
     Setter->AddTextChunk(" = ");
     Setter->AddPlaceholderChunk("method");
-    Results.AddResult(CodeCompleteConsumer::Result(Setter));
+    Results.AddResult(CodeCompletionResult(Setter));
   }
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_getter)) {
     CodeCompletionString *Getter = new CodeCompletionString;
     Getter->AddTypedTextChunk("getter");
     Getter->AddTextChunk(" = ");
     Getter->AddPlaceholderChunk("method");
-    Results.AddResult(CodeCompleteConsumer::Result(Getter));
+    Results.AddResult(CodeCompletionResult(Getter));
   }
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 /// \brief Descripts the kind of Objective-C method that we want to find
@@ -3151,26 +3756,33 @@ enum ObjCMethodKind {
   MK_OneArgSelector //< One-argument selector.
 };
 
-static bool isAcceptableObjCMethod(ObjCMethodDecl *Method,
-                                   ObjCMethodKind WantKind,
-                                   IdentifierInfo **SelIdents,
-                                   unsigned NumSelIdents) {
-  Selector Sel = Method->getSelector();
+static bool isAcceptableObjCSelector(Selector Sel,
+                                     ObjCMethodKind WantKind,
+                                     IdentifierInfo **SelIdents,
+                                     unsigned NumSelIdents) {
   if (NumSelIdents > Sel.getNumArgs())
     return false;
-      
+  
   switch (WantKind) {
-  case MK_Any:             break;
-  case MK_ZeroArgSelector: return Sel.isUnarySelector();
-  case MK_OneArgSelector:  return Sel.getNumArgs() == 1;
+    case MK_Any:             break;
+    case MK_ZeroArgSelector: return Sel.isUnarySelector();
+    case MK_OneArgSelector:  return Sel.getNumArgs() == 1;
   }
-
+  
   for (unsigned I = 0; I != NumSelIdents; ++I)
     if (SelIdents[I] != Sel.getIdentifierInfoForSlot(I))
       return false;
-
+  
   return true;
 }
+
+static bool isAcceptableObjCMethod(ObjCMethodDecl *Method,
+                                   ObjCMethodKind WantKind,
+                                   IdentifierInfo **SelIdents,
+                                   unsigned NumSelIdents) {
+  return isAcceptableObjCSelector(Method->getSelector(), WantKind, SelIdents,
+                                  NumSelIdents);
+}
                                    
 /// \brief Add all of the Objective-C methods in the given Objective-C 
 /// container to the set of results.
@@ -3195,8 +3807,9 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                            IdentifierInfo **SelIdents,
                            unsigned NumSelIdents,
                            DeclContext *CurContext,
-                           ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+                           ResultBuilder &Results,
+                           bool InOriginalClass = true) {
+  typedef CodeCompletionResult Result;
   for (ObjCContainerDecl::method_iterator M = Container->meth_begin(),
                                        MEnd = Container->meth_end();
        M != MEnd; ++M) {
@@ -3209,6 +3822,8 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
       Result R = Result(*M, 0);
       R.StartParameter = NumSelIdents;
       R.AllParametersAreInformative = (WantKind != MK_Any);
+      if (!InOriginalClass)
+        R.Priority += CCD_InBaseClass;
       Results.MaybeAddResult(R, CurContext);
     }
   }
@@ -3223,13 +3838,13 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                                             E = Protocols.end(); 
        I != E; ++I)
     AddObjCMethods(*I, WantInstanceMethods, WantKind, SelIdents, NumSelIdents, 
-                   CurContext, Results);
+                   CurContext, Results, false);
   
   // Add methods in categories.
   for (ObjCCategoryDecl *CatDecl = IFace->getCategoryList(); CatDecl;
        CatDecl = CatDecl->getNextClassCategory()) {
     AddObjCMethods(CatDecl, WantInstanceMethods, WantKind, SelIdents, 
-                   NumSelIdents, CurContext, Results);
+                   NumSelIdents, CurContext, Results, InOriginalClass);
     
     // Add a categories protocol methods.
     const ObjCList<ObjCProtocolDecl> &Protocols 
@@ -3238,37 +3853,36 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
                                               E = Protocols.end();
          I != E; ++I)
       AddObjCMethods(*I, WantInstanceMethods, WantKind, SelIdents, 
-                     NumSelIdents, CurContext, Results);
+                     NumSelIdents, CurContext, Results, false);
     
     // Add methods in category implementations.
     if (ObjCCategoryImplDecl *Impl = CatDecl->getImplementation())
       AddObjCMethods(Impl, WantInstanceMethods, WantKind, SelIdents, 
-                     NumSelIdents, CurContext, Results);
+                     NumSelIdents, CurContext, Results, InOriginalClass);
   }
   
   // Add methods in superclass.
   if (IFace->getSuperClass())
     AddObjCMethods(IFace->getSuperClass(), WantInstanceMethods, WantKind, 
-                   SelIdents, NumSelIdents, CurContext, Results);
+                   SelIdents, NumSelIdents, CurContext, Results, false);
 
   // Add methods in our implementation, if any.
   if (ObjCImplementationDecl *Impl = IFace->getImplementation())
     AddObjCMethods(Impl, WantInstanceMethods, WantKind, SelIdents,
-                   NumSelIdents, CurContext, Results);
+                   NumSelIdents, CurContext, Results, InOriginalClass);
 }
 
 
-void Sema::CodeCompleteObjCPropertyGetter(Scope *S, DeclPtrTy ClassDecl,
-                                          DeclPtrTy *Methods,
+void Sema::CodeCompleteObjCPropertyGetter(Scope *S, Decl *ClassDecl,
+                                          Decl **Methods,
                                           unsigned NumMethods) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
 
   // Try to find the interface where getters might live.
-  ObjCInterfaceDecl *Class
-    = dyn_cast_or_null<ObjCInterfaceDecl>(ClassDecl.getAs<Decl>());
+  ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(ClassDecl);
   if (!Class) {
     if (ObjCCategoryDecl *Category
-          = dyn_cast_or_null<ObjCCategoryDecl>(ClassDecl.getAs<Decl>()))
+          = dyn_cast_or_null<ObjCCategoryDecl>(ClassDecl))
       Class = Category->getClassInterface();
 
     if (!Class)
@@ -3283,7 +3897,7 @@ void Sema::CodeCompleteObjCPropertyGetter(Scope *S, DeclPtrTy ClassDecl,
   // pushed into DeclContexts early enough. Argh!
   for (unsigned I = 0; I != NumMethods; ++I) { 
     if (ObjCMethodDecl *Method
-            = dyn_cast_or_null<ObjCMethodDecl>(Methods[I].getAs<Decl>()))
+            = dyn_cast_or_null<ObjCMethodDecl>(Methods[I]))
       if (Method->isInstanceMethod() &&
           isAcceptableObjCMethod(Method, MK_ZeroArgSelector, 0, 0)) {
         Result R = Result(Method, 0);
@@ -3294,20 +3908,22 @@ void Sema::CodeCompleteObjCPropertyGetter(Scope *S, DeclPtrTy ClassDecl,
 
   AddObjCMethods(Class, true, MK_ZeroArgSelector, 0, 0, CurContext, Results);
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter,Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter,
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
-void Sema::CodeCompleteObjCPropertySetter(Scope *S, DeclPtrTy ObjCImplDecl,
-                                          DeclPtrTy *Methods,
+void Sema::CodeCompleteObjCPropertySetter(Scope *S, Decl *ObjCImplDecl,
+                                          Decl **Methods,
                                           unsigned NumMethods) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
 
   // Try to find the interface where setters might live.
   ObjCInterfaceDecl *Class
-    = dyn_cast_or_null<ObjCInterfaceDecl>(ObjCImplDecl.getAs<Decl>());
+    = dyn_cast_or_null<ObjCInterfaceDecl>(ObjCImplDecl);
   if (!Class) {
     if (ObjCCategoryDecl *Category
-          = dyn_cast_or_null<ObjCCategoryDecl>(ObjCImplDecl.getAs<Decl>()))
+          = dyn_cast_or_null<ObjCCategoryDecl>(ObjCImplDecl))
       Class = Category->getClassInterface();
 
     if (!Class)
@@ -3322,7 +3938,7 @@ void Sema::CodeCompleteObjCPropertySetter(Scope *S, DeclPtrTy ObjCImplDecl,
   // pushed into DeclContexts early enough. Argh!
   for (unsigned I = 0; I != NumMethods; ++I) { 
     if (ObjCMethodDecl *Method
-            = dyn_cast_or_null<ObjCMethodDecl>(Methods[I].getAs<Decl>()))
+            = dyn_cast_or_null<ObjCMethodDecl>(Methods[I]))
       if (Method->isInstanceMethod() &&
           isAcceptableObjCMethod(Method, MK_OneArgSelector, 0, 0)) {
         Result R = Result(Method, 0);
@@ -3334,7 +3950,54 @@ void Sema::CodeCompleteObjCPropertySetter(Scope *S, DeclPtrTy ObjCImplDecl,
   AddObjCMethods(Class, true, MK_OneArgSelector, 0, 0, CurContext, Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter,Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter,
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS) {
+  typedef CodeCompletionResult Result;
+  ResultBuilder Results(*this);
+  Results.EnterNewScope();
+  
+  // Add context-sensitive, Objective-C parameter-passing keywords.
+  bool AddedInOut = false;
+  if ((DS.getObjCDeclQualifier() & 
+       (ObjCDeclSpec::DQ_In | ObjCDeclSpec::DQ_Inout)) == 0) {
+    Results.AddResult("in");
+    Results.AddResult("inout");
+    AddedInOut = true;
+  }
+  if ((DS.getObjCDeclQualifier() & 
+       (ObjCDeclSpec::DQ_Out | ObjCDeclSpec::DQ_Inout)) == 0) {
+    Results.AddResult("out");
+    if (!AddedInOut)
+      Results.AddResult("inout");
+  }
+  if ((DS.getObjCDeclQualifier() & 
+       (ObjCDeclSpec::DQ_Bycopy | ObjCDeclSpec::DQ_Byref |
+        ObjCDeclSpec::DQ_Oneway)) == 0) {
+     Results.AddResult("bycopy");
+     Results.AddResult("byref");
+     Results.AddResult("oneway");
+  }
+  
+  // Add various builtin type names and specifiers.
+  AddOrdinaryNameResults(PCC_Type, S, *this, Results);
+  Results.ExitScope();
+  
+  // Add the various type names
+  Results.setFilter(&ResultBuilder::IsOrdinaryNonValueName);
+  CodeCompletionDeclConsumer Consumer(Results, CurContext);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
+  
+  if (CodeCompleter->includeMacros())
+    AddMacroResults(PP, Results);
+
+  HandleCodeCompleteResults(this, CodeCompleter,
+                            CodeCompletionContext::CCC_Type,
+                            Results.data(), Results.size());
 }
 
 /// \brief When we have an expression with type "id", we may assume
@@ -3407,28 +4070,138 @@ static ObjCInterfaceDecl *GetAssumedMessageSendExprType(Expr *E) {
     .Default(0);
 }
 
+// Add a special completion for a message send to "super", which fills in the
+// most likely case of forwarding all of our arguments to the superclass 
+// function.
+///
+/// \param S The semantic analysis object.
+///
+/// \param S NeedSuperKeyword Whether we need to prefix this completion with
+/// the "super" keyword. Otherwise, we just need to provide the arguments.
+///
+/// \param SelIdents The identifiers in the selector that have already been
+/// provided as arguments for a send to "super".
+///
+/// \param NumSelIdents The number of identifiers in \p SelIdents.
+///
+/// \param Results The set of results to augment.
+///
+/// \returns the Objective-C method declaration that would be invoked by 
+/// this "super" completion. If NULL, no completion was added.
+static ObjCMethodDecl *AddSuperSendCompletion(Sema &S, bool NeedSuperKeyword,
+                                              IdentifierInfo **SelIdents,
+                                              unsigned NumSelIdents,
+                                              ResultBuilder &Results) {
+  ObjCMethodDecl *CurMethod = S.getCurMethodDecl();
+  if (!CurMethod)
+    return 0;
+  
+  ObjCInterfaceDecl *Class = CurMethod->getClassInterface();
+  if (!Class)
+    return 0;
+  
+  // Try to find a superclass method with the same selector.
+  ObjCMethodDecl *SuperMethod = 0;
+  while ((Class = Class->getSuperClass()) && !SuperMethod)
+    SuperMethod = Class->getMethod(CurMethod->getSelector(), 
+                                   CurMethod->isInstanceMethod());
+
+  if (!SuperMethod)
+    return 0;
+  
+  // Check whether the superclass method has the same signature.
+  if (CurMethod->param_size() != SuperMethod->param_size() ||
+      CurMethod->isVariadic() != SuperMethod->isVariadic())
+    return 0;
+      
+  for (ObjCMethodDecl::param_iterator CurP = CurMethod->param_begin(),
+                                   CurPEnd = CurMethod->param_end(),
+                                    SuperP = SuperMethod->param_begin();
+       CurP != CurPEnd; ++CurP, ++SuperP) {
+    // Make sure the parameter types are compatible.
+    if (!S.Context.hasSameUnqualifiedType((*CurP)->getType(), 
+                                          (*SuperP)->getType()))
+      return 0;
+    
+    // Make sure we have a parameter name to forward!
+    if (!(*CurP)->getIdentifier())
+      return 0;
+  }
+  
+  // We have a superclass method. Now, form the send-to-super completion.
+  CodeCompletionString *Pattern = new CodeCompletionString;
+  
+  // Give this completion a return type.
+  AddResultTypeChunk(S.Context, SuperMethod, Pattern);
+
+  // If we need the "super" keyword, add it (plus some spacing).
+  if (NeedSuperKeyword) {
+    Pattern->AddTypedTextChunk("super");
+    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  }
+  
+  Selector Sel = CurMethod->getSelector();
+  if (Sel.isUnarySelector()) {
+    if (NeedSuperKeyword)
+      Pattern->AddTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
+    else
+      Pattern->AddTypedTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
+  } else {
+    ObjCMethodDecl::param_iterator CurP = CurMethod->param_begin();
+    for (unsigned I = 0, N = Sel.getNumArgs(); I != N; ++I, ++CurP) {
+      if (I > NumSelIdents)
+        Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+      
+      if (I < NumSelIdents)
+        Pattern->AddInformativeChunk(
+                       Sel.getIdentifierInfoForSlot(I)->getName().str() + ":");
+      else if (NeedSuperKeyword || I > NumSelIdents) {
+        Pattern->AddTextChunk(
+                        Sel.getIdentifierInfoForSlot(I)->getName().str() + ":");
+        Pattern->AddPlaceholderChunk((*CurP)->getIdentifier()->getName());
+      } else {
+        Pattern->AddTypedTextChunk(
+                              Sel.getIdentifierInfoForSlot(I)->getName().str() + ":");
+        Pattern->AddPlaceholderChunk((*CurP)->getIdentifier()->getName());        
+      }
+    }
+  }
+  
+  Results.AddResult(CodeCompletionResult(Pattern, CCP_SuperCompletion,
+                                         SuperMethod->isInstanceMethod()
+                                           ? CXCursor_ObjCInstanceMethodDecl
+                                           : CXCursor_ObjCClassMethodDecl));
+  return SuperMethod;
+}
+                                   
 void Sema::CodeCompleteObjCMessageReceiver(Scope *S) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
   
   // Find anything that looks like it could be a message receiver.
   Results.setFilter(&ResultBuilder::IsObjCMessageReceiver);
   CodeCompletionDeclConsumer Consumer(Results, CurContext);
   Results.EnterNewScope();
-  LookupVisibleDecls(S, LookupOrdinaryName, Consumer);
+  LookupVisibleDecls(S, LookupOrdinaryName, Consumer,
+                     CodeCompleter->includeGlobals());
   
   // If we are in an Objective-C method inside a class that has a superclass,
   // add "super" as an option.
   if (ObjCMethodDecl *Method = getCurMethodDecl())
     if (ObjCInterfaceDecl *Iface = Method->getClassInterface())
-      if (Iface->getSuperClass())
+      if (Iface->getSuperClass()) {
         Results.AddResult(Result("super"));
+        
+        AddSuperSendCompletion(*this, /*NeedSuperKeyword=*/true, 0, 0, Results);
+      }
   
   Results.ExitScope();
   
   if (CodeCompleter->includeMacros())
     AddMacroResults(PP, Results);
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_ObjCMessageReceiver,
+                            Results.data(), Results.size());
   
 }
 
@@ -3454,10 +4227,11 @@ void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
       // an instance method.
       QualType SuperTy = Context.getObjCInterfaceType(CDecl);
       SuperTy = Context.getObjCObjectPointerType(SuperTy);
-      OwningExprResult Super
+      ExprResult Super
         = Owned(new (Context) ObjCSuperExpr(SuperLoc, SuperTy));
       return CodeCompleteObjCInstanceMessage(S, (Expr *)Super.get(),
-                                             SelIdents, NumSelIdents);
+                                             SelIdents, NumSelIdents,
+                                             /*IsSuper=*/true);
     }
 
     // Fall through to send to the superclass in CDecl.
@@ -3480,7 +4254,7 @@ void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
       CXXScopeSpec SS;
       UnqualifiedId id;
       id.setIdentifier(Super, SuperLoc);
-      OwningExprResult SuperExpr = ActOnIdExpression(S, SS, id, false, false);
+      ExprResult SuperExpr = ActOnIdExpression(S, SS, id, false, false);
       return CodeCompleteObjCInstanceMessage(S, (Expr *)SuperExpr.get(),
                                              SelIdents, NumSelIdents);
     }
@@ -3488,17 +4262,24 @@ void Sema::CodeCompleteObjCSuperMessage(Scope *S, SourceLocation SuperLoc,
     // Fall through
   }
 
-  TypeTy *Receiver = 0;
+  ParsedType Receiver;
   if (CDecl)
-    Receiver = Context.getObjCInterfaceType(CDecl).getAsOpaquePtr();
+    Receiver = ParsedType::make(Context.getObjCInterfaceType(CDecl));
   return CodeCompleteObjCClassMessage(S, Receiver, SelIdents, 
-                                      NumSelIdents);
+                                      NumSelIdents, /*IsSuper=*/true);
 }
 
-void Sema::CodeCompleteObjCClassMessage(Scope *S, TypeTy *Receiver,
+void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
                                         IdentifierInfo **SelIdents,
                                         unsigned NumSelIdents) {
-  typedef CodeCompleteConsumer::Result Result;
+  CodeCompleteObjCClassMessage(S, Receiver, SelIdents, NumSelIdents, false);
+}
+
+void Sema::CodeCompleteObjCClassMessage(Scope *S, ParsedType Receiver,
+                                        IdentifierInfo **SelIdents,
+                                        unsigned NumSelIdents,
+                                        bool IsSuper) {
+  typedef CodeCompletionResult Result;
   ObjCInterfaceDecl *CDecl = 0;
 
   // If the given name refers to an interface type, retrieve the
@@ -3515,6 +4296,20 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, TypeTy *Receiver,
   ResultBuilder Results(*this);
   Results.EnterNewScope();
 
+  // If this is a send-to-super, try to add the special "super" send 
+  // completion.
+  if (IsSuper) {
+    if (ObjCMethodDecl *SuperMethod
+          = AddSuperSendCompletion(*this, false, SelIdents, NumSelIdents, 
+                                   Results))
+      Results.Ignore(SuperMethod);
+  }
+
+  // If we're inside an Objective-C method definition, prefer its selector to
+  // others.
+  if (ObjCMethodDecl *CurMethod = getCurMethodDecl())
+    Results.setPreferredSelector(CurMethod->getSelector());
+
   if (CDecl) 
     AddObjCMethods(CDecl, false, MK_Any, SelIdents, NumSelIdents, CurContext, 
                    Results);  
@@ -3522,25 +4317,23 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, TypeTy *Receiver,
     // We're messaging "id" as a type; provide all class/factory methods.
 
     // If we have an external source, load the entire class method
-    // pool from the PCH file.
+    // pool from the AST file.
     if (ExternalSource) {
       for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
            I != N; ++I) {
         Selector Sel = ExternalSource->GetExternalSelector(I);
-        if (Sel.isNull() || FactoryMethodPool.count(Sel) || 
-            InstanceMethodPool.count(Sel))
+        if (Sel.isNull() || MethodPool.count(Sel))
           continue;
 
-        ReadMethodPool(Sel, /*isInstance=*/false);
+        ReadMethodPool(Sel);
       }
     }
 
-    for (llvm::DenseMap<Selector, ObjCMethodList>::iterator
-           M = FactoryMethodPool.begin(),
-           MEnd = FactoryMethodPool.end();
-         M != MEnd;
-         ++M) {
-      for (ObjCMethodList *MethList = &M->second; MethList && MethList->Method; 
+    for (GlobalMethodPool::iterator M = MethodPool.begin(),
+                                    MEnd = MethodPool.end();
+         M != MEnd; ++M) {
+      for (ObjCMethodList *MethList = &M->second.second;
+           MethList && MethList->Method; 
            MethList = MethList->Next) {
         if (!isAcceptableObjCMethod(MethList->Method, MK_Any, SelIdents, 
                                     NumSelIdents))
@@ -3555,13 +4348,22 @@ void Sema::CodeCompleteObjCClassMessage(Scope *S, TypeTy *Receiver,
   }
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(), Results.size());
 }
 
 void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
                                            IdentifierInfo **SelIdents,
                                            unsigned NumSelIdents) {
-  typedef CodeCompleteConsumer::Result Result;
+  CodeCompleteObjCInstanceMessage(S, Receiver, SelIdents, NumSelIdents, false);
+}
+
+void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
+                                           IdentifierInfo **SelIdents,
+                                           unsigned NumSelIdents,
+                                           bool IsSuper) {
+  typedef CodeCompletionResult Result;
   
   Expr *RecExpr = static_cast<Expr *>(Receiver);
   
@@ -3574,6 +4376,20 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
   ResultBuilder Results(*this);
   Results.EnterNewScope();
 
+  // If this is a send-to-super, try to add the special "super" send 
+  // completion.
+  if (IsSuper) {
+    if (ObjCMethodDecl *SuperMethod
+          = AddSuperSendCompletion(*this, false, SelIdents, NumSelIdents, 
+                                   Results))
+      Results.Ignore(SuperMethod);
+  }
+  
+  // If we're inside an Objective-C method definition, prefer its selector to
+  // others.
+  if (ObjCMethodDecl *CurMethod = getCurMethodDecl())
+    Results.setPreferredSelector(CurMethod->getSelector());
+
   // If we're messaging an expression with type "id" or "Class", check
   // whether we know something special about the receiver that allows
   // us to assume a more-specific receiver type.
@@ -3623,25 +4439,23 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
     // about as code-completion results.
 
     // If we have an external source, load the entire class method
-    // pool from the PCH file.
+    // pool from the AST file.
     if (ExternalSource) {
       for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
            I != N; ++I) {
         Selector Sel = ExternalSource->GetExternalSelector(I);
-        if (Sel.isNull() || InstanceMethodPool.count(Sel) ||
-            FactoryMethodPool.count(Sel))
+        if (Sel.isNull() || MethodPool.count(Sel))
           continue;
 
-        ReadMethodPool(Sel, /*isInstance=*/true);
+        ReadMethodPool(Sel);
       }
     }
 
-    for (llvm::DenseMap<Selector, ObjCMethodList>::iterator
-           M = InstanceMethodPool.begin(),
-           MEnd = InstanceMethodPool.end();
-         M != MEnd;
-         ++M) {
-      for (ObjCMethodList *MethList = &M->second; MethList && MethList->Method; 
+    for (GlobalMethodPool::iterator M = MethodPool.begin(),
+                                    MEnd = MethodPool.end();
+         M != MEnd; ++M) {
+      for (ObjCMethodList *MethList = &M->second.first;
+           MethList && MethList->Method; 
            MethList = MethList->Next) {
         if (!isAcceptableObjCMethod(MethList->Method, MK_Any, SelIdents, 
                                     NumSelIdents))
@@ -3656,7 +4470,79 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
   }
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompleteObjCForCollection(Scope *S, 
+                                         DeclGroupPtrTy IterationVar) {
+  CodeCompleteExpressionData Data;
+  Data.ObjCCollection = true;
+  
+  if (IterationVar.getAsOpaquePtr()) {
+    DeclGroupRef DG = IterationVar.getAsVal<DeclGroupRef>();
+    for (DeclGroupRef::iterator I = DG.begin(), End = DG.end(); I != End; ++I) {
+      if (*I)
+        Data.IgnoreDecls.push_back(*I);
+    }
+  }
+  
+  CodeCompleteExpression(S, Data);
+}
+
+void Sema::CodeCompleteObjCSelector(Scope *S, IdentifierInfo **SelIdents,
+                                    unsigned NumSelIdents) {
+  // If we have an external source, load the entire class method
+  // pool from the AST file.
+  if (ExternalSource) {
+    for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
+         I != N; ++I) {
+      Selector Sel = ExternalSource->GetExternalSelector(I);
+      if (Sel.isNull() || MethodPool.count(Sel))
+        continue;
+      
+      ReadMethodPool(Sel);
+    }
+  }
+  
+  ResultBuilder Results(*this);
+  Results.EnterNewScope();
+  for (GlobalMethodPool::iterator M = MethodPool.begin(),
+                               MEnd = MethodPool.end();
+       M != MEnd; ++M) {
+    
+    Selector Sel = M->first;
+    if (!isAcceptableObjCSelector(Sel, MK_Any, SelIdents, NumSelIdents))
+      continue;
+
+    CodeCompletionString *Pattern = new CodeCompletionString;
+    if (Sel.isUnarySelector()) {
+      Pattern->AddTypedTextChunk(Sel.getIdentifierInfoForSlot(0)->getName());
+      Results.AddResult(Pattern);
+      continue;
+    }
+    
+    std::string Accumulator;
+    for (unsigned I = 0, N = Sel.getNumArgs(); I != N; ++I) {
+      if (I == NumSelIdents) {
+        if (!Accumulator.empty()) {
+          Pattern->AddInformativeChunk(Accumulator);
+          Accumulator.clear();
+        }
+      }
+      
+      Accumulator += Sel.getIdentifierInfoForSlot(I)->getName().str();
+      Accumulator += ':';
+    }
+    Pattern->AddTypedTextChunk(Accumulator);
+    Results.AddResult(Pattern);
+  }
+  Results.ExitScope();
+  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_SelectorName,
+                            Results.data(), Results.size());
 }
 
 /// \brief Add all of the protocol declarations that we find in the given
@@ -3664,7 +4550,7 @@ void Sema::CodeCompleteObjCInstanceMessage(Scope *S, ExprTy *Receiver,
 static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
                                bool OnlyForwardDeclarations,
                                ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   for (DeclContext::decl_iterator D = Ctx->decls_begin(), 
                                DEnd = Ctx->decls_end();
@@ -3704,7 +4590,9 @@ void Sema::CodeCompleteObjCProtocolReferences(IdentifierLocPair *Protocols,
                      Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_ObjCProtocolName,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCProtocolDecl(Scope *) {
@@ -3716,7 +4604,9 @@ void Sema::CodeCompleteObjCProtocolDecl(Scope *) {
                      Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_ObjCProtocolName,
+                            Results.data(),Results.size());
 }
 
 /// \brief Add all of the Objective-C interface declarations that we find in
@@ -3725,7 +4615,7 @@ static void AddInterfaceResults(DeclContext *Ctx, DeclContext *CurContext,
                                 bool OnlyForwardDeclarations,
                                 bool OnlyUnimplemented,
                                 ResultBuilder &Results) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   for (DeclContext::decl_iterator D = Ctx->decls_begin(), 
                                DEnd = Ctx->decls_end();
@@ -3757,7 +4647,9 @@ void Sema::CodeCompleteObjCInterfaceDecl(Scope *S) {
                       false, Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter,
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
@@ -3776,7 +4668,9 @@ void Sema::CodeCompleteObjCSuperclass(Scope *S, IdentifierInfo *ClassName,
                       false, Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCImplementationDecl(Scope *S) { 
@@ -3788,13 +4682,15 @@ void Sema::CodeCompleteObjCImplementationDecl(Scope *S) {
                       true, Results);
 
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCInterfaceCategory(Scope *S, 
                                              IdentifierInfo *ClassName,
                                              SourceLocation ClassNameLoc) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   ResultBuilder Results(*this);
   
@@ -3819,13 +4715,15 @@ void Sema::CodeCompleteObjCInterfaceCategory(Scope *S,
         Results.AddResult(Result(Category, 0), CurContext, 0, false);
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());  
 }
 
 void Sema::CodeCompleteObjCImplementationCategory(Scope *S, 
                                                   IdentifierInfo *ClassName,
                                                   SourceLocation ClassNameLoc) {
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   
   // Find the corresponding interface. If we couldn't find the interface, the
   // program itself is ill-formed. However, we'll try to be helpful still by
@@ -3856,16 +4754,18 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
   }
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());  
 }
 
-void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, DeclPtrTy ObjCImpDecl) {
-  typedef CodeCompleteConsumer::Result Result;
+void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, Decl *ObjCImpDecl) {
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container
-    = dyn_cast_or_null<ObjCContainerDecl>(ObjCImpDecl.getAs<Decl>());
+    = dyn_cast_or_null<ObjCContainerDecl>(ObjCImpDecl);
   if (!Container || 
       (!isa<ObjCImplementationDecl>(Container) && 
        !isa<ObjCCategoryImplDecl>(Container)))
@@ -3889,18 +4789,20 @@ void Sema::CodeCompleteObjCPropertyDefinition(Scope *S, DeclPtrTy ObjCImpDecl) {
                       false, CurContext, Results);
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());  
 }
 
 void Sema::CodeCompleteObjCPropertySynthesizeIvar(Scope *S, 
                                                   IdentifierInfo *PropertyName,
-                                                  DeclPtrTy ObjCImpDecl) {
-  typedef CodeCompleteConsumer::Result Result;
+                                                  Decl *ObjCImpDecl) {
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
 
   // Figure out where this @synthesize lives.
   ObjCContainerDecl *Container
-    = dyn_cast_or_null<ObjCContainerDecl>(ObjCImpDecl.getAs<Decl>());
+    = dyn_cast_or_null<ObjCContainerDecl>(ObjCImpDecl);
   if (!Container || 
       (!isa<ObjCImplementationDecl>(Container) && 
        !isa<ObjCCategoryImplDecl>(Container)))
@@ -3927,10 +4829,15 @@ void Sema::CodeCompleteObjCPropertySynthesizeIvar(Scope *S,
   }
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
-typedef llvm::DenseMap<Selector, ObjCMethodDecl *> KnownMethodsMap;
+// Mapping from selectors to the methods that implement that selector, along
+// with the "in original class" flag.
+typedef llvm::DenseMap<Selector, std::pair<ObjCMethodDecl *, bool> > 
+  KnownMethodsMap;
 
 /// \brief Find all of the methods that reside in the given container
 /// (and its superclasses, protocols, etc.) that meet the given
@@ -3941,7 +4848,8 @@ static void FindImplementableMethods(ASTContext &Context,
                                      bool WantInstanceMethods,
                                      QualType ReturnType,
                                      bool IsInImplementation,
-                                     KnownMethodsMap &KnownMethods) {
+                                     KnownMethodsMap &KnownMethods,
+                                     bool InOriginalClass = true) {
   if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>(Container)) {
     // Recurse into protocols.
     const ObjCList<ObjCProtocolDecl> &Protocols
@@ -3950,14 +4858,16 @@ static void FindImplementableMethods(ASTContext &Context,
            E = Protocols.end(); 
          I != E; ++I)
       FindImplementableMethods(Context, *I, WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods);
+                               IsInImplementation, KnownMethods,
+                               InOriginalClass);
 
     // If we're not in the implementation of a class, also visit the
     // superclass.
     if (!IsInImplementation && IFace->getSuperClass())
       FindImplementableMethods(Context, IFace->getSuperClass(), 
                                WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods);
+                               IsInImplementation, KnownMethods,
+                               false);
 
     // Add methods from any class extensions (but not from categories;
     // those should go into category implementations).
@@ -3965,7 +4875,8 @@ static void FindImplementableMethods(ASTContext &Context,
          Cat = Cat->getNextClassExtension())
       FindImplementableMethods(Context, const_cast<ObjCCategoryDecl*>(Cat), 
                                WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods);      
+                               IsInImplementation, KnownMethods,
+                               InOriginalClass);      
   }
 
   if (ObjCCategoryDecl *Category = dyn_cast<ObjCCategoryDecl>(Container)) {
@@ -3976,7 +4887,8 @@ static void FindImplementableMethods(ASTContext &Context,
            E = Protocols.end(); 
          I != E; ++I)
       FindImplementableMethods(Context, *I, WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods);
+                               IsInImplementation, KnownMethods,
+                               InOriginalClass);
   }
 
   if (ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(Container)) {
@@ -3987,7 +4899,7 @@ static void FindImplementableMethods(ASTContext &Context,
            E = Protocols.end(); 
          I != E; ++I)
       FindImplementableMethods(Context, *I, WantInstanceMethods, ReturnType,
-                               IsInImplementation, KnownMethods);
+                               IsInImplementation, KnownMethods, false);
   }
 
   // Add methods in this container. This operation occurs last because
@@ -4001,15 +4913,15 @@ static void FindImplementableMethods(ASTContext &Context,
           !Context.hasSameUnqualifiedType(ReturnType, (*M)->getResultType()))
         continue;
 
-      KnownMethods[(*M)->getSelector()] = *M;
+      KnownMethods[(*M)->getSelector()] = std::make_pair(*M, InOriginalClass);
     }
   }
 }
 
 void Sema::CodeCompleteObjCMethodDecl(Scope *S, 
                                       bool IsInstanceMethod,
-                                      TypeTy *ReturnTy,
-                                      DeclPtrTy IDecl) {
+                                      ParsedType ReturnTy,
+                                      Decl *IDecl) {
   // Determine the return type of the method we're declaring, if
   // provided.
   QualType ReturnType = GetTypeFromParser(ReturnTy);
@@ -4017,7 +4929,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   // Determine where we should start searching for methods, and where we 
   ObjCContainerDecl *SearchDecl = 0, *CurrentDecl = 0;
   bool IsInImplementation = false;
-  if (Decl *D = IDecl.getAs<Decl>()) {
+  if (Decl *D = IDecl) {
     if (ObjCImplementationDecl *Impl = dyn_cast<ObjCImplementationDecl>(D)) {
       SearchDecl = Impl->getClassInterface();
       CurrentDecl = Impl;
@@ -4041,7 +4953,9 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   }
 
   if (!SearchDecl || !CurrentDecl) {
-    HandleCodeCompleteResults(this, CodeCompleter, 0, 0);
+    HandleCodeCompleteResults(this, CodeCompleter, 
+                              CodeCompletionContext::CCC_Other,
+                              0, 0);
     return;
   }
     
@@ -4064,7 +4978,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   }
 
   // Add declarations or definitions for each of the known methods.
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
   Results.EnterNewScope();
   PrintingPolicy Policy(Context.PrintingPolicy);
@@ -4072,7 +4986,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
   for (KnownMethodsMap::iterator M = KnownMethods.begin(), 
                               MEnd = KnownMethods.end();
        M != MEnd; ++M) {
-    ObjCMethodDecl *Method = M->second;
+    ObjCMethodDecl *Method = M->second.first;
     CodeCompletionString *Pattern = new CodeCompletionString;
     
     // If the result type was not already provided, add it to the
@@ -4100,7 +5014,7 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
         Pattern->AddChunk(CodeCompletionString::CK_Colon);
       else if (I < Sel.getNumArgs()) {
         Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
-        Pattern->AddTextChunk(Sel.getIdentifierInfoForSlot(1)->getName());
+        Pattern->AddTextChunk(Sel.getIdentifierInfoForSlot(I)->getName());
         Pattern->AddChunk(CodeCompletionString::CK_Colon);
       } else
         break;
@@ -4113,14 +5027,14 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
       Pattern->AddChunk(CodeCompletionString::CK_RightParen);
       
       if (IdentifierInfo *Id = (*P)->getIdentifier())
-        Pattern->AddTextChunk(Id->getName());
+        Pattern->AddTextChunk(Id->getName());      
     }
 
     if (Method->isVariadic()) {
       if (Method->param_size() > 0)
         Pattern->AddChunk(CodeCompletionString::CK_Comma);
       Pattern->AddTextChunk("...");
-    }
+    }        
 
     if (IsInImplementation && Results.includeCodePatterns()) {
       // We will be defining the method here, so add a compound statement.
@@ -4140,50 +5054,56 @@ void Sema::CodeCompleteObjCMethodDecl(Scope *S,
       Pattern->AddChunk(CodeCompletionString::CK_RightBrace);
     }
 
-    Results.AddResult(Result(Pattern));
+    unsigned Priority = CCP_CodePattern;
+    if (!M->second.second)
+      Priority += CCD_InBaseClass;
+    
+    Results.AddResult(Result(Pattern, Priority, 
+                             Method->isInstanceMethod()
+                               ? CXCursor_ObjCInstanceMethodDecl
+                               : CXCursor_ObjCClassMethodDecl));
   }
 
   Results.ExitScope();
   
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
 }
 
 void Sema::CodeCompleteObjCMethodDeclSelector(Scope *S, 
                                               bool IsInstanceMethod,
                                               bool AtParameterName,
-                                              TypeTy *ReturnTy,
+                                              ParsedType ReturnTy,
                                               IdentifierInfo **SelIdents,
                                               unsigned NumSelIdents) {
-  llvm::DenseMap<Selector, ObjCMethodList> &Pool 
-    = IsInstanceMethod? InstanceMethodPool : FactoryMethodPool;
-  
   // If we have an external source, load the entire class method
-  // pool from the PCH file.
+  // pool from the AST file.
   if (ExternalSource) {
     for (uint32_t I = 0, N = ExternalSource->GetNumExternalSelectors();
          I != N; ++I) {
       Selector Sel = ExternalSource->GetExternalSelector(I);
-      if (Sel.isNull() || InstanceMethodPool.count(Sel) ||
-          FactoryMethodPool.count(Sel))
+      if (Sel.isNull() || MethodPool.count(Sel))
         continue;
-      
-      ReadMethodPool(Sel, IsInstanceMethod);
+
+      ReadMethodPool(Sel);
     }
   }
 
   // Build the set of methods we can see.
-  typedef CodeCompleteConsumer::Result Result;
+  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this);
   
   if (ReturnTy)
     Results.setPreferredType(GetTypeFromParser(ReturnTy).getNonReferenceType());
-  
+
   Results.EnterNewScope();  
-  for (llvm::DenseMap<Selector, ObjCMethodList>::iterator M = Pool.begin(),
-                                                       MEnd = Pool.end();
-       M != MEnd;
-       ++M) {
-    for (ObjCMethodList *MethList = &M->second; MethList && MethList->Method; 
+  for (GlobalMethodPool::iterator M = MethodPool.begin(),
+                                  MEnd = MethodPool.end();
+       M != MEnd; ++M) {
+    for (ObjCMethodList *MethList = IsInstanceMethod ? &M->second.first :
+                                                       &M->second.second;
+         MethList && MethList->Method; 
          MethList = MethList->Next) {
       if (!isAcceptableObjCMethod(MethList->Method, MK_Any, SelIdents, 
                                   NumSelIdents))
@@ -4212,5 +5132,270 @@ void Sema::CodeCompleteObjCMethodDeclSelector(Scope *S,
   }
   
   Results.ExitScope();
-  HandleCodeCompleteResults(this, CodeCompleter, Results.data(),Results.size());
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_Other,
+                            Results.data(),Results.size());
+}
+
+void Sema::CodeCompletePreprocessorDirective(bool InConditional) {
+  ResultBuilder Results(*this);
+  Results.EnterNewScope();
+  
+  // #if <condition>
+  CodeCompletionString *Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("if");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("condition");
+  Results.AddResult(Pattern);
+  
+  // #ifdef <macro>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("ifdef");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("macro");
+  Results.AddResult(Pattern);
+
+  // #ifndef <macro>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("ifndef");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("macro");
+  Results.AddResult(Pattern);
+
+  if (InConditional) {
+    // #elif <condition>
+    Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk("elif");
+    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Pattern->AddPlaceholderChunk("condition");
+    Results.AddResult(Pattern);
+
+    // #else
+    Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk("else");
+    Results.AddResult(Pattern);
+
+    // #endif
+    Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk("endif");
+    Results.AddResult(Pattern);
+  }
+  
+  // #include "header"
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("include");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddTextChunk("\"");
+  Pattern->AddPlaceholderChunk("header");
+  Pattern->AddTextChunk("\"");
+  Results.AddResult(Pattern);
+
+  // #include <header>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("include");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddTextChunk("<");
+  Pattern->AddPlaceholderChunk("header");
+  Pattern->AddTextChunk(">");
+  Results.AddResult(Pattern);
+  
+  // #define <macro>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("define");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("macro");
+  Results.AddResult(Pattern);
+  
+  // #define <macro>(<args>)
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("define");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("macro");
+  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+  Pattern->AddPlaceholderChunk("args");
+  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Pattern);
+  
+  // #undef <macro>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("undef");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("macro");
+  Results.AddResult(Pattern);
+
+  // #line <number>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("line");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("number");
+  Results.AddResult(Pattern);
+  
+  // #line <number> "filename"
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("line");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("number");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddTextChunk("\"");
+  Pattern->AddPlaceholderChunk("filename");
+  Pattern->AddTextChunk("\"");
+  Results.AddResult(Pattern);
+  
+  // #error <message>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("error");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("message");
+  Results.AddResult(Pattern);
+
+  // #pragma <arguments>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("pragma");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("arguments");
+  Results.AddResult(Pattern);
+
+  if (getLangOptions().ObjC1) {
+    // #import "header"
+    Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk("import");
+    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Pattern->AddTextChunk("\"");
+    Pattern->AddPlaceholderChunk("header");
+    Pattern->AddTextChunk("\"");
+    Results.AddResult(Pattern);
+    
+    // #import <header>
+    Pattern = new CodeCompletionString;
+    Pattern->AddTypedTextChunk("import");
+    Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+    Pattern->AddTextChunk("<");
+    Pattern->AddPlaceholderChunk("header");
+    Pattern->AddTextChunk(">");
+    Results.AddResult(Pattern);
+  }
+  
+  // #include_next "header"
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("include_next");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddTextChunk("\"");
+  Pattern->AddPlaceholderChunk("header");
+  Pattern->AddTextChunk("\"");
+  Results.AddResult(Pattern);
+  
+  // #include_next <header>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("include_next");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddTextChunk("<");
+  Pattern->AddPlaceholderChunk("header");
+  Pattern->AddTextChunk(">");
+  Results.AddResult(Pattern);
+
+  // #warning <message>
+  Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("warning");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddPlaceholderChunk("message");
+  Results.AddResult(Pattern);
+
+  // Note: #ident and #sccs are such crazy anachronisms that we don't provide
+  // completions for them. And __include_macros is a Clang-internal extension
+  // that we don't want to encourage anyone to use.
+
+  // FIXME: we don't support #assert or #unassert, so don't suggest them.
+  Results.ExitScope();
+  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_PreprocessorDirective,
+                            Results.data(), Results.size());
+}
+
+void Sema::CodeCompleteInPreprocessorConditionalExclusion(Scope *S) {
+  CodeCompleteOrdinaryName(S,
+                           S->getFnParent()? Sema::PCC_RecoveryInFunction 
+                                           : Sema::PCC_Namespace);
+}
+
+void Sema::CodeCompletePreprocessorMacroName(bool IsDefinition) {
+  ResultBuilder Results(*this);
+  if (!IsDefinition && (!CodeCompleter || CodeCompleter->includeMacros())) {
+    // Add just the names of macros, not their arguments.    
+    Results.EnterNewScope();
+    for (Preprocessor::macro_iterator M = PP.macro_begin(), 
+                                   MEnd = PP.macro_end();
+         M != MEnd; ++M) {
+      CodeCompletionString *Pattern = new CodeCompletionString;
+      Pattern->AddTypedTextChunk(M->first->getName());
+      Results.AddResult(Pattern);
+    }
+    Results.ExitScope();
+  } else if (IsDefinition) {
+    // FIXME: Can we detect when the user just wrote an include guard above?
+  }
+  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                      IsDefinition? CodeCompletionContext::CCC_MacroName
+                                  : CodeCompletionContext::CCC_MacroNameUse,
+                            Results.data(), Results.size()); 
+}
+
+void Sema::CodeCompletePreprocessorExpression() {
+  ResultBuilder Results(*this);
+  
+  if (!CodeCompleter || CodeCompleter->includeMacros())
+    AddMacroResults(PP, Results);
+  
+    // defined (<macro>)
+  Results.EnterNewScope();
+  CodeCompletionString *Pattern = new CodeCompletionString;
+  Pattern->AddTypedTextChunk("defined");
+  Pattern->AddChunk(CodeCompletionString::CK_HorizontalSpace);
+  Pattern->AddChunk(CodeCompletionString::CK_LeftParen);
+  Pattern->AddPlaceholderChunk("macro");
+  Pattern->AddChunk(CodeCompletionString::CK_RightParen);
+  Results.AddResult(Pattern);
+  Results.ExitScope();
+  
+  HandleCodeCompleteResults(this, CodeCompleter, 
+                            CodeCompletionContext::CCC_PreprocessorExpression,
+                            Results.data(), Results.size()); 
+}
+
+void Sema::CodeCompletePreprocessorMacroArgument(Scope *S,
+                                                 IdentifierInfo *Macro,
+                                                 MacroInfo *MacroInfo,
+                                                 unsigned Argument) {
+  // FIXME: In the future, we could provide "overload" results, much like we
+  // do for function calls.
+  
+  CodeCompleteOrdinaryName(S,
+                           S->getFnParent()? Sema::PCC_RecoveryInFunction 
+                                           : Sema::PCC_Namespace);
+}
+
+void Sema::CodeCompleteNaturalLanguage() {
+  HandleCodeCompleteResults(this, CodeCompleter,
+                            CodeCompletionContext::CCC_NaturalLanguage,
+                            0, 0);
+}
+
+void Sema::GatherGlobalCodeCompletions(
+                 llvm::SmallVectorImpl<CodeCompletionResult> &Results) {
+  ResultBuilder Builder(*this);
+
+  if (!CodeCompleter || CodeCompleter->includeGlobals()) {
+    CodeCompletionDeclConsumer Consumer(Builder, 
+                                        Context.getTranslationUnitDecl());
+    LookupVisibleDecls(Context.getTranslationUnitDecl(), LookupAnyName, 
+                       Consumer);
+  }
+  
+  if (!CodeCompleter || CodeCompleter->includeMacros())
+    AddMacroResults(PP, Builder);
+  
+  Results.clear();
+  Results.insert(Results.end(), 
+                 Builder.data(), Builder.data() + Builder.size());
 }
diff --git a/lib/Sema/SemaDecl.cpp b/lib/Sema/SemaDecl.cpp
index c1c898fac5f9..f5e045a722a0 100644
--- a/lib/Sema/SemaDecl.cpp
+++ b/lib/Sema/SemaDecl.cpp
@@ -11,19 +11,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/CXXFieldCollector.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/StmtCXX.h"
-#include "clang/Parse/DeclSpec.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "clang/Parse/ParseDiagnostic.h"
-#include "clang/Parse/Template.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -35,18 +40,10 @@
 #include <cstring>
 #include <functional>
 using namespace clang;
+using namespace sema;
 
-/// getDeclName - Return a pretty name for the specified decl if possible, or
-/// an empty string if not.  This is used for pretty crash reporting.
-std::string Sema::getDeclName(DeclPtrTy d) {
-  Decl *D = d.getAs<Decl>();
-  if (NamedDecl *DN = dyn_cast_or_null<NamedDecl>(D))
-    return DN->getQualifiedNameAsString();
-  return "";
-}
-
-Sema::DeclGroupPtrTy Sema::ConvertDeclToDeclGroup(DeclPtrTy Ptr) {
-  return DeclGroupPtrTy::make(DeclGroupRef(Ptr.getAs<Decl>()));
+Sema::DeclGroupPtrTy Sema::ConvertDeclToDeclGroup(Decl *Ptr) {
+  return DeclGroupPtrTy::make(DeclGroupRef(Ptr));
 }
 
 /// \brief If the identifier refers to a type name within this scope,
@@ -60,14 +57,14 @@ Sema::DeclGroupPtrTy Sema::ConvertDeclToDeclGroup(DeclPtrTy Ptr) {
 ///
 /// If name lookup results in an ambiguity, this routine will complain
 /// and then return NULL.
-Sema::TypeTy *Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
-                                Scope *S, CXXScopeSpec *SS,
-                                bool isClassName,
-                                TypeTy *ObjectTypePtr) {
+ParsedType Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
+                             Scope *S, CXXScopeSpec *SS,
+                             bool isClassName,
+                             ParsedType ObjectTypePtr) {
   // Determine where we will perform name lookup.
   DeclContext *LookupCtx = 0;
   if (ObjectTypePtr) {
-    QualType ObjectType = QualType::getFromOpaquePtr(ObjectTypePtr);
+    QualType ObjectType = ObjectTypePtr.get();
     if (ObjectType->isRecordType())
       LookupCtx = computeDeclContext(ObjectType);
   } else if (SS && SS->isNotEmpty()) {
@@ -85,22 +82,22 @@ Sema::TypeTy *Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
         // We therefore do not perform any name lookup if the result would
         // refer to a member of an unknown specialization.
         if (!isClassName)
-          return 0;
+          return ParsedType();
         
         // We know from the grammar that this name refers to a type,
         // so build a dependent node to describe the type.
-        return CheckTypenameType(ETK_None,
-                                 (NestedNameSpecifier *)SS->getScopeRep(), II,
-                                 SourceLocation(), SS->getRange(), NameLoc
-                                 ).getAsOpaquePtr();
+        QualType T =
+          CheckTypenameType(ETK_None, SS->getScopeRep(), II,
+                            SourceLocation(), SS->getRange(), NameLoc);
+        return ParsedType::make(T);
       }
       
-      return 0;
+      return ParsedType();
     }
     
     if (!LookupCtx->isDependentContext() &&
         RequireCompleteDeclContext(*SS, LookupCtx))
-      return 0;
+      return ParsedType();
   }
 
   // FIXME: LookupNestedNameSpecifierName isn't the right kind of
@@ -136,7 +133,7 @@ Sema::TypeTy *Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
   case LookupResult::FoundOverloaded:
   case LookupResult::FoundUnresolvedValue:
     Result.suppressDiagnostics();
-    return 0;
+    return ParsedType();
 
   case LookupResult::Ambiguous:
     // Recover from type-hiding ambiguities by hiding the type.  We'll
@@ -146,7 +143,7 @@ Sema::TypeTy *Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
     // that only makes sense if the identifier was treated like a type.
     if (Result.getAmbiguityKind() == LookupResult::AmbiguousTagHiding) {
       Result.suppressDiagnostics();
-      return 0;
+      return ParsedType();
     }
 
     // Look to see if we have a type anywhere in the list of results.
@@ -168,7 +165,7 @@ Sema::TypeTy *Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
       // will produce the ambiguity, or will complain that it expected
       // a type name.
       Result.suppressDiagnostics();
-      return 0;
+      return ParsedType();
     }
 
     // We found a type within the ambiguous lookup; diagnose the
@@ -199,10 +196,10 @@ Sema::TypeTy *Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
   } else {
     // If it's not plausibly a type, suppress diagnostics.
     Result.suppressDiagnostics();
-    return 0;
+    return ParsedType();
   }
 
-  return T.getAsOpaquePtr();
+  return ParsedType::make(T);
 }
 
 /// isTagName() - This method is called *for error recovery purposes only*
@@ -233,9 +230,9 @@ bool Sema::DiagnoseUnknownTypeName(const IdentifierInfo &II,
                                    SourceLocation IILoc,
                                    Scope *S,
                                    CXXScopeSpec *SS,
-                                   TypeTy *&SuggestedType) {
+                                   ParsedType &SuggestedType) {
   // We don't have anything to suggest (yet).
-  SuggestedType = 0;
+  SuggestedType = ParsedType();
   
   // There may have been a typo in the name of the type. Look up typo
   // results, in case we have something that we can suggest.
@@ -282,7 +279,8 @@ bool Sema::DiagnoseUnknownTypeName(const IdentifierInfo &II,
     CXXScopeSpec EmptySS;
     TemplateTy TemplateResult;
     bool MemberOfUnknownSpecialization;
-    if (isTemplateName(S, SS ? *SS : EmptySS, Name, 0, true, TemplateResult,
+    if (isTemplateName(S, SS ? *SS : EmptySS, /*hasTemplateKeyword=*/false,
+                       Name, ParsedType(), true, TemplateResult,
                        MemberOfUnknownSpecialization) == TNK_Type_template) {
       TemplateName TplName = TemplateResult.getAsVal<TemplateName>();
       Diag(IILoc, diag::err_template_missing_args) << TplName;
@@ -343,8 +341,10 @@ DeclContext *Sema::getContainingDC(DeclContext *DC) {
     return DC;
   }
 
+  // ObjCMethodDecls are parsed (for some reason) outside the context
+  // of the class.
   if (isa<ObjCMethodDecl>(DC))
-    return Context.getTranslationUnitDecl();
+    return DC->getLexicalParent()->getLexicalParent();
 
   return DC->getLexicalParent();
 }
@@ -360,6 +360,7 @@ void Sema::PopDeclContext() {
   assert(CurContext && "DeclContext imbalance!");
 
   CurContext = getContainingDC(CurContext);
+  assert(CurContext && "Popped translation unit!");
 }
 
 /// EnterDeclaratorContext - Used when we must lookup names in the context
@@ -458,8 +459,8 @@ void Sema::PushOnScopeChains(NamedDecl *D, Scope *S, bool AddToContext) {
   IdentifierResolver::iterator I = IdResolver.begin(D->getDeclName()),
                                IEnd = IdResolver.end();
   for (; I != IEnd; ++I) {
-    if (S->isDeclScope(DeclPtrTy::make(*I)) && D->declarationReplaces(*I)) {
-      S->RemoveDecl(DeclPtrTy::make(*I));
+    if (S->isDeclScope(*I) && D->declarationReplaces(*I)) {
+      S->RemoveDecl(*I);
       IdResolver.RemoveDecl(*I);
 
       // Should only need to replace one decl.
@@ -467,7 +468,7 @@ void Sema::PushOnScopeChains(NamedDecl *D, Scope *S, bool AddToContext) {
     }
   }
 
-  S->AddDecl(DeclPtrTy::make(D));
+  S->AddDecl(D);
   IdResolver.AddDecl(D);
 }
 
@@ -475,6 +476,17 @@ bool Sema::isDeclInScope(NamedDecl *&D, DeclContext *Ctx, Scope *S) {
   return IdResolver.isDeclInScope(D, Ctx, Context, S);
 }
 
+Scope *Sema::getScopeForDeclContext(Scope *S, DeclContext *DC) {
+  DeclContext *TargetDC = DC->getPrimaryContext();
+  do {
+    if (DeclContext *ScopeDC = (DeclContext*) S->getEntity())
+      if (ScopeDC->getPrimaryContext() == TargetDC)
+        return S;
+  } while ((S = S->getParent()));
+
+  return 0;
+}
+
 static bool isOutOfScopePreviousDeclaration(NamedDecl *,
                                             DeclContext*,
                                             ASTContext&);
@@ -517,6 +529,90 @@ static void RemoveUsingDecls(LookupResult &R) {
   F.done();
 }
 
+/// \brief Check for this common pattern:
+/// @code
+/// class S {
+///   S(const S&); // DO NOT IMPLEMENT
+///   void operator=(const S&); // DO NOT IMPLEMENT
+/// };
+/// @endcode
+static bool IsDisallowedCopyOrAssign(const CXXMethodDecl *D) {
+  // FIXME: Should check for private access too but access is set after we get
+  // the decl here.
+  if (D->isThisDeclarationADefinition())
+    return false;
+
+  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(D))
+    return CD->isCopyConstructor();
+  return D->isCopyAssignment();
+}
+
+bool Sema::ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const {
+  assert(D);
+
+  if (D->isInvalidDecl() || D->isUsed() || D->hasAttr<UnusedAttr>())
+    return false;
+
+  // Ignore class templates.
+  if (D->getDeclContext()->isDependentContext())
+    return false;
+
+  // We warn for unused decls internal to the translation unit.
+  if (D->getLinkage() == ExternalLinkage)
+    return false;
+
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return false;
+
+    if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) {
+      if (MD->isVirtual() || IsDisallowedCopyOrAssign(MD))
+        return false;
+    } else {
+      // 'static inline' functions are used in headers; don't warn.
+      if (FD->getStorageClass() == SC_Static &&
+          FD->isInlineSpecified())
+        return false;
+    }
+
+    if (FD->isThisDeclarationADefinition())
+      return !Context.DeclMustBeEmitted(FD);
+    return true;
+  }
+
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    if (VD->isStaticDataMember() &&
+        VD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return false;
+
+    if ( VD->isFileVarDecl() &&
+        !VD->getType().isConstant(Context))
+      return !Context.DeclMustBeEmitted(VD);
+  }
+
+   return false;
+ }
+
+ void Sema::MarkUnusedFileScopedDecl(const DeclaratorDecl *D) {
+  if (!D)
+    return;
+
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    const FunctionDecl *First = FD->getFirstDeclaration();
+    if (FD != First && ShouldWarnIfUnusedFileScopedDecl(First))
+      return; // First should already be in the vector.
+  }
+
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    const VarDecl *First = VD->getFirstDeclaration();
+    if (VD != First && ShouldWarnIfUnusedFileScopedDecl(First))
+      return; // First should already be in the vector.
+  }
+
+   if (ShouldWarnIfUnusedFileScopedDecl(D))
+     UnusedFileScopedDecls.push_back(D);
+ }
+
 static bool ShouldDiagnoseUnusedDecl(const NamedDecl *D) {
   if (D->isInvalidDecl())
     return false;
@@ -585,7 +681,7 @@ void Sema::ActOnPopScope(SourceLocation Loc, Scope *S) {
 
   for (Scope::decl_iterator I = S->decl_begin(), E = S->decl_end();
        I != E; ++I) {
-    Decl *TmpD = (*I).getAs<Decl>();
+    Decl *TmpD = (*I);
     assert(TmpD && "This decl didn't get pushed??");
 
     assert(isa<NamedDecl>(TmpD) && "Decl isn't NamedDecl?");
@@ -731,8 +827,8 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
   FunctionDecl *New = FunctionDecl::Create(Context,
                                            Context.getTranslationUnitDecl(),
                                            Loc, II, R, /*TInfo=*/0,
-                                           FunctionDecl::Extern,
-                                           FunctionDecl::None, false,
+                                           SC_Extern,
+                                           SC_None, false,
                                            /*hasPrototype=*/true);
   New->setImplicit();
 
@@ -743,7 +839,7 @@ NamedDecl *Sema::LazilyCreateBuiltin(IdentifierInfo *II, unsigned bid,
     for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i)
       Params.push_back(ParmVarDecl::Create(Context, New, SourceLocation(), 0,
                                            FT->getArgType(i), /*TInfo=*/0,
-                                           VarDecl::None, VarDecl::None, 0));
+                                           SC_None, SC_None, 0));
     New->setParams(Params.data(), Params.size());
   }
 
@@ -909,25 +1005,40 @@ void Sema::MergeTypeDefDecl(TypedefDecl *New, LookupResult &OldDecls) {
 /// DeclhasAttr - returns true if decl Declaration already has the target
 /// attribute.
 static bool
-DeclHasAttr(const Decl *decl, const Attr *target) {
-  for (const Attr *attr = decl->getAttrs(); attr; attr = attr->getNext())
-    if (attr->getKind() == target->getKind())
+DeclHasAttr(const Decl *D, const Attr *A) {
+  const OwnershipAttr *OA = dyn_cast<OwnershipAttr>(A);
+  for (Decl::attr_iterator i = D->attr_begin(), e = D->attr_end(); i != e; ++i)
+    if ((*i)->getKind() == A->getKind()) {
+      // FIXME: Don't hardcode this check
+      if (OA && isa<OwnershipAttr>(*i))
+        return OA->getOwnKind() == cast<OwnershipAttr>(*i)->getOwnKind();
       return true;
+    }
 
   return false;
 }
 
-/// MergeAttributes - append attributes from the Old decl to the New one.
-static void MergeAttributes(Decl *New, Decl *Old, ASTContext &C) {
-  for (const Attr *attr = Old->getAttrs(); attr; attr = attr->getNext()) {
-    if (!DeclHasAttr(New, attr) && attr->isMerged()) {
-      Attr *NewAttr = attr->clone(C);
+/// MergeDeclAttributes - append attributes from the Old decl to the New one.
+static void MergeDeclAttributes(Decl *New, Decl *Old, ASTContext &C) {
+  if (!Old->hasAttrs())
+    return;
+  // Ensure that any moving of objects within the allocated map is done before
+  // we process them.
+  if (!New->hasAttrs())
+    New->setAttrs(AttrVec());
+  for (Decl::attr_iterator i = Old->attr_begin(), e = Old->attr_end(); i != e;
+       ++i) {
+    // FIXME: Make this more general than just checking for Overloadable.
+    if (!DeclHasAttr(New, *i) && (*i)->getKind() != attr::Overloadable) {
+      Attr *NewAttr = (*i)->clone(C);
       NewAttr->setInherited(true);
       New->addAttr(NewAttr);
     }
   }
 }
 
+namespace {
+
 /// Used in MergeFunctionDecl to keep track of function parameters in
 /// C.
 struct GNUCompatibleParamWarning {
@@ -936,6 +1047,7 @@ struct GNUCompatibleParamWarning {
   QualType PromotedType;
 };
 
+}
 
 /// getSpecialMember - get the special member enum for a method.
 Sema::CXXSpecialMember Sema::getSpecialMember(const CXXMethodDecl *MD) {
@@ -960,7 +1072,7 @@ static bool canRedefineFunction(const FunctionDecl *FD,
                                 const LangOptions& LangOpts) {
   return (LangOpts.GNUMode && !LangOpts.C99 && !LangOpts.CPlusPlus &&
           FD->isInlineSpecified() &&
-          FD->getStorageClass() == FunctionDecl::Extern);
+          FD->getStorageClass() == SC_Extern);
 }
 
 /// MergeFunctionDecl - We just parsed a function 'New' from
@@ -1014,8 +1126,8 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
   // Don't complain about this if we're in GNU89 mode and the old function
   // is an extern inline function.
   if (!isa<CXXMethodDecl>(New) && !isa<CXXMethodDecl>(Old) &&
-      New->getStorageClass() == FunctionDecl::Static &&
-      Old->getStorageClass() != FunctionDecl::Static &&
+      New->getStorageClass() == SC_Static &&
+      Old->getStorageClass() != SC_Static &&
       !canRedefineFunction(Old, getLangOptions())) {
     Diag(New->getLocation(), diag::err_static_non_static)
       << New;
@@ -1196,7 +1308,7 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
         ParmVarDecl *Param = ParmVarDecl::Create(Context, New,
                                                  SourceLocation(), 0,
                                                  *ParamType, /*TInfo=*/0,
-                                                 VarDecl::None, VarDecl::None,
+                                                 SC_None, SC_None,
                                                  0);
         Param->setImplicit();
         Params.push_back(Param);
@@ -1242,7 +1354,8 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
                                      NewProto->getArgType(Idx))) {
         ArgTypes.push_back(NewParm->getType());
       } else if (Context.typesAreCompatible(OldParm->getType(),
-                                            NewParm->getType())) {
+                                            NewParm->getType(),
+                                            /*CompareUnqualified=*/true)) {
         GNUCompatibleParamWarning Warn
           = { OldParm, NewParm, NewProto->getArgType(Idx) };
         Warnings.push_back(Warn);
@@ -1257,8 +1370,9 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
              diag::ext_param_promoted_not_compatible_with_prototype)
           << Warnings[Warn].PromotedType
           << Warnings[Warn].OldParm->getType();
-        Diag(Warnings[Warn].OldParm->getLocation(),
-             diag::note_previous_declaration);
+        if (Warnings[Warn].OldParm->getLocation().isValid())
+          Diag(Warnings[Warn].OldParm->getLocation(),
+               diag::note_previous_declaration);
       }
 
       New->setType(Context.getFunctionType(MergedReturn, &ArgTypes[0],
@@ -1309,11 +1423,11 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD) {
 /// \returns false
 bool Sema::MergeCompatibleFunctionDecls(FunctionDecl *New, FunctionDecl *Old) {
   // Merge the attributes
-  MergeAttributes(New, Old, Context);
+  MergeDeclAttributes(New, Old, Context);
 
   // Merge the storage class.
-  if (Old->getStorageClass() != FunctionDecl::Extern &&
-      Old->getStorageClass() != FunctionDecl::None)
+  if (Old->getStorageClass() != SC_Extern &&
+      Old->getStorageClass() != SC_None)
     New->setStorageClass(Old->getStorageClass());
 
   // Merge "pure" flag.
@@ -1354,7 +1468,18 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
     return New->setInvalidDecl();
   }
 
-  MergeAttributes(New, Old, Context);
+  // C++ [class.mem]p1:
+  //   A member shall not be declared twice in the member-specification [...]
+  // 
+  // Here, we need only consider static data members.
+  if (Old->isStaticDataMember() && !New->isOutOfLine()) {
+    Diag(New->getLocation(), diag::err_duplicate_member) 
+      << New->getIdentifier();
+    Diag(Old->getLocation(), diag::note_previous_declaration);
+    New->setInvalidDecl();
+  }
+  
+  MergeDeclAttributes(New, Old, Context);
 
   // Merge the types
   QualType MergedT;
@@ -1398,8 +1523,8 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
   New->setType(MergedT);
 
   // C99 6.2.2p4: Check if we have a static decl followed by a non-static.
-  if (New->getStorageClass() == VarDecl::Static &&
-      (Old->getStorageClass() == VarDecl::None || Old->hasExternalStorage())) {
+  if (New->getStorageClass() == SC_Static &&
+      (Old->getStorageClass() == SC_None || Old->hasExternalStorage())) {
     Diag(New->getLocation(), diag::err_static_non_static) << New->getDeclName();
     Diag(Old->getLocation(), diag::note_previous_definition);
     return New->setInvalidDecl();
@@ -1415,8 +1540,8 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
   //   identifier has external linkage.
   if (New->hasExternalStorage() && Old->hasLinkage())
     /* Okay */;
-  else if (New->getStorageClass() != VarDecl::Static &&
-           Old->getStorageClass() == VarDecl::Static) {
+  else if (New->getStorageClass() != SC_Static &&
+           Old->getStorageClass() == SC_Static) {
     Diag(New->getLocation(), diag::err_non_static_static) << New->getDeclName();
     Diag(Old->getLocation(), diag::note_previous_definition);
     return New->setInvalidDecl();
@@ -1475,8 +1600,8 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
 
 /// ParsedFreeStandingDeclSpec - This method is invoked when a declspec with
 /// no declarator (e.g. "struct foo;") is parsed.
-Sema::DeclPtrTy Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
-                                                 DeclSpec &DS) {
+Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
+                                            DeclSpec &DS) {
   // FIXME: Error on auto/register at file scope
   // FIXME: Error on inline/virtual/explicit
   // FIXME: Warn on useless __thread
@@ -1489,10 +1614,10 @@ Sema::DeclPtrTy Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
       DS.getTypeSpecType() == DeclSpec::TST_struct ||
       DS.getTypeSpecType() == DeclSpec::TST_union ||
       DS.getTypeSpecType() == DeclSpec::TST_enum) {
-    TagD = static_cast<Decl *>(DS.getTypeRep());
+    TagD = DS.getRepAsDecl();
 
     if (!TagD) // We probably had an error
-      return DeclPtrTy();
+      return 0;
 
     // Note that the above type specs guarantee that the
     // type rep is a Decl, whereas in many of the others
@@ -1513,14 +1638,12 @@ Sema::DeclPtrTy Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     // If we're dealing with a class template decl, assume that the
     // template routines are handling it.
     if (TagD && isa<ClassTemplateDecl>(TagD))
-      return DeclPtrTy();
+      return 0;
     return ActOnFriendTypeDecl(S, DS, MultiTemplateParamsArg(*this, 0, 0));
   }
          
   if (RecordDecl *Record = dyn_cast_or_null<RecordDecl>(Tag)) {
-    // If there are attributes in the DeclSpec, apply them to the record.
-    if (const AttributeList *AL = DS.getAttributes())
-      ProcessDeclAttributeList(S, Record, AL);
+    ProcessDeclAttributeList(S, Record, DS.getAttributes());
     
     if (!Record->getDeclName() && Record->isDefinition() &&
         DS.getStorageClassSpec() != DeclSpec::SCS_typedef) {
@@ -1536,7 +1659,7 @@ Sema::DeclPtrTy Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     // about them.
     // FIXME: Should we support Microsoft's extensions in this area?
     if (Record->getDeclName() && getLangOptions().Microsoft)
-      return DeclPtrTy::make(Tag);
+      return Tag;
   }
   
   if (getLangOptions().CPlusPlus && 
@@ -1550,19 +1673,19 @@ Sema::DeclPtrTy Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   if (!DS.isMissingDeclaratorOk() &&
       DS.getTypeSpecType() != DeclSpec::TST_error) {
     // Warn about typedefs of enums without names, since this is an
-    // extension in both Microsoft an GNU.
+    // extension in both Microsoft and GNU.
     if (DS.getStorageClassSpec() == DeclSpec::SCS_typedef &&
         Tag && isa<EnumDecl>(Tag)) {
       Diag(DS.getSourceRange().getBegin(), diag::ext_typedef_without_a_name)
         << DS.getSourceRange();
-      return DeclPtrTy::make(Tag);
+      return Tag;
     }
 
     Diag(DS.getSourceRange().getBegin(), diag::ext_no_declarators)
       << DS.getSourceRange();
   }
 
-  return DeclPtrTy::make(Tag);
+  return TagD;
 }
 
 /// We are trying to inject an anonymous member into the given scope;
@@ -1639,7 +1762,7 @@ static bool InjectAnonymousStructOrUnionMembers(Sema &SemaRef, Scope *S,
         //   considered to have been defined in the scope in which the
         //   anonymous union is declared.
         Owner->makeDeclVisibleInContext(*F);
-        S->AddDecl(Sema::DeclPtrTy::make(*F));
+        S->AddDecl(*F);
         SemaRef.IdResolver.AddDecl(*F);
 
         // That includes picking up the appropriate access specifier.
@@ -1660,58 +1783,38 @@ static bool InjectAnonymousStructOrUnionMembers(Sema &SemaRef, Scope *S,
 
 /// StorageClassSpecToVarDeclStorageClass - Maps a DeclSpec::SCS to
 /// a VarDecl::StorageClass. Any error reporting is up to the caller:
-/// illegal input values are mapped to VarDecl::None.
-/// If the input declaration context is a linkage specification
-/// with no braces, then Extern is mapped to None.
-static VarDecl::StorageClass
-StorageClassSpecToVarDeclStorageClass(DeclSpec::SCS StorageClassSpec,
-                                      DeclContext *DC) {
+/// illegal input values are mapped to SC_None.
+static StorageClass
+StorageClassSpecToVarDeclStorageClass(DeclSpec::SCS StorageClassSpec) {
   switch (StorageClassSpec) {
-  case DeclSpec::SCS_unspecified:    return VarDecl::None;
-  case DeclSpec::SCS_extern:
-    // If the current context is a C++ linkage specification
-    // having no braces, then the keyword "extern" is properly part
-    // of the linkage specification itself, rather than being
-    // the written storage class specifier.
-    return (DC && isa<LinkageSpecDecl>(DC) &&
-            !cast<LinkageSpecDecl>(DC)->hasBraces())
-      ? VarDecl::None : VarDecl::Extern;
-  case DeclSpec::SCS_static:         return VarDecl::Static;
-  case DeclSpec::SCS_auto:           return VarDecl::Auto;
-  case DeclSpec::SCS_register:       return VarDecl::Register;
-  case DeclSpec::SCS_private_extern: return VarDecl::PrivateExtern;
+  case DeclSpec::SCS_unspecified:    return SC_None;
+  case DeclSpec::SCS_extern:         return SC_Extern;
+  case DeclSpec::SCS_static:         return SC_Static;
+  case DeclSpec::SCS_auto:           return SC_Auto;
+  case DeclSpec::SCS_register:       return SC_Register;
+  case DeclSpec::SCS_private_extern: return SC_PrivateExtern;
     // Illegal SCSs map to None: error reporting is up to the caller.
   case DeclSpec::SCS_mutable:        // Fall through.
-  case DeclSpec::SCS_typedef:        return VarDecl::None;
+  case DeclSpec::SCS_typedef:        return SC_None;
   }
   llvm_unreachable("unknown storage class specifier");
 }
 
 /// StorageClassSpecToFunctionDeclStorageClass - Maps a DeclSpec::SCS to
-/// a FunctionDecl::StorageClass. Any error reporting is up to the caller:
-/// illegal input values are mapped to FunctionDecl::None.
-/// If the input declaration context is a linkage specification
-/// with no braces, then Extern is mapped to None.
-static FunctionDecl::StorageClass
-StorageClassSpecToFunctionDeclStorageClass(DeclSpec::SCS StorageClassSpec,
-                                           DeclContext *DC) {
+/// a StorageClass. Any error reporting is up to the caller:
+/// illegal input values are mapped to SC_None.
+static StorageClass
+StorageClassSpecToFunctionDeclStorageClass(DeclSpec::SCS StorageClassSpec) {
   switch (StorageClassSpec) {
-  case DeclSpec::SCS_unspecified:    return FunctionDecl::None;
-  case DeclSpec::SCS_extern:
-    // If the current context is a C++ linkage specification
-    // having no braces, then the keyword "extern" is properly part
-    // of the linkage specification itself, rather than being
-    // the written storage class specifier.
-    return (DC && isa<LinkageSpecDecl>(DC) &&
-            !cast<LinkageSpecDecl>(DC)->hasBraces())
-      ? FunctionDecl::None : FunctionDecl::Extern;
-  case DeclSpec::SCS_static:         return FunctionDecl::Static;
-  case DeclSpec::SCS_private_extern: return FunctionDecl::PrivateExtern;
+  case DeclSpec::SCS_unspecified:    return SC_None;
+  case DeclSpec::SCS_extern:         return SC_Extern;
+  case DeclSpec::SCS_static:         return SC_Static;
+  case DeclSpec::SCS_private_extern: return SC_PrivateExtern;
     // Illegal SCSs map to None: error reporting is up to the caller.
   case DeclSpec::SCS_auto:           // Fall through.
   case DeclSpec::SCS_mutable:        // Fall through.
   case DeclSpec::SCS_register:       // Fall through.
-  case DeclSpec::SCS_typedef:        return FunctionDecl::None;
+  case DeclSpec::SCS_typedef:        return SC_None;
   }
   llvm_unreachable("unknown storage class specifier");
 }
@@ -1720,9 +1823,9 @@ StorageClassSpecToFunctionDeclStorageClass(DeclSpec::SCS StorageClassSpec,
 /// anonymous structure or union. Anonymous unions are a C++ feature
 /// (C++ [class.union]) and a GNU C extension; anonymous structures
 /// are a GNU C and GNU C++ extension.
-Sema::DeclPtrTy Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
-                                                  AccessSpecifier AS,
-                                                  RecordDecl *Record) {
+Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
+                                             AccessSpecifier AS,
+                                             RecordDecl *Record) {
   DeclContext *Owner = Record->getDeclContext();
 
   // Diagnose whether this anonymous struct/union is an extension.
@@ -1782,6 +1885,9 @@ Sema::DeclPtrTy Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
             << (int)Record->isUnion() << (int)(FD->getAccess() == AS_protected);
           Invalid = true;
         }
+
+        if (CheckNontrivialField(FD))
+          Invalid = true;
       } else if ((*Mem)->isImplicit()) {
         // Any implicit members are fine.
       } else if (isa<TagDecl>(*Mem) && (*Mem)->getDeclContext() != Record) {
@@ -1845,17 +1951,17 @@ Sema::DeclPtrTy Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
     DeclSpec::SCS SCSpec = DS.getStorageClassSpec();
     assert(SCSpec != DeclSpec::SCS_typedef &&
            "Parser allowed 'typedef' as storage class VarDecl.");
-    VarDecl::StorageClass SC = StorageClassSpecToVarDeclStorageClass(SCSpec, 0);
+    VarDecl::StorageClass SC = StorageClassSpecToVarDeclStorageClass(SCSpec);
     if (SCSpec == DeclSpec::SCS_mutable) {
       // mutable can only appear on non-static class members, so it's always
       // an error here
       Diag(Record->getLocation(), diag::err_mutable_nonmember);
       Invalid = true;
-      SC = VarDecl::None;
+      SC = SC_None;
     }
     SCSpec = DS.getStorageClassSpecAsWritten();
     VarDecl::StorageClass SCAsWritten
-      = StorageClassSpecToVarDeclStorageClass(SCSpec, 0);
+      = StorageClassSpecToVarDeclStorageClass(SCSpec);
 
     Anon = VarDecl::Create(Context, Owner, Record->getLocation(),
                            /*IdentifierInfo=*/0,
@@ -1886,85 +1992,115 @@ Sema::DeclPtrTy Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
   if (Invalid)
     Anon->setInvalidDecl();
 
-  return DeclPtrTy::make(Anon);
+  return Anon;
 }
 
 
 /// GetNameForDeclarator - Determine the full declaration name for the
 /// given Declarator.
-DeclarationName Sema::GetNameForDeclarator(Declarator &D) {
+DeclarationNameInfo Sema::GetNameForDeclarator(Declarator &D) {
   return GetNameFromUnqualifiedId(D.getName());
 }
 
-/// \brief Retrieves the canonicalized name from a parsed unqualified-id.
-DeclarationName Sema::GetNameFromUnqualifiedId(const UnqualifiedId &Name) {
-  switch (Name.getKind()) {
-    case UnqualifiedId::IK_Identifier:
-      return DeclarationName(Name.Identifier);
-      
-    case UnqualifiedId::IK_OperatorFunctionId:
-      return Context.DeclarationNames.getCXXOperatorName(
-                                              Name.OperatorFunctionId.Operator);
-
-    case UnqualifiedId::IK_LiteralOperatorId:
-      return Context.DeclarationNames.getCXXLiteralOperatorName(
-                                                               Name.Identifier);
-
-    case UnqualifiedId::IK_ConversionFunctionId: {
-      QualType Ty = GetTypeFromParser(Name.ConversionFunctionId);
-      if (Ty.isNull())
-        return DeclarationName();
-      
-      return Context.DeclarationNames.getCXXConversionFunctionName(
-                                                  Context.getCanonicalType(Ty));
-    }
-      
-    case UnqualifiedId::IK_ConstructorName: {
-      QualType Ty = GetTypeFromParser(Name.ConstructorName);
-      if (Ty.isNull())
-        return DeclarationName();
-      
-      return Context.DeclarationNames.getCXXConstructorName(
-                                                  Context.getCanonicalType(Ty));
-    }
-      
-    case UnqualifiedId::IK_ConstructorTemplateId: {
-      // In well-formed code, we can only have a constructor
-      // template-id that refers to the current context, so go there
-      // to find the actual type being constructed.
-      CXXRecordDecl *CurClass = dyn_cast<CXXRecordDecl>(CurContext);
-      if (!CurClass || CurClass->getIdentifier() != Name.TemplateId->Name)
-        return DeclarationName();
+/// \brief Retrieves the declaration name from a parsed unqualified-id.
+DeclarationNameInfo
+Sema::GetNameFromUnqualifiedId(const UnqualifiedId &Name) {
+  DeclarationNameInfo NameInfo;
+  NameInfo.setLoc(Name.StartLocation);
 
-      // Determine the type of the class being constructed.
-      QualType CurClassType = Context.getTypeDeclType(CurClass);
+  switch (Name.getKind()) {
 
-      // FIXME: Check two things: that the template-id names the same type as
-      // CurClassType, and that the template-id does not occur when the name
-      // was qualified.
+  case UnqualifiedId::IK_Identifier:
+    NameInfo.setName(Name.Identifier);
+    NameInfo.setLoc(Name.StartLocation);
+    return NameInfo;
+
+  case UnqualifiedId::IK_OperatorFunctionId:
+    NameInfo.setName(Context.DeclarationNames.getCXXOperatorName(
+                                           Name.OperatorFunctionId.Operator));
+    NameInfo.setLoc(Name.StartLocation);
+    NameInfo.getInfo().CXXOperatorName.BeginOpNameLoc
+      = Name.OperatorFunctionId.SymbolLocations[0];
+    NameInfo.getInfo().CXXOperatorName.EndOpNameLoc
+      = Name.EndLocation.getRawEncoding();
+    return NameInfo;
+
+  case UnqualifiedId::IK_LiteralOperatorId:
+    NameInfo.setName(Context.DeclarationNames.getCXXLiteralOperatorName(
+                                                           Name.Identifier));
+    NameInfo.setLoc(Name.StartLocation);
+    NameInfo.setCXXLiteralOperatorNameLoc(Name.EndLocation);
+    return NameInfo;
+
+  case UnqualifiedId::IK_ConversionFunctionId: {
+    TypeSourceInfo *TInfo;
+    QualType Ty = GetTypeFromParser(Name.ConversionFunctionId, &TInfo);
+    if (Ty.isNull())
+      return DeclarationNameInfo();
+    NameInfo.setName(Context.DeclarationNames.getCXXConversionFunctionName(
+                                               Context.getCanonicalType(Ty)));
+    NameInfo.setLoc(Name.StartLocation);
+    NameInfo.setNamedTypeInfo(TInfo);
+    return NameInfo;
+  }
+
+  case UnqualifiedId::IK_ConstructorName: {
+    TypeSourceInfo *TInfo;
+    QualType Ty = GetTypeFromParser(Name.ConstructorName, &TInfo);
+    if (Ty.isNull())
+      return DeclarationNameInfo();
+    NameInfo.setName(Context.DeclarationNames.getCXXConstructorName(
+                                              Context.getCanonicalType(Ty)));
+    NameInfo.setLoc(Name.StartLocation);
+    NameInfo.setNamedTypeInfo(TInfo);
+    return NameInfo;
+  }
+
+  case UnqualifiedId::IK_ConstructorTemplateId: {
+    // In well-formed code, we can only have a constructor
+    // template-id that refers to the current context, so go there
+    // to find the actual type being constructed.
+    CXXRecordDecl *CurClass = dyn_cast<CXXRecordDecl>(CurContext);
+    if (!CurClass || CurClass->getIdentifier() != Name.TemplateId->Name)
+      return DeclarationNameInfo();
+
+    // Determine the type of the class being constructed.
+    QualType CurClassType = Context.getTypeDeclType(CurClass);
+
+    // FIXME: Check two things: that the template-id names the same type as
+    // CurClassType, and that the template-id does not occur when the name
+    // was qualified.
+
+    NameInfo.setName(Context.DeclarationNames.getCXXConstructorName(
+                                    Context.getCanonicalType(CurClassType)));
+    NameInfo.setLoc(Name.StartLocation);
+    // FIXME: should we retrieve TypeSourceInfo?
+    NameInfo.setNamedTypeInfo(0);
+    return NameInfo;
+  }
+
+  case UnqualifiedId::IK_DestructorName: {
+    TypeSourceInfo *TInfo;
+    QualType Ty = GetTypeFromParser(Name.DestructorName, &TInfo);
+    if (Ty.isNull())
+      return DeclarationNameInfo();
+    NameInfo.setName(Context.DeclarationNames.getCXXDestructorName(
+                                              Context.getCanonicalType(Ty)));
+    NameInfo.setLoc(Name.StartLocation);
+    NameInfo.setNamedTypeInfo(TInfo);
+    return NameInfo;
+  }
+
+  case UnqualifiedId::IK_TemplateId: {
+    TemplateName TName = Name.TemplateId->Template.get();
+    SourceLocation TNameLoc = Name.TemplateId->TemplateNameLoc;
+    return Context.getNameForTemplate(TName, TNameLoc);
+  }
+
+  } // switch (Name.getKind())
 
-      return Context.DeclarationNames.getCXXConstructorName(
-                                       Context.getCanonicalType(CurClassType));
-    }
-
-    case UnqualifiedId::IK_DestructorName: {
-      QualType Ty = GetTypeFromParser(Name.DestructorName);
-      if (Ty.isNull())
-        return DeclarationName();
-      
-      return Context.DeclarationNames.getCXXDestructorName(
-                                                           Context.getCanonicalType(Ty));
-    }
-      
-    case UnqualifiedId::IK_TemplateId: {
-      TemplateName TName
-        = TemplateName::getFromVoidPointer(Name.TemplateId->Template);
-      return Context.getNameForTemplate(TName);
-    }
-  }
-  
   assert(false && "Unknown name kind");
-  return DeclarationName();  
+  return DeclarationNameInfo();
 }
 
 /// isNearlyMatchingFunction - Determine whether the C++ functions
@@ -2007,11 +2143,10 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
   switch (DS.getTypeSpecType()) {
   case DeclSpec::TST_typename:
   case DeclSpec::TST_typeofType:
-  case DeclSpec::TST_typeofExpr:
   case DeclSpec::TST_decltype: {
     // Grab the type from the parser.
     TypeSourceInfo *TSI = 0;
-    QualType T = S.GetTypeFromParser(DS.getTypeRep(), &TSI);
+    QualType T = S.GetTypeFromParser(DS.getRepAsType(), &TSI);
     if (T.isNull() || !T->isDependentType()) break;
 
     // Make sure there's a type source info.  This isn't really much
@@ -2025,8 +2160,16 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
     if (!TSI) return true;
 
     // Store the new type back in the decl spec.
-    QualType LocType = S.CreateLocInfoType(TSI->getType(), TSI);
-    DS.UpdateTypeRep(LocType.getAsOpaquePtr());
+    ParsedType LocType = S.CreateParsedType(TSI->getType(), TSI);
+    DS.UpdateTypeRep(LocType);
+    break;
+  }
+
+  case DeclSpec::TST_typeofExpr: {
+    Expr *E = DS.getRepAsExpr();
+    ExprResult Result = S.RebuildExprInCurrentInstantiation(E);
+    if (Result.isInvalid()) return true;
+    DS.UpdateExprRep(Result.get());
     break;
   }
 
@@ -2054,11 +2197,16 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
   return false;
 }
 
-Sema::DeclPtrTy
-Sema::HandleDeclarator(Scope *S, Declarator &D,
-                       MultiTemplateParamsArg TemplateParamLists,
-                       bool IsFunctionDefinition) {
-  DeclarationName Name = GetNameForDeclarator(D);
+Decl *Sema::ActOnDeclarator(Scope *S, Declarator &D) {
+  return HandleDeclarator(S, D, MultiTemplateParamsArg(*this), false);
+}
+
+Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
+                             MultiTemplateParamsArg TemplateParamLists,
+                             bool IsFunctionDefinition) {
+  // TODO: consider using NameInfo for diagnostic.
+  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
 
   // All of these full declarators require an identifier.  If it doesn't have
   // one, the ParsedFreeStandingDeclSpec action should be used.
@@ -2067,7 +2215,7 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
       Diag(D.getDeclSpec().getSourceRange().getBegin(),
            diag::err_declarator_need_ident)
         << D.getDeclSpec().getSourceRange() << D.getSourceRange();
-    return DeclPtrTy();
+    return 0;
   }
 
   // The scope passed in may not be a decl scope.  Zip up the scope tree until
@@ -2091,14 +2239,14 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
            diag::err_template_qualified_declarator_no_match)
         << (NestedNameSpecifier*)D.getCXXScopeSpec().getScopeRep()
         << D.getCXXScopeSpec().getRange();
-      return DeclPtrTy();
+      return 0;
     }
 
     bool IsDependentContext = DC->isDependentContext();
 
     if (!IsDependentContext && 
         RequireCompleteDeclContext(D.getCXXScopeSpec(), DC))
-      return DeclPtrTy();
+      return 0;
 
     if (isa<CXXRecordDecl>(DC) && !cast<CXXRecordDecl>(DC)->hasDefinition()) {
       Diag(D.getIdentifierLoc(),
@@ -2122,7 +2270,7 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
   QualType R = TInfo->getType();
 
-  LookupResult Previous(*this, Name, D.getIdentifierLoc(), LookupOrdinaryName,
+  LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
                         ForRedeclaration);
 
   // See if this is a redefinition of a variable in the same scope.
@@ -2140,7 +2288,7 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
         IsLinkageLookup = true;
     } else if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_extern)
       IsLinkageLookup = true;
-    else if (CurContext->getLookupContext()->isTranslationUnit() &&
+    else if (CurContext->getRedeclContext()->isTranslationUnit() &&
              D.getDeclSpec().getStorageClassSpec() != DeclSpec::SCS_static)
       IsLinkageLookup = true;
 
@@ -2225,7 +2373,7 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
   if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_typedef) {
     if (TemplateParamLists.size()) {
       Diag(D.getIdentifierLoc(), diag::err_template_typedef);
-      return DeclPtrTy();
+      return 0;
     }
 
     New = ActOnTypedefDeclarator(S, D, DC, R, TInfo, Previous, Redeclaration);
@@ -2240,14 +2388,14 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
   }
 
   if (New == 0)
-    return DeclPtrTy();
+    return 0;
 
   // If this has an identifier and is not an invalid redeclaration or 
   // function template specialization, add it to the scope stack.
-  if (Name && !(Redeclaration && New->isInvalidDecl()))
+  if (New->getDeclName() && !(Redeclaration && New->isInvalidDecl()))
     PushOnScopeChains(New, S);
 
-  return DeclPtrTy::make(New);
+  return New;
 }
 
 /// TryToFixInvalidVariablyModifiedType - Helper method to turn variable array
@@ -2255,20 +2403,26 @@ Sema::HandleDeclarator(Scope *S, Declarator &D,
 /// be errors (for GCC compatibility).
 static QualType TryToFixInvalidVariablyModifiedType(QualType T,
                                                     ASTContext &Context,
-                                                    bool &SizeIsNegative) {
+                                                    bool &SizeIsNegative,
+                                                    llvm::APSInt &Oversized) {
   // This method tries to turn a variable array into a constant
   // array even when the size isn't an ICE.  This is necessary
   // for compatibility with code that depends on gcc's buggy
   // constant expression folding, like struct {char x[(int)(char*)2];}
   SizeIsNegative = false;
-
+  Oversized = 0;
+  
+  if (T->isDependentType())
+    return QualType();
+  
   QualifierCollector Qs;
   const Type *Ty = Qs.strip(T);
 
   if (const PointerType* PTy = dyn_cast<PointerType>(Ty)) {
     QualType Pointee = PTy->getPointeeType();
     QualType FixedType =
-        TryToFixInvalidVariablyModifiedType(Pointee, Context, SizeIsNegative);
+        TryToFixInvalidVariablyModifiedType(Pointee, Context, SizeIsNegative,
+                                            Oversized);
     if (FixedType.isNull()) return FixedType;
     FixedType = Context.getPointerType(FixedType);
     return Qs.apply(FixedType);
@@ -2287,15 +2441,24 @@ static QualType TryToFixInvalidVariablyModifiedType(QualType T,
       !EvalResult.Val.isInt())
     return QualType();
 
+  // Check whether the array size is negative.
   llvm::APSInt &Res = EvalResult.Val.getInt();
-  if (Res >= llvm::APSInt(Res.getBitWidth(), Res.isUnsigned())) {
-    // TODO: preserve the size expression in declarator info
-    return Context.getConstantArrayType(VLATy->getElementType(),
-                                        Res, ArrayType::Normal, 0);
+  if (Res.isSigned() && Res.isNegative()) {
+    SizeIsNegative = true;
+    return QualType();
   }
 
-  SizeIsNegative = true;
-  return QualType();
+  // Check whether the array is too large to be addressed.
+  unsigned ActiveSizeBits
+    = ConstantArrayType::getNumAddressingBits(Context, VLATy->getElementType(),
+                                              Res);
+  if (ActiveSizeBits > ConstantArrayType::getMaxSizeBits(Context)) {
+    Oversized = Res;
+    return QualType();
+  }
+  
+  return Context.getConstantArrayType(VLATy->getElementType(),
+                                      Res, ArrayType::Normal, 0);
 }
 
 /// \brief Register the given locally-scoped external C declaration so
@@ -2320,11 +2483,11 @@ Sema::RegisterLocallyScopedExternCDecl(NamedDecl *ND,
   if (S && IdResolver.ReplaceDecl(PrevDecl, ND)) {
     // The previous declaration was found on the identifer resolver
     // chain, so remove it from its scope.
-    while (S && !S->isDeclScope(DeclPtrTy::make(PrevDecl)))
+    while (S && !S->isDeclScope(PrevDecl))
       S = S->getParent();
 
     if (S)
-      S->RemoveDecl(DeclPtrTy::make(PrevDecl));
+      S->RemoveDecl(PrevDecl);
   }
 }
 
@@ -2382,24 +2545,20 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   // Handle attributes prior to checking for duplicates in MergeVarDecl
   ProcessDeclAttributes(S, NewTD, D);
 
-  // Merge the decl with the existing one if appropriate. If the decl is
-  // in an outer scope, it isn't the same thing.
-  FilterLookupForScope(*this, Previous, DC, S, /*ConsiderLinkage*/ false);
-  if (!Previous.empty()) {
-    Redeclaration = true;
-    MergeTypeDefDecl(NewTD, Previous);
-  }
-
   // C99 6.7.7p2: If a typedef name specifies a variably modified type
   // then it shall have block scope.
+  // Note that variably modified types must be fixed before merging the decl so
+  // that redeclarations will match.
   QualType T = NewTD->getUnderlyingType();
   if (T->isVariablyModifiedType()) {
-    FunctionNeedsScopeChecking() = true;
+    getCurFunction()->setHasBranchProtectedScope();
 
     if (S->getFnParent() == 0) {
       bool SizeIsNegative;
+      llvm::APSInt Oversized;
       QualType FixedTy =
-          TryToFixInvalidVariablyModifiedType(T, Context, SizeIsNegative);
+          TryToFixInvalidVariablyModifiedType(T, Context, SizeIsNegative,
+                                              Oversized);
       if (!FixedTy.isNull()) {
         Diag(D.getIdentifierLoc(), diag::warn_illegal_constant_array_size);
         NewTD->setTypeSourceInfo(Context.getTrivialTypeSourceInfo(FixedTy));
@@ -2408,6 +2567,9 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
           Diag(D.getIdentifierLoc(), diag::err_typecheck_negative_array_size);
         else if (T->isVariableArrayType())
           Diag(D.getIdentifierLoc(), diag::err_vla_decl_in_file_scope);
+        else if (Oversized.getBoolValue())
+          Diag(D.getIdentifierLoc(), diag::err_array_too_large)
+            << Oversized.toString(10);
         else
           Diag(D.getIdentifierLoc(), diag::err_vm_decl_in_file_scope);
         NewTD->setInvalidDecl();
@@ -2415,10 +2577,18 @@ Sema::ActOnTypedefDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     }
   }
 
+  // Merge the decl with the existing one if appropriate. If the decl is
+  // in an outer scope, it isn't the same thing.
+  FilterLookupForScope(*this, Previous, DC, S, /*ConsiderLinkage*/ false);
+  if (!Previous.empty()) {
+    Redeclaration = true;
+    MergeTypeDefDecl(NewTD, Previous);
+  }
+
   // If this is the C FILE type, notify the AST context.
   if (IdentifierInfo *II = NewTD->getIdentifier())
     if (!NewTD->isInvalidDecl() &&
-        NewTD->getDeclContext()->getLookupContext()->isTranslationUnit()) {
+        NewTD->getDeclContext()->getRedeclContext()->isTranslationUnit()) {
       if (II->isStr("FILE"))
         Context.setFILEDecl(NewTD);
       else if (II->isStr("jmp_buf"))
@@ -2452,7 +2622,7 @@ static bool
 isOutOfScopePreviousDeclaration(NamedDecl *PrevDecl, DeclContext *DC,
                                 ASTContext &Context) {
   if (!PrevDecl)
-    return 0;
+    return false;
 
   if (!PrevDecl->hasLinkage())
     return false;
@@ -2464,30 +2634,25 @@ isOutOfScopePreviousDeclaration(NamedDecl *PrevDecl, DeclContext *DC,
     //   outside the innermost enclosing namespace scope, the block
     //   scope declaration declares that same entity and receives the
     //   linkage of the previous declaration.
-    DeclContext *OuterContext = DC->getLookupContext();
+    DeclContext *OuterContext = DC->getRedeclContext();
     if (!OuterContext->isFunctionOrMethod())
       // This rule only applies to block-scope declarations.
       return false;
-    else {
-      DeclContext *PrevOuterContext = PrevDecl->getDeclContext();
-      if (PrevOuterContext->isRecord())
-        // We found a member function: ignore it.
-        return false;
-      else {
-        // Find the innermost enclosing namespace for the new and
-        // previous declarations.
-        while (!OuterContext->isFileContext())
-          OuterContext = OuterContext->getParent();
-        while (!PrevOuterContext->isFileContext())
-          PrevOuterContext = PrevOuterContext->getParent();
-
-        // The previous declaration is in a different namespace, so it
-        // isn't the same function.
-        if (OuterContext->getPrimaryContext() !=
-            PrevOuterContext->getPrimaryContext())
-          return false;
-      }
-    }
+    
+    DeclContext *PrevOuterContext = PrevDecl->getDeclContext();
+    if (PrevOuterContext->isRecord())
+      // We found a member function: ignore it.
+      return false;
+    
+    // Find the innermost enclosing namespace for the new and
+    // previous declarations.
+    OuterContext = OuterContext->getEnclosingNamespaceContext();
+    PrevOuterContext = PrevOuterContext->getEnclosingNamespaceContext();
+
+    // The previous declaration is in a different namespace, so it
+    // isn't the same function.
+    if (!OuterContext->Equals(PrevOuterContext))
+      return false;
   }
 
   return true;
@@ -2506,7 +2671,7 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
                               LookupResult &Previous,
                               MultiTemplateParamsArg TemplateParamLists,
                               bool &Redeclaration) {
-  DeclarationName Name = GetNameForDeclarator(D);
+  DeclarationName Name = GetNameForDeclarator(D).getName();
 
   // Check that there are no default arguments (C++ only).
   if (getLangOptions().CPlusPlus)
@@ -2515,17 +2680,17 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   DeclSpec::SCS SCSpec = D.getDeclSpec().getStorageClassSpec();
   assert(SCSpec != DeclSpec::SCS_typedef &&
          "Parser allowed 'typedef' as storage class VarDecl.");
-  VarDecl::StorageClass SC = StorageClassSpecToVarDeclStorageClass(SCSpec, 0);
+  VarDecl::StorageClass SC = StorageClassSpecToVarDeclStorageClass(SCSpec);
   if (SCSpec == DeclSpec::SCS_mutable) {
     // mutable can only appear on non-static class members, so it's always
     // an error here
     Diag(D.getIdentifierLoc(), diag::err_mutable_nonmember);
     D.setInvalidType();
-    SC = VarDecl::None;
+    SC = SC_None;
   }
   SCSpec = D.getDeclSpec().getStorageClassSpecAsWritten();
   VarDecl::StorageClass SCAsWritten
-    = StorageClassSpecToVarDeclStorageClass(SCSpec, DC);
+    = StorageClassSpecToVarDeclStorageClass(SCSpec);
 
   IdentifierInfo *II = Name.getAsIdentifierInfo();
   if (!II) {
@@ -2539,11 +2704,11 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   if (!DC->isRecord() && S->getFnParent() == 0) {
     // C99 6.9p2: The storage-class specifiers auto and register shall not
     // appear in the declaration specifiers in an external declaration.
-    if (SC == VarDecl::Auto || SC == VarDecl::Register) {
+    if (SC == SC_Auto || SC == SC_Register) {
 
       // If this is a register variable with an asm label specified, then this
       // is a GNU extension.
-      if (SC == VarDecl::Register && D.getAsmLabel())
+      if (SC == SC_Register && D.getAsmLabel())
         Diag(D.getIdentifierLoc(), diag::err_unsupported_global_register);
       else
         Diag(D.getIdentifierLoc(), diag::err_typecheck_sclass_fscope);
@@ -2552,14 +2717,14 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   }
   if (DC->isRecord() && !CurContext->isRecord()) {
     // This is an out-of-line definition of a static data member.
-    if (SC == VarDecl::Static) {
+    if (SC == SC_Static) {
       Diag(D.getDeclSpec().getStorageClassSpecLoc(),
            diag::err_static_out_of_line)
         << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
-    } else if (SC == VarDecl::None)
-      SC = VarDecl::Static;
+    } else if (SC == SC_None)
+      SC = SC_Static;
   }
-  if (SC == VarDecl::Static) {
+  if (SC == SC_Static) {
     if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(DC)) {
       if (RD->isLocalClass())
         Diag(D.getIdentifierLoc(),
@@ -2613,7 +2778,7 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
   SetNestedNameSpecifier(NewVD, D);
 
-  if (NumMatchedTemplateParamLists > 0) {
+  if (NumMatchedTemplateParamLists > 0 && D.getCXXScopeSpec().isSet()) {
     NewVD->setTemplateParameterListsInfo(Context,
                                          NumMatchedTemplateParamLists,
                         (TemplateParameterList**)TemplateParamLists.release());
@@ -2639,7 +2804,8 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   if (Expr *E = (Expr*) D.getAsmLabel()) {
     // The parser guarantees this is a string.
     StringLiteral *SE = cast<StringLiteral>(E);
-    NewVD->addAttr(::new (Context) AsmLabelAttr(Context, SE->getString()));
+    NewVD->addAttr(::new (Context) AsmLabelAttr(SE->getStrTokenLoc(0), 
+                                                Context, SE->getString()));
   }
 
   // Diagnose shadowed variables before filtering for scope.
@@ -2679,6 +2845,8 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     NewVD->setInvalidDecl();
 
   // attributes declared post-definition are currently ignored
+  // FIXME: This should be handled in attribute merging, not
+  // here.
   if (Previous.isSingleResult()) {
     VarDecl *Def = dyn_cast<VarDecl>(Previous.getFoundDecl());
     if (Def && (Def = Def->getDefinition()) &&
@@ -2694,6 +2862,13 @@ Sema::ActOnVariableDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       !NewVD->isInvalidDecl())
     RegisterLocallyScopedExternCDecl(NewVD, Previous, S);
 
+  // If there's a #pragma GCC visibility in scope, and this isn't a class
+  // member, set the visibility of this variable.
+  if (NewVD->getLinkage() == ExternalLinkage && !DC->isRecord())
+    AddPushedVisibilityAttribute(NewVD);
+
+  MarkUnusedFileScopedDecl(NewVD);
+
   return NewVD;
 }
 
@@ -2807,19 +2982,16 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
 
   bool isVM = T->isVariablyModifiedType();
   if (isVM || NewVD->hasAttr<CleanupAttr>() ||
-      NewVD->hasAttr<BlocksAttr>() ||
-      // FIXME: We need to diagnose jumps passed initialized variables in C++.
-      // However, this turns on the scope checker for everything with a variable
-      // which may impact compile time.  See if we can find a better solution
-      // to this, perhaps only checking functions that contain gotos in C++?
-      (LangOpts.CPlusPlus && NewVD->hasLocalStorage()))
-    FunctionNeedsScopeChecking() = true;
+      NewVD->hasAttr<BlocksAttr>())
+    getCurFunction()->setHasBranchProtectedScope();
 
   if ((isVM && NewVD->hasLinkage()) ||
       (T->isVariableArrayType() && NewVD->hasGlobalStorage())) {
     bool SizeIsNegative;
+    llvm::APSInt Oversized;
     QualType FixedTy =
-        TryToFixInvalidVariablyModifiedType(T, Context, SizeIsNegative);
+        TryToFixInvalidVariablyModifiedType(T, Context, SizeIsNegative,
+                                            Oversized);
 
     if (FixedTy.isNull() && T->isVariableArrayType()) {
       const VariableArrayType *VAT = Context.getAsVariableArrayType(T);
@@ -2830,7 +3002,7 @@ void Sema::CheckVariableDeclaration(VarDecl *NewVD,
       if (NewVD->isFileVarDecl())
         Diag(NewVD->getLocation(), diag::err_vla_decl_in_file_scope)
         << SizeRange;
-      else if (NewVD->getStorageClass() == VarDecl::Static)
+      else if (NewVD->getStorageClass() == SC_Static)
         Diag(NewVD->getLocation(), diag::err_vla_decl_has_static_storage)
         << SizeRange;
       else
@@ -2970,8 +3142,10 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
                               bool IsFunctionDefinition, bool &Redeclaration) {
   assert(R.getTypePtr()->isFunctionType());
 
-  DeclarationName Name = GetNameForDeclarator(D);
-  FunctionDecl::StorageClass SC = FunctionDecl::None;
+  // TODO: consider using NameInfo for diagnostic.
+  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
+  FunctionDecl::StorageClass SC = SC_None;
   switch (D.getDeclSpec().getStorageClassSpec()) {
   default: assert(0 && "Unknown storage class!");
   case DeclSpec::SCS_auto:
@@ -2981,10 +3155,10 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
          diag::err_typecheck_sclass_func);
     D.setInvalidType();
     break;
-  case DeclSpec::SCS_unspecified: SC = FunctionDecl::None; break;
-  case DeclSpec::SCS_extern:      SC = FunctionDecl::Extern; break;
+  case DeclSpec::SCS_unspecified: SC = SC_None; break;
+  case DeclSpec::SCS_extern:      SC = SC_Extern; break;
   case DeclSpec::SCS_static: {
-    if (CurContext->getLookupContext()->isFunctionOrMethod()) {
+    if (CurContext->getRedeclContext()->isFunctionOrMethod()) {
       // C99 6.7.1p5:
       //   The declaration of an identifier for a function that has
       //   block scope shall have no explicit storage-class specifier
@@ -2992,12 +3166,12 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       // See also (C++ [dcl.stc]p4).
       Diag(D.getDeclSpec().getStorageClassSpecLoc(),
            diag::err_static_block_func);
-      SC = FunctionDecl::None;
+      SC = SC_None;
     } else
-      SC = FunctionDecl::Static;
+      SC = SC_Static;
     break;
   }
-  case DeclSpec::SCS_private_extern: SC = FunctionDecl::PrivateExtern;break;
+  case DeclSpec::SCS_private_extern: SC = SC_PrivateExtern;break;
   }
 
   if (D.getDeclSpec().isThreadSpecified())
@@ -3010,7 +3184,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
   DeclSpec::SCS SCSpec = D.getDeclSpec().getStorageClassSpecAsWritten();
   FunctionDecl::StorageClass SCAsWritten
-    = StorageClassSpecToFunctionDeclStorageClass(SCSpec, DC);
+    = StorageClassSpecToFunctionDeclStorageClass(SCSpec);
 
   // Check that the return type is not an abstract class type.
   // For record types, this is done by the AbstractClassUsageDiagnoser once
@@ -3050,7 +3224,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     // Create the new declaration
     NewFD = CXXConstructorDecl::Create(Context,
                                        cast<CXXRecordDecl>(DC),
-                                       D.getIdentifierLoc(), Name, R, TInfo,
+                                       NameInfo, R, TInfo,
                                        isExplicit, isInline,
                                        /*isImplicitlyDeclared=*/false);
   } else if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
@@ -3060,7 +3234,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
       NewFD = CXXDestructorDecl::Create(Context,
                                         cast<CXXRecordDecl>(DC),
-                                        D.getIdentifierLoc(), Name, R,
+                                        NameInfo, R,
                                         isInline,
                                         /*isImplicitlyDeclared=*/false);
       NewFD->setTypeSourceInfo(TInfo);
@@ -3085,7 +3259,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
     CheckConversionDeclarator(D, R, SC);
     NewFD = CXXConversionDecl::Create(Context, cast<CXXRecordDecl>(DC),
-                                      D.getIdentifierLoc(), Name, R, TInfo,
+                                      NameInfo, R, TInfo,
                                       isInline, isExplicit);
 
     isVirtualOkay = true;
@@ -3103,7 +3277,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       return 0;
     }
 
-    bool isStatic = SC == FunctionDecl::Static;
+    bool isStatic = SC == SC_Static;
     
     // [class.free]p1:
     // Any allocation function for a class T is a static member
@@ -3120,7 +3294,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     
     // This is a C++ method declaration.
     NewFD = CXXMethodDecl::Create(Context, cast<CXXRecordDecl>(DC),
-                                  D.getIdentifierLoc(), Name, R, TInfo,
+                                  NameInfo, R, TInfo,
                                   isStatic, SCAsWritten, isInline);
 
     isVirtualOkay = !isStatic;
@@ -3137,8 +3311,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
        (!isa<FunctionType>(R.getTypePtr()) && R->isFunctionProtoType());
 
     NewFD = FunctionDecl::Create(Context, DC,
-                                 D.getIdentifierLoc(),
-                                 Name, R, TInfo, SC, SCAsWritten, isInline,
+                                 NameInfo, R, TInfo, SC, SCAsWritten, isInline,
                                  HasPrototype);
   }
 
@@ -3212,7 +3385,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     }
   }
 
-  if (NumMatchedTemplateParamLists > 0) {
+  if (NumMatchedTemplateParamLists > 0 && D.getCXXScopeSpec().isSet()) {
     NewFD->setTemplateParameterListsInfo(Context,
                                          NumMatchedTemplateParamLists,
                         (TemplateParameterList**)TemplateParamLists.release());
@@ -3244,6 +3417,17 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     }
   }
 
+  // C++ [dcl.fct.spec]p3:
+  //  The inline specifier shall not appear on a block scope function declaration.
+  if (isInline && !NewFD->isInvalidDecl() && getLangOptions().CPlusPlus) {
+    if (CurContext->isFunctionOrMethod()) {
+      // 'inline' is not allowed on block scope function declaration.
+      Diag(D.getDeclSpec().getInlineSpecLoc(), 
+           diag::err_inline_declaration_block_scope) << Name
+        << FixItHint::CreateRemoval(D.getDeclSpec().getInlineSpecLoc());
+    }
+  }
+ 
   // C++ [dcl.fct.spec]p6:
   //  The explicit specifier shall be used only in the declaration of a
   //  constructor or conversion function within its class definition; see 12.3.1
@@ -3282,7 +3466,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     NewFD->setAccess(AS_public);
   }
 
-  if (SC == FunctionDecl::Static && isa<CXXMethodDecl>(NewFD) &&
+  if (SC == SC_Static && isa<CXXMethodDecl>(NewFD) &&
       !CurContext->isRecord()) {
     // C++ [class.static]p1:
     //   A data or function member of a class may be declared static
@@ -3300,7 +3484,8 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   if (Expr *E = (Expr*) D.getAsmLabel()) {
     // The parser guarantees this is a string.
     StringLiteral *SE = cast<StringLiteral>(E);
-    NewFD->addAttr(::new (Context) AsmLabelAttr(Context, SE->getString()));
+    NewFD->addAttr(::new (Context) AsmLabelAttr(SE->getStrTokenLoc(0), Context,
+                                                SE->getString()));
   }
 
   // Copy the parameter declarations from the declarator D to the function
@@ -3316,9 +3501,9 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     // already checks for that case.
     if (FTI.NumArgs == 1 && !FTI.isVariadic && FTI.ArgInfo[0].Ident == 0 &&
         FTI.ArgInfo[0].Param &&
-        FTI.ArgInfo[0].Param.getAs<ParmVarDecl>()->getType()->isVoidType()) {
+        cast<ParmVarDecl>(FTI.ArgInfo[0].Param)->getType()->isVoidType()) {
       // Empty arg list, don't push any params.
-      ParmVarDecl *Param = FTI.ArgInfo[0].Param.getAs<ParmVarDecl>();
+      ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[0].Param);
 
       // In C++, the empty parameter-type-list must be spelled "void"; a
       // typedef of void is not permitted.
@@ -3328,7 +3513,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
       // FIXME: Leaks decl?
     } else if (FTI.NumArgs > 0 && FTI.ArgInfo[0].Param != 0) {
       for (unsigned i = 0, e = FTI.NumArgs; i != e; ++i) {
-        ParmVarDecl *Param = FTI.ArgInfo[i].Param.getAs<ParmVarDecl>();
+        ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[i].Param);
         assert(Param->getDeclContext() != NewFD && "Was set before ?");
         Param->setDeclContext(NewFD);
         Params.push_back(Param);
@@ -3479,14 +3664,14 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     // definition (C++ [dcl.meaning]p1).
     // Note that this is not the case for explicit specializations of
     // function templates or member functions of class templates, per
-    // C++ [temp.expl.spec]p2.
+    // C++ [temp.expl.spec]p2. We also allow these declarations as an extension
+    // for compatibility with old SWIG code which likes to generate them.
     if (!IsFunctionDefinition && !isFriend &&
         !isFunctionTemplateSpecialization && !isExplicitSpecialization) {
-      Diag(NewFD->getLocation(), diag::err_out_of_line_declaration)
+      Diag(NewFD->getLocation(), diag::ext_out_of_line_declaration)
         << D.getCXXScopeSpec().getRange();
-      NewFD->setInvalidDecl();
-    } else if (!Redeclaration && 
-               !(isFriend && CurContext->isDependentContext())) {
+    }
+    if (!Redeclaration && !(isFriend && CurContext->isDependentContext())) {
       // The user tried to provide an out-of-line definition for a
       // function that is a member of a class or namespace, but there
       // was no such member function declared (C++ [class.mfct]p2,
@@ -3526,6 +3711,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   ProcessDeclAttributes(S, NewFD, D);
 
   // attributes declared post-definition are currently ignored
+  // FIXME: This should happen during attribute merging
   if (Redeclaration && Previous.isSingleResult()) {
     const FunctionDecl *Def;
     FunctionDecl *PrevFD = dyn_cast<FunctionDecl>(Previous.getFoundDecl());
@@ -3537,7 +3723,7 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
 
   AddKnownFunctionAttributes(NewFD);
 
-  if (OverloadableAttrRequired && !NewFD->getAttr<OverloadableAttr>()) {
+  if (OverloadableAttrRequired && !NewFD->hasAttr<OverloadableAttr>()) {
     // If a function name is overloadable in C, then every function
     // with that name must be marked "overloadable".
     Diag(NewFD->getLocation(), diag::err_attribute_overloadable_missing)
@@ -3545,9 +3731,28 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
     if (!Previous.empty())
       Diag(Previous.getRepresentativeDecl()->getLocation(),
            diag::note_attribute_overloadable_prev_overload);
-    NewFD->addAttr(::new (Context) OverloadableAttr());
+    NewFD->addAttr(::new (Context) OverloadableAttr(SourceLocation(), Context));
+  }
+
+  if (NewFD->hasAttr<OverloadableAttr>() && 
+      !NewFD->getType()->getAs<FunctionProtoType>()) {
+    Diag(NewFD->getLocation(),
+         diag::err_attribute_overloadable_no_prototype)
+      << NewFD;
+
+    // Turn this into a variadic function with no parameters.
+    const FunctionType *FT = NewFD->getType()->getAs<FunctionType>();
+    QualType R = Context.getFunctionType(FT->getResultType(),
+                                         0, 0, true, 0, false, false, 0, 0,
+                                         FT->getExtInfo());
+    NewFD->setType(R);
   }
 
+  // If there's a #pragma GCC visibility in scope, and this isn't a class
+  // member, set the visibility of this function.
+  if (NewFD->getLinkage() == ExternalLinkage && !DC->isRecord())
+    AddPushedVisibilityAttribute(NewFD);
+
   // If this is a locally-scoped extern C function, update the
   // map of such names.
   if (CurContext->isFunctionOrMethod() && NewFD->isExternC()
@@ -3563,17 +3768,8 @@ Sema::ActOnFunctionDeclarator(Scope* S, Declarator& D, DeclContext* DC,
   if (FunctionTemplate)
     return FunctionTemplate;
 
-  
-  // Keep track of static, non-inlined function definitions that
-  // have not been used. We will warn later.
-  // FIXME: Also include static functions declared but not defined.
-  if (!NewFD->isInvalidDecl() && IsFunctionDefinition 
-      && !NewFD->isInlined() && NewFD->getLinkage() == InternalLinkage
-      && !NewFD->isUsed() && !NewFD->hasAttr<UnusedAttr>()
-      && !NewFD->hasAttr<ConstructorAttr>()
-      && !NewFD->hasAttr<DestructorAttr>())
-    UnusedStaticFuncs.push_back(NewFD);
-  
+  MarkUnusedFileScopedDecl(NewFD);
+
   return NewFD;
 }
 
@@ -3597,8 +3793,14 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
                                     bool &Redeclaration,
                                     bool &OverloadableAttrRequired) {
   // If NewFD is already known erroneous, don't do any of this checking.
-  if (NewFD->isInvalidDecl())
+  if (NewFD->isInvalidDecl()) {
+    // If this is a class member, mark the class invalid immediately.
+    // This avoids some consistency errors later.
+    if (isa<CXXMethodDecl>(NewFD))
+      cast<CXXMethodDecl>(NewFD)->getParent()->setInvalidDecl();
+
     return;
+  }
 
   if (NewFD->getResultType()->isVariablyModifiedType()) {
     // Functions returning a variably modified type violate C99 6.7.5.2p2
@@ -3634,27 +3836,9 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
       Redeclaration = true;
       OldDecl = Previous.getFoundDecl();
     } else {
-      if (!getLangOptions().CPlusPlus) {
+      if (!getLangOptions().CPlusPlus)
         OverloadableAttrRequired = true;
 
-        // Functions marked "overloadable" must have a prototype (that
-        // we can't get through declaration merging).
-        if (!NewFD->getType()->getAs<FunctionProtoType>()) {
-          Diag(NewFD->getLocation(),
-               diag::err_attribute_overloadable_no_prototype)
-            << NewFD;
-          Redeclaration = true;
-
-          // Turn this into a variadic function with no parameters.
-          QualType R = Context.getFunctionType(
-                     NewFD->getType()->getAs<FunctionType>()->getResultType(),
-                     0, 0, true, 0, false, false, 0, 0,
-                     FunctionType::ExtInfo());
-          NewFD->setType(R);
-          return NewFD->setInvalidDecl();
-        }
-      }
-
       switch (CheckOverload(S, NewFD, Previous, OldDecl,
                             /*NewIsUsingDecl*/ false)) {
       case Ovl_Match:
@@ -3723,6 +3907,8 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
         DeclarationName Name
           = Context.DeclarationNames.getCXXDestructorName(
                                         Context.getCanonicalType(ClassType));
+//         NewFD->getDeclName().dump();
+//         Name.dump();
         if (NewFD->getDeclName() != Name) {
           Diag(NewFD->getLocation(), diag::err_destructor_name);
           return NewFD->setInvalidDecl();
@@ -3750,12 +3936,6 @@ void Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
         AddOverriddenMethods(Method->getParent(), Method);
     }
 
-    // Additional checks for the destructor; make sure we do this after we
-    // figure out whether the destructor is virtual.
-    if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(NewFD))
-      if (!Destructor->getParent()->isDependentType())
-        CheckDestructor(Destructor);
-
     // Extra checking for C++ overloaded operators (C++ [over.oper]).
     if (NewFD->isOverloadedOperator() &&
         CheckOverloadedOperatorDeclaration(NewFD))
@@ -3781,7 +3961,7 @@ void Sema::CheckMain(FunctionDecl* FD) {
   //   shall not appear in a declaration of main.
   // static main is not an error under C99, but we should warn about it.
   bool isInline = FD->isInlineSpecified();
-  bool isStatic = FD->getStorageClass() == FunctionDecl::Static;
+  bool isStatic = FD->getStorageClass() == SC_Static;
   if (isInline || isStatic) {
     unsigned diagID = diag::warn_unusual_main_decl;
     if (isInline || getLangOptions().CPlusPlus)
@@ -3874,22 +4054,21 @@ bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
   // "may accept other forms of constant expressions" exception.
   // (We never end up here for C++, so the constant expression
   // rules there don't matter.)
-  if (Init->isConstantInitializer(Context))
+  if (Init->isConstantInitializer(Context, false))
     return false;
   Diag(Init->getExprLoc(), diag::err_init_element_not_constant)
     << Init->getSourceRange();
   return true;
 }
 
-void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init) {
-  AddInitializerToDecl(dcl, move(init), /*DirectInit=*/false);
+void Sema::AddInitializerToDecl(Decl *dcl, Expr *init) {
+  AddInitializerToDecl(dcl, init, /*DirectInit=*/false);
 }
 
 /// AddInitializerToDecl - Adds the initializer Init to the
 /// declaration dcl. If DirectInit is true, this is C++ direct
 /// initialization rather than copy initialization.
-void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
-  Decl *RealDecl = dcl.getAs<Decl>();
+void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
   // If there is no declaration, there was an error parsing it.  Just ignore
   // the initializer.
   if (RealDecl == 0)
@@ -3900,7 +4079,6 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
     // distinguish between a normal initializer and a pure-specifier.
     // Thus this grotesque test.
     IntegerLiteral *IL;
-    Expr *Init = static_cast<Expr *>(init.get());
     if ((IL = dyn_cast<IntegerLiteral>(Init)) && IL->getValue() == 0 &&
         Context.getCanonicalType(IL->getType()) == Context.IntTy)
       CheckPureMethod(Method, Init->getSourceRange());
@@ -3925,6 +4103,8 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
     return;
   }
 
+  
+
   // A definition must end up with a complete type, which means it must be
   // complete with the restriction that an array type might be completed by the
   // initializer; note that later code assumes this restriction.
@@ -3951,11 +4131,28 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
     VDecl->setInvalidDecl();
     return;
   }
+  
+  // C++ [class.static.data]p4
+  //   If a static data member is of const integral or const
+  //   enumeration type, its declaration in the class definition can
+  //   specify a constant-initializer which shall be an integral
+  //   constant expression (5.19). In that case, the member can appear
+  //   in integral constant expressions. The member shall still be
+  //   defined in a namespace scope if it is used in the program and the
+  //   namespace scope definition shall not contain an initializer.
+  //
+  // We already performed a redefinition check above, but for static
+  // data members we also need to check whether there was an in-class
+  // declaration with an initializer.
+  const VarDecl* PrevInit = 0;
+  if (VDecl->isStaticDataMember() && VDecl->getAnyInitializer(PrevInit)) {
+    Diag(VDecl->getLocation(), diag::err_redefinition) << VDecl->getDeclName();
+    Diag(PrevInit->getLocation(), diag::note_previous_definition);
+    return;
+  }  
 
-  // Take ownership of the expression, now that we're sure we have somewhere
-  // to put it.
-  Expr *Init = init.takeAs<Expr>();
-  assert(Init && "missing initializer");
+  if (getLangOptions().CPlusPlus && VDecl->hasLocalStorage())
+    getCurFunction()->setHasBranchProtectedScope();
 
   // Capture the variable that is being initialized and the style of
   // initialization.
@@ -3978,8 +4175,8 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
       VDecl->setInvalidDecl();
     } else if (!VDecl->isInvalidDecl()) {
       InitializationSequence InitSeq(*this, Entity, Kind, &Init, 1);
-      OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                          MultiExprArg(*this, (void**)&Init, 1),
+      ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
+                                                MultiExprArg(*this, &Init, 1),
                                                 &DclT);
       if (Result.isInvalid()) {
         VDecl->setInvalidDecl();
@@ -3991,7 +4188,7 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
       // C++ 3.6.2p2, allow dynamic initialization of static initializers.
       // Don't check invalid declarations to avoid emitting useless diagnostics.
       if (!getLangOptions().CPlusPlus && !VDecl->isInvalidDecl()) {
-        if (VDecl->getStorageClass() == VarDecl::Static) // C99 6.7.8p4.
+        if (VDecl->getStorageClass() == SC_Static) // C99 6.7.8p4.
           CheckForConstantInitializer(Init, DclT);
       }
     }
@@ -4039,18 +4236,18 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
             << Init->getSourceRange();
           VDecl->setInvalidDecl();
         } else if (!VDecl->getType()->isDependentType())
-          ImpCastExprToType(Init, VDecl->getType(), CastExpr::CK_IntegralCast);
+          ImpCastExprToType(Init, VDecl->getType(), CK_IntegralCast);
       }
     }
   } else if (VDecl->isFileVarDecl()) {
-    if (VDecl->getStorageClass() == VarDecl::Extern && 
+    if (VDecl->getStorageClass() == SC_Extern && 
         (!getLangOptions().CPlusPlus || 
          !Context.getBaseElementType(VDecl->getType()).isConstQualified()))
       Diag(VDecl->getLocation(), diag::warn_extern_init);
     if (!VDecl->isInvalidDecl()) {
       InitializationSequence InitSeq(*this, Entity, Kind, &Init, 1);
-      OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                          MultiExprArg(*this, (void**)&Init, 1),
+      ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
+                                                MultiExprArg(*this, &Init, 1),
                                                 &DclT);
       if (Result.isInvalid()) {
         VDecl->setInvalidDecl();
@@ -4081,6 +4278,14 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
   VDecl->setInit(Init);
 
   if (getLangOptions().CPlusPlus) {
+    if (!VDecl->isInvalidDecl() &&
+        !VDecl->getDeclContext()->isDependentContext() &&
+        VDecl->hasGlobalStorage() && !VDecl->isStaticLocal() &&
+        !Init->isConstantInitializer(Context,
+                                     VDecl->getType()->isReferenceType()))
+      Diag(VDecl->getLocation(), diag::warn_global_constructor)
+        << Init->getSourceRange();
+
     // Make sure we mark the destructor as used if necessary.
     QualType InitType = VDecl->getType();
     while (const ArrayType *Array = Context.getAsArrayType(InitType))
@@ -4095,10 +4300,9 @@ void Sema::AddInitializerToDecl(DeclPtrTy dcl, ExprArg init, bool DirectInit) {
 /// ActOnInitializerError - Given that there was an error parsing an
 /// initializer for the given declaration, try to return to some form
 /// of sanity.
-void Sema::ActOnInitializerError(DeclPtrTy dcl) {
+void Sema::ActOnInitializerError(Decl *D) {
   // Our main concern here is re-establishing invariants like "a
   // variable's type is either dependent or complete".
-  Decl *D = dcl.getAs<Decl>();
   if (!D || D->isInvalidDecl()) return;
 
   VarDecl *VD = dyn_cast<VarDecl>(D);
@@ -4127,10 +4331,8 @@ void Sema::ActOnInitializerError(DeclPtrTy dcl) {
   // though.
 }
 
-void Sema::ActOnUninitializedDecl(DeclPtrTy dcl,
+void Sema::ActOnUninitializedDecl(Decl *RealDecl,
                                   bool TypeContainsUndeducedAuto) {
-  Decl *RealDecl = dcl.getAs<Decl>();
-
   // If there is no declaration, there was an error parsing it. Just ignore it.
   if (RealDecl == 0)
     return;
@@ -4190,7 +4392,7 @@ void Sema::ActOnUninitializedDecl(DeclPtrTy dcl,
                                   ArrayT->getElementType(),
                                   diag::err_illegal_decl_array_incomplete_type))
             Var->setInvalidDecl();
-        } else if (Var->getStorageClass() == VarDecl::Static) {
+        } else if (Var->getStorageClass() == SC_Static) {
           // C99 6.9.2p3: If the declaration of an identifier for an object is
           // a tentative definition and has internal linkage (C99 6.2.2p3), the
           // declared type shall not be an incomplete type.
@@ -4221,15 +4423,15 @@ void Sema::ActOnUninitializedDecl(DeclPtrTy dcl,
       return;
     }
 
-   // Provide a specific diagnostic for uninitialized variable
-   // definitions with reference type.
-   if (Type->isReferenceType()) {
-     Diag(Var->getLocation(), diag::err_reference_var_requires_init)
-       << Var->getDeclName()
-       << SourceRange(Var->getLocation(), Var->getLocation());
-     Var->setInvalidDecl();
-     return;
-   }
+    // Provide a specific diagnostic for uninitialized variable
+    // definitions with reference type.
+    if (Type->isReferenceType()) {
+      Diag(Var->getLocation(), diag::err_reference_var_requires_init)
+        << Var->getDeclName()
+        << SourceRange(Var->getLocation(), Var->getLocation());
+      Var->setInvalidDecl();
+      return;
+    }
 
     // Do not attempt to type-check the default initializer for a
     // variable with dependent type.
@@ -4271,17 +4473,30 @@ void Sema::ActOnUninitializedDecl(DeclPtrTy dcl,
       //   program is ill-formed.
       // FIXME: DPG thinks it is very fishy that C++0x disables this.
     } else {
+      // Check for jumps past the implicit initializer.  C++0x
+      // clarifies that this applies to a "variable with automatic
+      // storage duration", not a "local variable".
+      if (getLangOptions().CPlusPlus && Var->hasLocalStorage())
+        getCurFunction()->setHasBranchProtectedScope();
+
       InitializedEntity Entity = InitializedEntity::InitializeVariable(Var);
       InitializationKind Kind
         = InitializationKind::CreateDefault(Var->getLocation());
     
       InitializationSequence InitSeq(*this, Entity, Kind, 0, 0);
-      OwningExprResult Init = InitSeq.Perform(*this, Entity, Kind,
-                                              MultiExprArg(*this, 0, 0));
+      ExprResult Init = InitSeq.Perform(*this, Entity, Kind,
+                                        MultiExprArg(*this, 0, 0));
       if (Init.isInvalid())
         Var->setInvalidDecl();
-      else if (Init.get())
+      else if (Init.get()) {
         Var->setInit(MaybeCreateCXXExprWithTemporaries(Init.takeAs<Expr>()));
+
+        if (getLangOptions().CPlusPlus && !Var->isInvalidDecl() && 
+            Var->hasGlobalStorage() && !Var->isStaticLocal() &&
+            !Var->getDeclContext()->isDependentContext() &&
+            !Var->getInit()->isConstantInitializer(Context, false))
+          Diag(Var->getLocation(), diag::warn_global_constructor);
+      }
     }
 
     if (!Var->isInvalidDecl() && getLangOptions().CPlusPlus && Record)
@@ -4289,16 +4504,16 @@ void Sema::ActOnUninitializedDecl(DeclPtrTy dcl,
   }
 }
 
-Sema::DeclGroupPtrTy Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
-                                                   DeclPtrTy *Group,
-                                                   unsigned NumDecls) {
+Sema::DeclGroupPtrTy
+Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
+                              Decl **Group, unsigned NumDecls) {
   llvm::SmallVector<Decl*, 8> Decls;
 
   if (DS.isTypeSpecOwned())
-    Decls.push_back((Decl*)DS.getTypeRep());
+    Decls.push_back(DS.getRepAsDecl());
 
   for (unsigned i = 0; i != NumDecls; ++i)
-    if (Decl *D = Group[i].getAs<Decl>())
+    if (Decl *D = Group[i])
       Decls.push_back(D);
 
   return DeclGroupPtrTy::make(DeclGroupRef::Create(Context,
@@ -4308,16 +4523,15 @@ Sema::DeclGroupPtrTy Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
 
 /// ActOnParamDeclarator - Called from Parser::ParseFunctionDeclarator()
 /// to introduce parameters into function prototype scope.
-Sema::DeclPtrTy
-Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
+Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
   const DeclSpec &DS = D.getDeclSpec();
 
   // Verify C99 6.7.5.3p2: The only SCS allowed is 'register'.
-  VarDecl::StorageClass StorageClass = VarDecl::None;
-  VarDecl::StorageClass StorageClassAsWritten = VarDecl::None;
+  VarDecl::StorageClass StorageClass = SC_None;
+  VarDecl::StorageClass StorageClassAsWritten = SC_None;
   if (DS.getStorageClassSpec() == DeclSpec::SCS_register) {
-    StorageClass = VarDecl::Register;
-    StorageClassAsWritten = VarDecl::Register;
+    StorageClass = SC_Register;
+    StorageClassAsWritten = SC_Register;
   } else if (DS.getStorageClassSpec() != DeclSpec::SCS_unspecified) {
     Diag(DS.getStorageClassSpecLoc(),
          diag::err_invalid_storage_class_in_func_decl);
@@ -4358,7 +4572,7 @@ Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
         DiagnoseTemplateParameterShadow(D.getIdentifierLoc(), PrevDecl);
         // Just pretend that we didn't see the previous declaration.
         PrevDecl = 0;
-      } else if (S->isDeclScope(DeclPtrTy::make(PrevDecl))) {
+      } else if (S->isDeclScope(PrevDecl)) {
         Diag(D.getIdentifierLoc(), diag::err_param_redefinition) << II;
         Diag(PrevDecl->getLocation(), diag::note_previous_declaration);
 
@@ -4389,7 +4603,7 @@ Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
   }
 
   // Add the parameter declaration into this scope.
-  S->AddDecl(DeclPtrTy::make(New));
+  S->AddDecl(New);
   if (II)
     IdResolver.AddDecl(New);
 
@@ -4398,7 +4612,7 @@ Sema::ActOnParamDeclarator(Scope *S, Declarator &D) {
   if (New->hasAttr<BlocksAttr>()) {
     Diag(New->getLocation(), diag::err_block_on_nonlocal);
   }
-  return DeclPtrTy::make(New);
+  return New;
 }
 
 /// \brief Synthesizes a variable for a parameter arising from a
@@ -4408,11 +4622,31 @@ ParmVarDecl *Sema::BuildParmVarDeclForTypedef(DeclContext *DC,
                                               QualType T) {
   ParmVarDecl *Param = ParmVarDecl::Create(Context, DC, Loc, 0,
                                 T, Context.getTrivialTypeSourceInfo(T, Loc),
-                                           VarDecl::None, VarDecl::None, 0);
+                                           SC_None, SC_None, 0);
   Param->setImplicit();
   return Param;
 }
 
+void Sema::DiagnoseUnusedParameters(ParmVarDecl * const *Param,
+                                    ParmVarDecl * const *ParamEnd) {
+  if (Diags.getDiagnosticLevel(diag::warn_unused_parameter) ==
+        Diagnostic::Ignored)
+    return;
+
+  // Don't diagnose unused-parameter errors in template instantiations; we
+  // will already have done so in the template itself.
+  if (!ActiveTemplateInstantiations.empty())
+    return;
+
+  for (; Param != ParamEnd; ++Param) {
+    if (!(*Param)->isUsed() && (*Param)->getDeclName() &&
+        !(*Param)->hasAttr<UnusedAttr>()) {
+      Diag((*Param)->getLocation(), diag::warn_unused_parameter)
+        << (*Param)->getDeclName();
+    }
+  }
+}
+
 ParmVarDecl *Sema::CheckParameter(DeclContext *DC, 
                                   TypeSourceInfo *TSInfo, QualType T,
                                   IdentifierInfo *Name,
@@ -4487,8 +4721,8 @@ void Sema::ActOnFinishKNRParamDeclarations(Scope *S, Declarator &D,
   }
 }
 
-Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope,
-                                              Declarator &D) {
+Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope,
+                                         Declarator &D) {
   assert(getCurFunctionDecl() == 0 && "Function parsing confused");
   assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
          "Not a function declarator!");
@@ -4500,9 +4734,9 @@ Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope,
 
   Scope *ParentScope = FnBodyScope->getParent();
 
-  DeclPtrTy DP = HandleDeclarator(ParentScope, D,
-                                  MultiTemplateParamsArg(*this),
-                                  /*IsFunctionDefinition=*/true);
+  Decl *DP = HandleDeclarator(ParentScope, D,
+                              MultiTemplateParamsArg(*this),
+                              /*IsFunctionDefinition=*/true);
   return ActOnStartOfFunctionDef(FnBodyScope, DP);
 }
 
@@ -4550,7 +4784,7 @@ static bool ShouldWarnAboutMissingPrototype(const FunctionDecl *FD) {
   return MissingPrototype;
 }
 
-Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, DeclPtrTy D) {
+Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
   // Clear the last template instantiation error context.
   LastTemplateInstantiationErrorContext = ActiveTemplateInstantiation();
   
@@ -4558,11 +4792,10 @@ Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, DeclPtrTy D) {
     return D;
   FunctionDecl *FD = 0;
 
-  if (FunctionTemplateDecl *FunTmpl
-        = dyn_cast<FunctionTemplateDecl>(D.getAs<Decl>()))
+  if (FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(D))
     FD = FunTmpl->getTemplatedDecl();
   else
-    FD = cast<FunctionDecl>(D.getAs<Decl>());
+    FD = cast<FunctionDecl>(D);
 
   // Enter a new function scope
   PushFunctionScope();
@@ -4627,15 +4860,15 @@ Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, DeclPtrTy D) {
 
   // Checking attributes of current function definition
   // dllimport attribute.
-  if (FD->getAttr<DLLImportAttr>() &&
-      (!FD->getAttr<DLLExportAttr>())) {
-    // dllimport attribute cannot be applied to definition.
-    if (!(FD->getAttr<DLLImportAttr>())->isInherited()) {
+  DLLImportAttr *DA = FD->getAttr<DLLImportAttr>();
+  if (DA && (!FD->getAttr<DLLExportAttr>())) {
+    // dllimport attribute cannot be directly applied to definition.
+    if (!DA->isInherited()) {
       Diag(FD->getLocation(),
            diag::err_attribute_can_be_applied_only_to_symbol_declaration)
         << "dllimport";
       FD->setInvalidDecl();
-      return DeclPtrTy::make(FD);
+      return FD;
     }
 
     // Visual C++ appears to not think this is an issue, so only issue
@@ -4646,10 +4879,10 @@ Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, DeclPtrTy D) {
       // emitted.
       Diag(FD->getLocation(),
            diag::warn_redeclaration_without_attribute_prev_attribute_ignored)
-        << FD->getNameAsCString() << "dllimport";
+        << FD->getName() << "dllimport";
     }
   }
-  return DeclPtrTy::make(FD);
+  return FD;
 }
 
 /// \brief Given the set of return statements within a function body,
@@ -4668,9 +4901,11 @@ Sema::DeclPtrTy Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, DeclPtrTy D) {
 /// FIXME: Employ a smarter algorithm that accounts for multiple return 
 /// statements and the lifetimes of the NRVO candidates. We should be able to
 /// find a maximal set of NRVO variables.
-static void ComputeNRVO(Stmt *Body, ReturnStmt **Returns, unsigned NumReturns) {
+static void ComputeNRVO(Stmt *Body, FunctionScopeInfo *Scope) {
+  ReturnStmt **Returns = Scope->Returns.data();
+
   const VarDecl *NRVOCandidate = 0;
-  for (unsigned I = 0; I != NumReturns; ++I) {
+  for (unsigned I = 0, E = Scope->Returns.size(); I != E; ++I) {
     if (!Returns[I]->getNRVOCandidate())
       return;
     
@@ -4684,15 +4919,12 @@ static void ComputeNRVO(Stmt *Body, ReturnStmt **Returns, unsigned NumReturns) {
     const_cast<VarDecl*>(NRVOCandidate)->setNRVOVariable(true);
 }
 
-Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg) {
+Decl *Sema::ActOnFinishFunctionBody(Decl *D, Stmt *BodyArg) {
   return ActOnFinishFunctionBody(D, move(BodyArg), false);
 }
 
-Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
-                                              bool IsInstantiation) {
-  Decl *dcl = D.getAs<Decl>();
-  Stmt *Body = BodyArg.takeAs<Stmt>();
-
+Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
+                                    bool IsInstantiation) {
   FunctionDecl *FD = 0;
   FunctionTemplateDecl *FunTmpl = dyn_cast_or_null<FunctionTemplateDecl>(dcl);
   if (FunTmpl)
@@ -4718,8 +4950,7 @@ Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
       if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(FD))
         MarkVTableUsed(FD->getLocation(), Constructor->getParent());
       
-      ComputeNRVO(Body, FunctionScopes.back()->Returns.data(),
-                  FunctionScopes.back()->Returns.size());
+      ComputeNRVO(Body, getCurFunction());
     }
     
     assert(FD == getCurFunctionDecl() && "Function parsing confused");
@@ -4730,15 +4961,15 @@ Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
     if (!MD->isInvalidDecl())
       DiagnoseUnusedParameters(MD->param_begin(), MD->param_end());
   } else {
-    Body->Destroy(Context);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Verify and clean out per-function state.
 
   // Check goto/label use.
+  FunctionScopeInfo *CurFn = getCurFunction();
   for (llvm::DenseMap<IdentifierInfo*, LabelStmt*>::iterator
-       I = getLabelMap().begin(), E = getLabelMap().end(); I != E; ++I) {
+         I = CurFn->LabelMap.begin(), E = CurFn->LabelMap.end(); I != E; ++I) {
     LabelStmt *L = I->second;
 
     // Verify that we have no forward references left.  If so, there was a goto
@@ -4754,8 +4985,7 @@ Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
     // the function body so that they aren't leaked and that the AST is well
     // formed.
     if (Body == 0) {
-      // The whole function wasn't parsed correctly, just delete this.
-      L->Destroy(Context);
+      // The whole function wasn't parsed correctly.
       continue;
     }
 
@@ -4785,14 +5015,18 @@ Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
     
     // Verify that that gotos and switch cases don't jump into scopes illegally.
     // Verify that that gotos and switch cases don't jump into scopes illegally.
-    if (FunctionNeedsScopeChecking() &&
+    if (getCurFunction()->NeedsScopeChecking() &&
         !dcl->isInvalidDecl() &&
         !hasAnyErrorsInThisFunction())
       DiagnoseInvalidJumps(Body);
 
-    if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(dcl))
+    if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(dcl)) {
+      if (!Destructor->getParent()->isDependentType())
+        CheckDestructor(Destructor);
+
       MarkBaseAndMemberDestructorsReferenced(Destructor->getLocation(),
                                              Destructor->getParent());
+    }
     
     // If any errors have occurred, clear out any temporaries that may have
     // been leftover. This ensures that these temporaries won't be picked up for
@@ -4804,13 +5038,11 @@ Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
       // enabled.
       QualType ResultType;
       if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(dcl)) {
-        ResultType = FD->getResultType();
-      }
-      else {
+        AnalysisWarnings.IssueWarnings(WP, FD);
+      } else {
         ObjCMethodDecl *MD = cast<ObjCMethodDecl>(dcl);
-        ResultType = MD->getResultType();
+        AnalysisWarnings.IssueWarnings(WP, MD);
       }
-      AnalysisWarnings.IssueWarnings(WP, dcl);
     }
 
     assert(ExprTemporaries.empty() && "Leftover temporaries in function");
@@ -4826,8 +5058,8 @@ Sema::DeclPtrTy Sema::ActOnFinishFunctionBody(DeclPtrTy D, StmtArg BodyArg,
   // deletion in some later function.
   if (getDiagnostics().hasErrorOccurred())
     ExprTemporaries.clear();
-  
-  return D;
+
+  return dcl;
 }
 
 /// ImplicitlyDefineFunction - An undeclared identifier was used in a function
@@ -4873,8 +5105,7 @@ NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
   DeclContext *PrevDC = CurContext;
   CurContext = Context.getTranslationUnitDecl();
 
-  FunctionDecl *FD =
- dyn_cast<FunctionDecl>(ActOnDeclarator(TUScope, D).getAs<Decl>());
+  FunctionDecl *FD = dyn_cast<FunctionDecl>(ActOnDeclarator(TUScope, D));
   FD->setImplicit();
 
   CurContext = PrevDC;
@@ -4902,9 +5133,17 @@ void Sema::AddKnownFunctionAttributes(FunctionDecl *FD) {
     bool HasVAListArg;
     if (Context.BuiltinInfo.isPrintfLike(BuiltinID, FormatIdx, HasVAListArg)) {
       if (!FD->getAttr<FormatAttr>())
-        FD->addAttr(::new (Context) FormatAttr(Context, "printf", FormatIdx+1,
+        FD->addAttr(::new (Context) FormatAttr(FD->getLocation(), Context,
+                                                "printf", FormatIdx+1,
                                                HasVAListArg ? 0 : FormatIdx+2));
     }
+    if (Context.BuiltinInfo.isScanfLike(BuiltinID, FormatIdx,
+                                             HasVAListArg)) {
+     if (!FD->getAttr<FormatAttr>())
+       FD->addAttr(::new (Context) FormatAttr(FD->getLocation(), Context,
+                                              "scanf", FormatIdx+1,
+                                              HasVAListArg ? 0 : FormatIdx+2));
+    }
 
     // Mark const if we don't care about errno and that is the only
     // thing preventing the function from being const. This allows
@@ -4912,15 +5151,15 @@ void Sema::AddKnownFunctionAttributes(FunctionDecl *FD) {
     if (!getLangOptions().MathErrno &&
         Context.BuiltinInfo.isConstWithoutErrno(BuiltinID)) {
       if (!FD->getAttr<ConstAttr>())
-        FD->addAttr(::new (Context) ConstAttr());
+        FD->addAttr(::new (Context) ConstAttr(FD->getLocation(), Context));
     }
 
     if (Context.BuiltinInfo.isNoReturn(BuiltinID))
       FD->setType(Context.getNoReturnType(FD->getType()));
     if (Context.BuiltinInfo.isNoThrow(BuiltinID))
-      FD->addAttr(::new (Context) NoThrowAttr());
+      FD->addAttr(::new (Context) NoThrowAttr(FD->getLocation(), Context));
     if (Context.BuiltinInfo.isConst(BuiltinID))
-      FD->addAttr(::new (Context) ConstAttr());
+      FD->addAttr(::new (Context) ConstAttr(FD->getLocation(), Context));
   }
 
   IdentifierInfo *Name = FD->getIdentifier();
@@ -4942,13 +5181,15 @@ void Sema::AddKnownFunctionAttributes(FunctionDecl *FD) {
       // FIXME: We known better than our headers.
       const_cast<FormatAttr *>(Format)->setType(Context, "printf");
     } else
-      FD->addAttr(::new (Context) FormatAttr(Context, "printf", 1,
+      FD->addAttr(::new (Context) FormatAttr(FD->getLocation(), Context,
+                                             "printf", 1,
                                              Name->isStr("NSLogv") ? 0 : 2));
   } else if (Name->isStr("asprintf") || Name->isStr("vasprintf")) {
     // FIXME: asprintf and vasprintf aren't C99 functions. Should they be
     // target-specific builtins, perhaps?
     if (!FD->getAttr<FormatAttr>())
-      FD->addAttr(::new (Context) FormatAttr(Context, "printf", 2,
+      FD->addAttr(::new (Context) FormatAttr(FD->getLocation(), Context,
+                                             "printf", 2,
                                              Name->isStr("vasprintf") ? 0 : 3));
   }
 }
@@ -5031,7 +5272,7 @@ bool Sema::isAcceptableTagRedeclaration(const TagDecl *Previous,
 /// former case, Name will be non-null.  In the later case, Name will be null.
 /// TagSpec indicates what kind of tag this is. TUK indicates whether this is a
 /// reference/declaration/definition of a tag.
-Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
+Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
                                SourceLocation KWLoc, CXXScopeSpec &SS,
                                IdentifierInfo *Name, SourceLocation NameLoc,
                                AttributeList *Attr, AccessSpecifier AS,
@@ -5063,7 +5304,7 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
         // This is a declaration or definition of a class template (which may
         // be a member of another template).
         if (Invalid)
-          return DeclPtrTy();
+          return 0;
         
         OwnedDecl = false;
         DeclResult Result = CheckClassTemplate(S, TagSpec, TUK, KWLoc,
@@ -5106,26 +5347,26 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       DC = computeDeclContext(SS, false);
       if (!DC) {
         IsDependent = true;
-        return DeclPtrTy();
+        return 0;
       }
     } else {
       DC = computeDeclContext(SS, true);
       if (!DC) {
         Diag(SS.getRange().getBegin(), diag::err_dependent_nested_name_spec)
           << SS.getRange();
-        return DeclPtrTy();
+        return 0;
       }
     }
 
     if (RequireCompleteDeclContext(SS, DC))
-      return DeclPtrTy::make((Decl *)0);
+      return 0;
 
     SearchDC = DC;
     // Look-up name inside 'foo::'.
     LookupQualifiedName(Previous, DC);
 
     if (Previous.isAmbiguous())
-      return DeclPtrTy();
+      return 0;
 
     if (Previous.empty()) {
       // Name lookup did not find anything. However, if the
@@ -5135,7 +5376,7 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       // this as a dependent elaborated-type-specifier.
       if (Previous.wasNotFoundInCurrentInstantiation()) {
         IsDependent = true;
-        return DeclPtrTy();
+        return 0;
       }
 
       // A tag 'foo::bar' must already exist.
@@ -5155,7 +5396,7 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
 
     // Note:  there used to be some attempt at recovery here.
     if (Previous.isAmbiguous())
-      return DeclPtrTy();
+      return 0;
 
     if (!getLangOptions().CPlusPlus && TUK != TUK_Reference) {
       // FIXME: This makes sure that we ignore the contexts associated
@@ -5176,7 +5417,7 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   }
 
   if (getLangOptions().CPlusPlus && Name && DC && StdNamespace &&
-      DC->Equals(StdNamespace) && Name->isStr("bad_alloc")) {
+      DC->Equals(getStdNamespace()) && Name->isStr("bad_alloc")) {
     // This is a declaration of or a reference to "std::bad_alloc".
     isStdBadAlloc = true;
     
@@ -5184,7 +5425,7 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       // std::bad_alloc has been implicitly declared (but made invisible to
       // name lookup). Fill in this implicit declaration as the previous 
       // declaration, so that the declarations get chained appropriately.
-      Previous.addDecl(StdBadAlloc);
+      Previous.addDecl(getStdBadAlloc());
     }
   }
 
@@ -5271,11 +5512,12 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
         if (const TagType *TT = TD->getUnderlyingType()->getAs<TagType>()) {
           TagDecl *Tag = TT->getDecl();
           if (Tag->getDeclName() == Name &&
-              Tag->getDeclContext()->getLookupContext()
-                          ->Equals(TD->getDeclContext()->getLookupContext())) {
+              Tag->getDeclContext()->getRedeclContext()
+                          ->Equals(TD->getDeclContext()->getRedeclContext())) {
             PrevDecl = Tag;
             Previous.clear();
             Previous.addDecl(Tag);
+            Previous.resolveKind();
           }
         }
       }
@@ -5321,7 +5563,7 @@ Sema::DeclPtrTy Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
           // need to be changed with DeclGroups.
           if ((TUK == TUK_Reference && !PrevTagDecl->getFriendObjectKind()) ||
               TUK == TUK_Friend)
-            return DeclPtrTy::make(PrevTagDecl);
+            return PrevTagDecl;
 
           // Diagnose attempts to redefine a tag.
           if (TUK == TUK_Definition) {
@@ -5475,7 +5717,7 @@ CreateNewDecl:
       New = CXXRecordDecl::Create(Context, Kind, SearchDC, Loc, Name, KWLoc,
                                   cast_or_null<CXXRecordDecl>(PrevDecl));
       
-      if (isStdBadAlloc && (!StdBadAlloc || StdBadAlloc->isImplicit()))
+      if (isStdBadAlloc && (!StdBadAlloc || getStdBadAlloc()->isImplicit()))
         StdBadAlloc = cast<CXXRecordDecl>(New);
     } else
       New = RecordDecl::Create(Context, Kind, SearchDC, Loc, Name, KWLoc,
@@ -5549,7 +5791,7 @@ CreateNewDecl:
     if (PrevDecl)
       New->setAccess(PrevDecl->getAccess());
 
-    DeclContext *DC = New->getDeclContext()->getLookupContext();
+    DeclContext *DC = New->getDeclContext()->getRedeclContext();
     DC->makeDeclVisibleInContext(New, /* Recoverable = */ false);
     if (Name) // can be null along some error paths
       if (Scope *EnclosingScope = getScopeForDeclContext(S, DC))
@@ -5564,26 +5806,26 @@ CreateNewDecl:
   // If this is the C FILE type, notify the AST context.
   if (IdentifierInfo *II = New->getIdentifier())
     if (!New->isInvalidDecl() &&
-        New->getDeclContext()->getLookupContext()->isTranslationUnit() &&
+        New->getDeclContext()->getRedeclContext()->isTranslationUnit() &&
         II->isStr("FILE"))
       Context.setFILEDecl(New);
 
   OwnedDecl = true;
-  return DeclPtrTy::make(New);
+  return New;
 }
 
-void Sema::ActOnTagStartDefinition(Scope *S, DeclPtrTy TagD) {
+void Sema::ActOnTagStartDefinition(Scope *S, Decl *TagD) {
   AdjustDeclIfTemplate(TagD);
-  TagDecl *Tag = cast<TagDecl>(TagD.getAs<Decl>());
+  TagDecl *Tag = cast<TagDecl>(TagD);
   
   // Enter the tag context.
   PushDeclContext(S, Tag);
 }
 
-void Sema::ActOnStartCXXMemberDeclarations(Scope *S, DeclPtrTy TagD,
+void Sema::ActOnStartCXXMemberDeclarations(Scope *S, Decl *TagD,
                                            SourceLocation LBraceLoc) {
   AdjustDeclIfTemplate(TagD);
-  CXXRecordDecl *Record = cast<CXXRecordDecl>(TagD.getAs<Decl>());
+  CXXRecordDecl *Record = cast<CXXRecordDecl>(TagD);
 
   FieldCollector->StartClass();
 
@@ -5610,10 +5852,10 @@ void Sema::ActOnStartCXXMemberDeclarations(Scope *S, DeclPtrTy TagD,
          "Broken injected-class-name");
 }
 
-void Sema::ActOnTagFinishDefinition(Scope *S, DeclPtrTy TagD,
+void Sema::ActOnTagFinishDefinition(Scope *S, Decl *TagD,
                                     SourceLocation RBraceLoc) {
   AdjustDeclIfTemplate(TagD);
-  TagDecl *Tag = cast<TagDecl>(TagD.getAs<Decl>());
+  TagDecl *Tag = cast<TagDecl>(TagD);
   Tag->setRBraceLoc(RBraceLoc);
 
   if (isa<CXXRecordDecl>(Tag))
@@ -5626,9 +5868,9 @@ void Sema::ActOnTagFinishDefinition(Scope *S, DeclPtrTy TagD,
   Consumer.HandleTagDeclDefinition(Tag);
 }
 
-void Sema::ActOnTagDefinitionError(Scope *S, DeclPtrTy TagD) {
+void Sema::ActOnTagDefinitionError(Scope *S, Decl *TagD) {
   AdjustDeclIfTemplate(TagD);
-  TagDecl *Tag = cast<TagDecl>(TagD.getAs<Decl>());
+  TagDecl *Tag = cast<TagDecl>(TagD);
   Tag->setInvalidDecl();
 
   // We're undoing ActOnTagStartDefinition here, not
@@ -5711,13 +5953,13 @@ bool Sema::VerifyBitField(SourceLocation FieldLoc, IdentifierInfo *FieldName,
 
 /// ActOnField - Each field of a struct/union/class is passed into this in order
 /// to create a FieldDecl object for it.
-Sema::DeclPtrTy Sema::ActOnField(Scope *S, DeclPtrTy TagD,
+Decl *Sema::ActOnField(Scope *S, Decl *TagD,
                                  SourceLocation DeclStart,
                                  Declarator &D, ExprTy *BitfieldWidth) {
-  FieldDecl *Res = HandleField(S, cast_or_null<RecordDecl>(TagD.getAs<Decl>()),
+  FieldDecl *Res = HandleField(S, cast_or_null<RecordDecl>(TagD),
                                DeclStart, D, static_cast<Expr*>(BitfieldWidth),
                                AS_public);
-  return DeclPtrTy::make(Res);
+  return Res;
 }
 
 /// HandleField - Analyze a field of a C struct or a C++ data member.
@@ -5739,9 +5981,17 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record,
 
   if (D.getDeclSpec().isThreadSpecified())
     Diag(D.getDeclSpec().getThreadSpecLoc(), diag::err_invalid_thread);
+  
+  // Check to see if this name was declared as a member previously
+  LookupResult Previous(*this, II, Loc, LookupMemberName, ForRedeclaration);
+  LookupName(Previous, S);
+  assert((Previous.empty() || Previous.isOverloadedResult() || 
+          Previous.isSingleResult()) 
+    && "Lookup of member name should be either overloaded, single or null");
 
-  NamedDecl *PrevDecl = LookupSingleName(S, II, Loc, LookupMemberName,
-                                         ForRedeclaration);
+  // If the name is overloaded then get any declaration else get the single result
+  NamedDecl *PrevDecl = Previous.isOverloadedResult() ?
+    Previous.getRepresentativeDecl() : Previous.getAsSingle<NamedDecl>();
 
   if (PrevDecl && PrevDecl->isTemplateParameter()) {
     // Maybe we will complain about the shadowed template parameter.
@@ -5804,21 +6054,29 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
 
   QualType EltTy = Context.getBaseElementType(T);
   if (!EltTy->isDependentType() &&
-      RequireCompleteType(Loc, EltTy, diag::err_field_incomplete))
+      RequireCompleteType(Loc, EltTy, diag::err_field_incomplete)) {
+    // Fields of incomplete type force their record to be invalid.
+    Record->setInvalidDecl();
     InvalidDecl = true;
+  }
 
   // C99 6.7.2.1p8: A member of a structure or union may have any type other
   // than a variably modified type.
   if (!InvalidDecl && T->isVariablyModifiedType()) {
     bool SizeIsNegative;
+    llvm::APSInt Oversized;
     QualType FixedTy = TryToFixInvalidVariablyModifiedType(T, Context,
-                                                           SizeIsNegative);
+                                                           SizeIsNegative,
+                                                           Oversized);
     if (!FixedTy.isNull()) {
       Diag(Loc, diag::warn_illegal_constant_array_size);
       T = FixedTy;
     } else {
       if (SizeIsNegative)
         Diag(Loc, diag::err_typecheck_negative_array_size);
+      else if (Oversized.getBoolValue())
+        Diag(Loc, diag::err_array_too_large)
+          << Oversized.toString(10);
       else
         Diag(Loc, diag::err_typecheck_field_variable_size);
       InvalidDecl = true;
@@ -5836,7 +6094,6 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
   if (!InvalidDecl && BitWidth &&
       VerifyBitField(Loc, II, T, BitWidth, &ZeroWidth)) {
     InvalidDecl = true;
-    DeleteExpr(BitWidth);
     BitWidth = 0;
     ZeroWidth = false;
   }
@@ -5877,6 +6134,8 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
       CXXRecord->setPOD(false);
     if (!ZeroWidth)
       CXXRecord->setEmpty(false);
+    if (T->isReferenceType())
+      CXXRecord->setHasTrivialConstructor(false);
 
     if (const RecordType *RT = EltTy->getAs<RecordType>()) {
       CXXRecordDecl* RDecl = cast<CXXRecordDecl>(RT->getDecl());
@@ -5896,27 +6155,8 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
         // cannot be a member of a union, nor can an array of such
         // objects.
         // TODO: C++0x alters this restriction significantly.
-        if (Record->isUnion()) {
-          // We check for copy constructors before constructors
-          // because otherwise we'll never get complaints about
-          // copy constructors.
-
-          CXXSpecialMember member = CXXInvalid;
-          if (!RDecl->hasTrivialCopyConstructor())
-            member = CXXCopyConstructor;
-          else if (!RDecl->hasTrivialConstructor())
-            member = CXXConstructor;
-          else if (!RDecl->hasTrivialCopyAssignment())
-            member = CXXCopyAssignment;
-          else if (!RDecl->hasTrivialDestructor())
-            member = CXXDestructor;
-
-          if (member != CXXInvalid) {
-            Diag(Loc, diag::err_illegal_union_member) << Name << member;
-            DiagnoseNontrivial(RT, member);
-            NewFD->setInvalidDecl();
-          }
-        }
+        if (Record->isUnion() && CheckNontrivialField(NewFD))
+          NewFD->setInvalidDecl();
       }
     }
   }
@@ -5946,6 +6186,43 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
   return NewFD;
 }
 
+bool Sema::CheckNontrivialField(FieldDecl *FD) {
+  assert(FD);
+  assert(getLangOptions().CPlusPlus && "valid check only for C++");
+
+  if (FD->isInvalidDecl())
+    return true;
+
+  QualType EltTy = Context.getBaseElementType(FD->getType());
+  if (const RecordType *RT = EltTy->getAs<RecordType>()) {
+    CXXRecordDecl* RDecl = cast<CXXRecordDecl>(RT->getDecl());
+    if (RDecl->getDefinition()) {
+      // We check for copy constructors before constructors
+      // because otherwise we'll never get complaints about
+      // copy constructors.
+
+      CXXSpecialMember member = CXXInvalid;
+      if (!RDecl->hasTrivialCopyConstructor())
+        member = CXXCopyConstructor;
+      else if (!RDecl->hasTrivialConstructor())
+        member = CXXConstructor;
+      else if (!RDecl->hasTrivialCopyAssignment())
+        member = CXXCopyAssignment;
+      else if (!RDecl->hasTrivialDestructor())
+        member = CXXDestructor;
+
+      if (member != CXXInvalid) {
+        Diag(FD->getLocation(), diag::err_illegal_union_or_anon_struct_member)
+              << (int)FD->getParent()->isUnion() << FD->getDeclName() << member;
+        DiagnoseNontrivial(RT, member);
+        return true;
+      }
+    }
+  }
+  
+  return false;
+}
+
 /// DiagnoseNontrivial - Given that a class has a non-trivial
 /// special member, figure out why.
 void Sema::DiagnoseNontrivial(const RecordType* T, CXXSpecialMember member) {
@@ -6091,9 +6368,9 @@ TranslateIvarVisibility(tok::ObjCKeywordKind ivarVisibility) {
 
 /// ActOnIvar - Each ivar field of an objective-c class is passed into this
 /// in order to create an IvarDecl object for it.
-Sema::DeclPtrTy Sema::ActOnIvar(Scope *S,
+Decl *Sema::ActOnIvar(Scope *S,
                                 SourceLocation DeclStart,
-                                DeclPtrTy IntfDecl,
+                                Decl *IntfDecl,
                                 Declarator &D, ExprTy *BitfieldWidth,
                                 tok::ObjCKeywordKind Visibility) {
 
@@ -6112,7 +6389,6 @@ Sema::DeclPtrTy Sema::ActOnIvar(Scope *S,
     // 6.7.2.1p3, 6.7.2.1p4
     if (VerifyBitField(Loc, II, T, BitWidth)) {
       D.setInvalidType();
-      DeleteExpr(BitWidth);
       BitWidth = 0;
     }
   } else {
@@ -6137,19 +6413,23 @@ Sema::DeclPtrTy Sema::ActOnIvar(Scope *S,
     Visibility != tok::objc_not_keyword ? TranslateIvarVisibility(Visibility)
                                         : ObjCIvarDecl::None;
   // Must set ivar's DeclContext to its enclosing interface.
-  ObjCContainerDecl *EnclosingDecl = IntfDecl.getAs<ObjCContainerDecl>();
+  ObjCContainerDecl *EnclosingDecl = cast<ObjCContainerDecl>(IntfDecl);
   ObjCContainerDecl *EnclosingContext;
   if (ObjCImplementationDecl *IMPDecl =
       dyn_cast<ObjCImplementationDecl>(EnclosingDecl)) {
+    if (!LangOpts.ObjCNonFragileABI2) {
     // Case of ivar declared in an implementation. Context is that of its class.
-    EnclosingContext = IMPDecl->getClassInterface();
-    assert(EnclosingContext && "Implementation has no class interface!");
+      EnclosingContext = IMPDecl->getClassInterface();
+      assert(EnclosingContext && "Implementation has no class interface!");
+    }
+    else
+      EnclosingContext = EnclosingDecl;
   } else {
     if (ObjCCategoryDecl *CDecl = 
         dyn_cast<ObjCCategoryDecl>(EnclosingDecl)) {
       if (!LangOpts.ObjCNonFragileABI2 || !CDecl->IsClassExtension()) {
         Diag(Loc, diag::err_misplaced_ivar) << CDecl->IsClassExtension();
-        return DeclPtrTy();
+        return 0;
       }
     }
     EnclosingContext = EnclosingDecl;
@@ -6180,19 +6460,59 @@ Sema::DeclPtrTy Sema::ActOnIvar(Scope *S,
   if (II) {
     // FIXME: When interfaces are DeclContexts, we'll need to add
     // these to the interface.
-    S->AddDecl(DeclPtrTy::make(NewID));
+    S->AddDecl(NewID);
     IdResolver.AddDecl(NewID);
   }
 
-  return DeclPtrTy::make(NewID);
+  return NewID;
+}
+
+/// ActOnLastBitfield - This routine handles synthesized bitfields rules for 
+/// class and class extensions. For every class @interface and class 
+/// extension @interface, if the last ivar is a bitfield of any type, 
+/// then add an implicit `char :0` ivar to the end of that interface.
+void Sema::ActOnLastBitfield(SourceLocation DeclLoc, Decl *EnclosingDecl,
+                             llvm::SmallVectorImpl<Decl *> &AllIvarDecls) {
+  if (!LangOpts.ObjCNonFragileABI2 || AllIvarDecls.empty())
+    return;
+  
+  Decl *ivarDecl = AllIvarDecls[AllIvarDecls.size()-1];
+  ObjCIvarDecl *Ivar = cast<ObjCIvarDecl>(ivarDecl);
+  
+  if (!Ivar->isBitField())
+    return;
+  uint64_t BitFieldSize =
+    Ivar->getBitWidth()->EvaluateAsInt(Context).getZExtValue();
+  if (BitFieldSize == 0)
+    return;
+  ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(EnclosingDecl);
+  if (!ID) {
+    if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(EnclosingDecl)) {
+      if (!CD->IsClassExtension())
+        return;
+    }
+    // No need to add this to end of @implementation.
+    else
+      return;
+  }
+  // All conditions are met. Add a new bitfield to the tail end of ivars.
+  llvm::APInt Zero(Context.getTypeSize(Context.CharTy), 0);
+  Expr * BW = IntegerLiteral::Create(Context, Zero, Context.CharTy, DeclLoc);
+
+  Ivar = ObjCIvarDecl::Create(Context, cast<ObjCContainerDecl>(EnclosingDecl),
+                              DeclLoc, 0,
+                              Context.CharTy, 
+                              Context.CreateTypeSourceInfo(Context.CharTy),
+                              ObjCIvarDecl::Private, BW,
+                              true);
+  AllIvarDecls.push_back(Ivar);
 }
 
 void Sema::ActOnFields(Scope* S,
-                       SourceLocation RecLoc, DeclPtrTy RecDecl,
-                       DeclPtrTy *Fields, unsigned NumFields,
+                       SourceLocation RecLoc, Decl *EnclosingDecl,
+                       Decl **Fields, unsigned NumFields,
                        SourceLocation LBrac, SourceLocation RBrac,
                        AttributeList *Attr) {
-  Decl *EnclosingDecl = RecDecl.getAs<Decl>();
   assert(EnclosingDecl && "missing record or interface decl");
 
   // If the decl this is being inserted into is invalid, then it may be a
@@ -6209,7 +6529,7 @@ void Sema::ActOnFields(Scope* S,
 
   RecordDecl *Record = dyn_cast<RecordDecl>(EnclosingDecl);
   for (unsigned i = 0; i != NumFields; ++i) {
-    FieldDecl *FD = cast<FieldDecl>(Fields[i].getAs<Decl>());
+    FieldDecl *FD = cast<FieldDecl>(Fields[i]);
 
     // Get the type for the field.
     Type *FDTy = FD->getType().getTypePtr();
@@ -6244,7 +6564,7 @@ void Sema::ActOnFields(Scope* S,
       EnclosingDecl->setInvalidDecl();
       continue;
     } else if (FDTy->isIncompleteArrayType() && i == NumFields - 1 &&
-               Record && Record->isStruct()) {
+               Record && !Record->isUnion()) {
       // Flexible array member.
       if (NumNamedMembers < 1) {
         Diag(FD->getLocation(), diag::err_flexible_array_empty_struct)
@@ -6360,6 +6680,11 @@ void Sema::ActOnFields(Scope* S,
 
   if (Attr)
     ProcessDeclAttributeList(S, Record, Attr);
+
+  // If there's a #pragma GCC visibility in scope, and this isn't a subclass,
+  // set the visibility of this record.
+  if (Record && !Record->getDeclContext()->isRecord())
+    AddPushedVisibilityAttribute(Record);
 }
 
 /// \brief Determine whether the given integral value is representable within
@@ -6405,9 +6730,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
                                           EnumConstantDecl *LastEnumConst,
                                           SourceLocation IdLoc,
                                           IdentifierInfo *Id,
-                                          ExprArg val) {
-  Expr *Val = (Expr *)val.get();
-
+                                          Expr *Val) {
   unsigned IntWidth = Context.Target.getIntWidth();
   llvm::APSInt EnumVal(IntWidth);
   QualType EltTy;
@@ -6434,12 +6757,7 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
               << (EnumVal.isUnsigned() || EnumVal.isNonNegative());
           else if (!Context.hasSameType(Val->getType(), Context.IntTy)) {
             // Force the type of the expression to 'int'.
-            ImpCastExprToType(Val, Context.IntTy, CastExpr::CK_IntegralCast);
-            
-            if (Val != val.get()) {
-              val.release();
-              val = Val;
-            }
+            ImpCastExprToType(Val, Context.IntTy, CK_IntegralCast);
           }
         }
         
@@ -6527,20 +6845,19 @@ EnumConstantDecl *Sema::CheckEnumConstant(EnumDecl *Enum,
     EnumVal.setIsSigned(EltTy->isSignedIntegerType());
   }
   
-  val.release();
   return EnumConstantDecl::Create(Context, Enum, IdLoc, Id, EltTy,
                                   Val, EnumVal);
 }
 
 
-Sema::DeclPtrTy Sema::ActOnEnumConstant(Scope *S, DeclPtrTy theEnumDecl,
-                                        DeclPtrTy lastEnumConst,
+Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl,
+                                        Decl *lastEnumConst,
                                         SourceLocation IdLoc,
                                         IdentifierInfo *Id,
                                         SourceLocation EqualLoc, ExprTy *val) {
-  EnumDecl *TheEnumDecl = cast<EnumDecl>(theEnumDecl.getAs<Decl>());
+  EnumDecl *TheEnumDecl = cast<EnumDecl>(theEnumDecl);
   EnumConstantDecl *LastEnumConst =
-    cast_or_null<EnumConstantDecl>(lastEnumConst.getAs<Decl>());
+    cast_or_null<EnumConstantDecl>(lastEnumConst);
   Expr *Val = static_cast<Expr*>(val);
 
   // The scope passed in may not be a decl scope.  Zip up the scope tree until
@@ -6569,13 +6886,12 @@ Sema::DeclPtrTy Sema::ActOnEnumConstant(Scope *S, DeclPtrTy theEnumDecl,
       else
         Diag(IdLoc, diag::err_redefinition) << Id;
       Diag(PrevDecl->getLocation(), diag::note_previous_definition);
-      if (Val) Val->Destroy(Context);
-      return DeclPtrTy();
+      return 0;
     }
   }
 
   EnumConstantDecl *New = CheckEnumConstant(TheEnumDecl, LastEnumConst,
-                                            IdLoc, Id, Owned(Val));
+                                            IdLoc, Id, Val);
 
   // Register this decl in the current scope stack.
   if (New) {
@@ -6583,14 +6899,14 @@ Sema::DeclPtrTy Sema::ActOnEnumConstant(Scope *S, DeclPtrTy theEnumDecl,
     PushOnScopeChains(New, S);
   }
 
-  return DeclPtrTy::make(New);
+  return New;
 }
 
 void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
-                         SourceLocation RBraceLoc, DeclPtrTy EnumDeclX,
-                         DeclPtrTy *Elements, unsigned NumElements,
+                         SourceLocation RBraceLoc, Decl *EnumDeclX,
+                         Decl **Elements, unsigned NumElements,
                          Scope *S, AttributeList *Attr) {
-  EnumDecl *Enum = cast<EnumDecl>(EnumDeclX.getAs<Decl>());
+  EnumDecl *Enum = cast<EnumDecl>(EnumDeclX);
   QualType EnumType = Context.getTypeDeclType(Enum);
 
   if (Attr)
@@ -6599,7 +6915,7 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
   if (Enum->isDependentType()) {
     for (unsigned i = 0; i != NumElements; ++i) {
       EnumConstantDecl *ECD =
-        cast_or_null<EnumConstantDecl>(Elements[i].getAs<Decl>());
+        cast_or_null<EnumConstantDecl>(Elements[i]);
       if (!ECD) continue;
 
       ECD->setType(EnumType);
@@ -6626,7 +6942,7 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
 
   for (unsigned i = 0; i != NumElements; ++i) {
     EnumConstantDecl *ECD =
-      cast_or_null<EnumConstantDecl>(Elements[i].getAs<Decl>());
+      cast_or_null<EnumConstantDecl>(Elements[i]);
     if (!ECD) continue;  // Already issued a diagnostic.
 
     const llvm::APSInt &InitVal = ECD->getInitVal();
@@ -6728,8 +7044,7 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
   // Loop over all of the enumerator constants, changing their types to match
   // the type of the enum if needed.
   for (unsigned i = 0; i != NumElements; ++i) {
-    EnumConstantDecl *ECD =
-      cast_or_null<EnumConstantDecl>(Elements[i].getAs<Decl>());
+    EnumConstantDecl *ECD = cast_or_null<EnumConstantDecl>(Elements[i]);
     if (!ECD) continue;  // Already issued a diagnostic.
 
     // Standard C says the enumerators have int type, but we allow, as an
@@ -6772,11 +7087,11 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
 
     // Adjust the Expr initializer and type.
     if (ECD->getInitExpr())
-      ECD->setInitExpr(new (Context) ImplicitCastExpr(NewTy,
-                                                      CastExpr::CK_IntegralCast,
-                                                      ECD->getInitExpr(),
-                                                      CXXBaseSpecifierArray(),
-                                                      /*isLvalue=*/false));
+      ECD->setInitExpr(ImplicitCastExpr::Create(Context, NewTy,
+                                                CK_IntegralCast,
+                                                ECD->getInitExpr(),
+                                                /*base paths*/ 0,
+                                                VK_RValue));
     if (getLangOptions().CPlusPlus)
       // C++ [dcl.enum]p4: Following the closing brace of an
       // enum-specifier, each enumerator has the type of its
@@ -6790,14 +7105,13 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
                            NumPositiveBits, NumNegativeBits);
 }
 
-Sema::DeclPtrTy Sema::ActOnFileScopeAsmDecl(SourceLocation Loc,
-                                            ExprArg expr) {
-  StringLiteral *AsmString = cast<StringLiteral>(expr.takeAs<Expr>());
+Decl *Sema::ActOnFileScopeAsmDecl(SourceLocation Loc, Expr *expr) {
+  StringLiteral *AsmString = cast<StringLiteral>(expr);
 
   FileScopeAsmDecl *New = FileScopeAsmDecl::Create(Context, CurContext,
                                                    Loc, AsmString);
   CurContext->addDecl(New);
-  return DeclPtrTy::make(New);
+  return New;
 }
 
 void Sema::ActOnPragmaWeakID(IdentifierInfo* Name,
@@ -6806,7 +7120,7 @@ void Sema::ActOnPragmaWeakID(IdentifierInfo* Name,
   Decl *PrevDecl = LookupSingleName(TUScope, Name, NameLoc, LookupOrdinaryName);
 
   if (PrevDecl) {
-    PrevDecl->addAttr(::new (Context) WeakAttr());
+    PrevDecl->addAttr(::new (Context) WeakAttr(PragmaLoc, Context));
   } else {
     (void)WeakUndeclaredIdentifiers.insert(
       std::pair<IdentifierInfo*,WeakInfo>
diff --git a/lib/Sema/SemaDeclAttr.cpp b/lib/Sema/SemaDeclAttr.cpp
index 3b82f58be43e..25af73ae13d3 100644
--- a/lib/Sema/SemaDeclAttr.cpp
+++ b/lib/Sema/SemaDeclAttr.cpp
@@ -11,15 +11,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
+#include "clang/Sema/SemaInternal.h"
 #include "TargetAttributesSema.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/Basic/TargetInfo.h"
-#include "clang/Parse/DeclSpec.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/DelayedDiagnostic.h"
 #include "llvm/ADT/StringExtras.h"
 using namespace clang;
+using namespace sema;
 
 //===----------------------------------------------------------------------===//
 //  Helper functions
@@ -193,7 +196,7 @@ static void HandleExtVectorTypeAttr(Scope *scope, Decl *d,
 
   // Instantiate/Install the vector type, and let Sema build the type for us.
   // This will run the reguired checks.
-  QualType T = S.BuildExtVectorType(curType, S.Owned(sizeExpr), Attr.getLoc());
+  QualType T = S.BuildExtVectorType(curType, sizeExpr, Attr.getLoc());
   if (!T.isNull()) {
     // FIXME: preserve the old source info.
     tDecl->setTypeSourceInfo(S.Context.getTrivialTypeSourceInfo(T));
@@ -211,7 +214,7 @@ static void HandlePackedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   }
 
   if (TagDecl *TD = dyn_cast<TagDecl>(d))
-    TD->addAttr(::new (S.Context) PackedAttr);
+    TD->addAttr(::new (S.Context) PackedAttr(Attr.getLoc(), S.Context));
   else if (FieldDecl *FD = dyn_cast<FieldDecl>(d)) {
     // If the alignment is less than or equal to 8 bits, the packed attribute
     // has no effect.
@@ -220,7 +223,7 @@ static void HandlePackedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
       S.Diag(Attr.getLoc(), diag::warn_attribute_ignored_for_field_of_type)
         << Attr.getName() << FD->getType();
     else
-      FD->addAttr(::new (S.Context) PackedAttr);
+      FD->addAttr(::new (S.Context) PackedAttr(Attr.getLoc(), S.Context));
   } else
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << Attr.getName();
 }
@@ -235,7 +238,7 @@ static void HandleIBAction(Decl *d, const AttributeList &Attr, Sema &S) {
   // The IBAction attributes only apply to instance methods.
   if (ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(d))
     if (MD->isInstanceMethod()) {
-      d->addAttr(::new (S.Context) IBActionAttr());
+      d->addAttr(::new (S.Context) IBActionAttr(Attr.getLoc(), S.Context));
       return;
     }
 
@@ -252,7 +255,7 @@ static void HandleIBOutlet(Decl *d, const AttributeList &Attr, Sema &S) {
   // The IBOutlet attributes only apply to instance variables of
   // Objective-C classes.
   if (isa<ObjCIvarDecl>(d) || isa<ObjCPropertyDecl>(d)) {
-    d->addAttr(::new (S.Context) IBOutletAttr());
+    d->addAttr(::new (S.Context) IBOutletAttr(Attr.getLoc(), S.Context));
     return;
   }
 
@@ -263,7 +266,7 @@ static void HandleIBOutletCollection(Decl *d, const AttributeList &Attr,
                                      Sema &S) {
 
   // The iboutletcollection attribute can have zero or one arguments.
-  if (Attr.getNumArgs() > 1) {
+  if (Attr.getParameterName() && Attr.getNumArgs() > 0) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 1;
     return;
   }
@@ -274,9 +277,41 @@ static void HandleIBOutletCollection(Decl *d, const AttributeList &Attr,
     S.Diag(Attr.getLoc(), diag::err_attribute_iboutlet) << Attr.getName();
     return;
   }
-
-  // FIXME: Eventually accept the type argument.
-  d->addAttr(::new (S.Context) IBOutletCollectionAttr());
+  if (const ValueDecl *VD = dyn_cast<ValueDecl>(d))
+    if (!VD->getType()->getAs<ObjCObjectPointerType>()) {
+      S.Diag(Attr.getLoc(), diag::err_iboutletcollection_object_type) 
+        << VD->getType() << 0;
+      return;
+    }
+  if (const ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(d))
+    if (!PD->getType()->getAs<ObjCObjectPointerType>()) {
+      S.Diag(Attr.getLoc(), diag::err_iboutletcollection_object_type) 
+        << PD->getType() << 1;
+      return;
+    }
+  
+  IdentifierInfo *II = Attr.getParameterName();
+  if (!II)
+    II = &S.Context.Idents.get("id");
+  
+  ParsedType TypeRep = S.getTypeName(*II, Attr.getLoc(), 
+                        S.getScopeForContext(d->getDeclContext()->getParent()));
+  if (!TypeRep) {
+    S.Diag(Attr.getLoc(), diag::err_iboutletcollection_type) << II;
+    return;
+  }
+  QualType QT = TypeRep.get();
+  // Diagnose use of non-object type in iboutletcollection attribute.
+  // FIXME. Gnu attribute extension ignores use of builtin types in
+  // attributes. So, __attribute__((iboutletcollection(char))) will be
+  // treated as __attribute__((iboutletcollection())).
+  if (!QT->isObjCIdType() && !QT->isObjCClassType() &&
+      !QT->isObjCObjectType()) {
+    S.Diag(Attr.getLoc(), diag::err_iboutletcollection_type) << II;
+    return;
+  }
+  d->addAttr(::new (S.Context) IBOutletCollectionAttr(Attr.getLoc(), S.Context,
+                                                      QT));
 }
 
 static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -321,7 +356,7 @@ static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     QualType T = getFunctionOrMethodArgType(d, x);
     if (!T->isAnyPointerType() && !T->isBlockPointerType()) {
       // FIXME: Should also highlight argument in decl.
-      S.Diag(Attr.getLoc(), diag::err_nonnull_pointers_only)
+      S.Diag(Attr.getLoc(), diag::warn_nonnull_pointers_only)
         << "nonnull" << Ex->getSourceRange();
       continue;
     }
@@ -346,15 +381,162 @@ static void HandleNonNullAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   unsigned* start = &NonNullArgs[0];
   unsigned size = NonNullArgs.size();
-  std::sort(start, start + size);
-  d->addAttr(::new (S.Context) NonNullAttr(S.Context, start, size));
+  llvm::array_pod_sort(start, start + size);
+  d->addAttr(::new (S.Context) NonNullAttr(Attr.getLoc(), S.Context, start,
+                                           size));
+}
+
+static void HandleOwnershipAttr(Decl *d, const AttributeList &AL, Sema &S) {
+  // This attribute must be applied to a function declaration.
+  // The first argument to the attribute must be a string,
+  // the name of the resource, for example "malloc".
+  // The following arguments must be argument indexes, the arguments must be
+  // of integer type for Returns, otherwise of pointer type.
+  // The difference between Holds and Takes is that a pointer may still be used
+  // after being held.  free() should be __attribute((ownership_takes)), whereas
+  // a list append function may well be __attribute((ownership_holds)).
+
+  if (!AL.getParameterName()) {
+    S.Diag(AL.getLoc(), diag::err_attribute_argument_n_not_string)
+        << AL.getName()->getName() << 1;
+    return;
+  }
+  // Figure out our Kind, and check arguments while we're at it.
+  OwnershipAttr::OwnershipKind K;
+  switch (AL.getKind()) {
+  case AttributeList::AT_ownership_takes:
+    K = OwnershipAttr::Takes;
+    if (AL.getNumArgs() < 1) {
+      S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments) << 2;
+      return;
+    }
+    break;
+  case AttributeList::AT_ownership_holds:
+    K = OwnershipAttr::Holds;
+    if (AL.getNumArgs() < 1) {
+      S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments) << 2;
+      return;
+    }
+    break;
+  case AttributeList::AT_ownership_returns:
+    K = OwnershipAttr::Returns;
+    if (AL.getNumArgs() > 1) {
+      S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments)
+          << AL.getNumArgs() + 1;
+      return;
+    }
+    break;
+  default:
+    // This should never happen given how we are called.
+    llvm_unreachable("Unknown ownership attribute");
+  }
+
+  if (!isFunction(d) || !hasFunctionProto(d)) {
+    S.Diag(AL.getLoc(), diag::warn_attribute_wrong_decl_type) << AL.getName()
+        << 0 /*function*/;
+    return;
+  }
+
+  unsigned NumArgs = getFunctionOrMethodNumArgs(d);
+
+  llvm::StringRef Module = AL.getParameterName()->getName();
+
+  // Normalize the argument, __foo__ becomes foo.
+  if (Module.startswith("__") && Module.endswith("__"))
+    Module = Module.substr(2, Module.size() - 4);
+
+  llvm::SmallVector<unsigned, 10> OwnershipArgs;
+
+  for (AttributeList::arg_iterator I = AL.arg_begin(), E = AL.arg_end(); I != E;
+       ++I) {
+
+    Expr *IdxExpr = static_cast<Expr *>(*I);
+    llvm::APSInt ArgNum(32);
+    if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent()
+        || !IdxExpr->isIntegerConstantExpr(ArgNum, S.Context)) {
+      S.Diag(AL.getLoc(), diag::err_attribute_argument_not_int)
+          << AL.getName()->getName() << IdxExpr->getSourceRange();
+      continue;
+    }
+
+    unsigned x = (unsigned) ArgNum.getZExtValue();
+
+    if (x > NumArgs || x < 1) {
+      S.Diag(AL.getLoc(), diag::err_attribute_argument_out_of_bounds)
+          << AL.getName()->getName() << x << IdxExpr->getSourceRange();
+      continue;
+    }
+    --x;
+    switch (K) {
+    case OwnershipAttr::Takes:
+    case OwnershipAttr::Holds: {
+      // Is the function argument a pointer type?
+      QualType T = getFunctionOrMethodArgType(d, x);
+      if (!T->isAnyPointerType() && !T->isBlockPointerType()) {
+        // FIXME: Should also highlight argument in decl.
+        S.Diag(AL.getLoc(), diag::err_ownership_type)
+            << ((K==OwnershipAttr::Takes)?"ownership_takes":"ownership_holds")
+            << "pointer"
+            << IdxExpr->getSourceRange();
+        continue;
+      }
+      break;
+    }
+    case OwnershipAttr::Returns: {
+      if (AL.getNumArgs() > 1) {
+          // Is the function argument an integer type?
+          Expr *IdxExpr = static_cast<Expr *>(AL.getArg(0));
+          llvm::APSInt ArgNum(32);
+          if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent()
+              || !IdxExpr->isIntegerConstantExpr(ArgNum, S.Context)) {
+            S.Diag(AL.getLoc(), diag::err_ownership_type)
+                << "ownership_returns" << "integer"
+                << IdxExpr->getSourceRange();
+            return;
+          }
+      }
+      break;
+    }
+    default:
+      llvm_unreachable("Unknown ownership attribute");
+    } // switch
+
+    // Check we don't have a conflict with another ownership attribute.
+    for (specific_attr_iterator<OwnershipAttr>
+          i = d->specific_attr_begin<OwnershipAttr>(),
+          e = d->specific_attr_end<OwnershipAttr>();
+        i != e; ++i) {
+      if ((*i)->getOwnKind() != K) {
+        for (const unsigned *I = (*i)->args_begin(), *E = (*i)->args_end();
+             I!=E; ++I) {
+          if (x == *I) {
+            S.Diag(AL.getLoc(), diag::err_attributes_are_not_compatible)
+                << AL.getName()->getName() << "ownership_*";
+          }
+        }
+      }
+    }
+    OwnershipArgs.push_back(x);
+  }
+
+  unsigned* start = OwnershipArgs.data();
+  unsigned size = OwnershipArgs.size();
+  llvm::array_pod_sort(start, start + size);
+
+  if (K != OwnershipAttr::Returns && OwnershipArgs.empty()) {
+    S.Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments) << 2;
+    return;
+  }
+
+  d->addAttr(::new (S.Context) OwnershipAttr(AL.getLoc(), S.Context, K, Module,
+                                             start, size));
 }
 
 static bool isStaticVarOrStaticFunciton(Decl *D) {
   if (VarDecl *VD = dyn_cast<VarDecl>(D))
-    return VD->getStorageClass() == VarDecl::Static;
+    return VD->getStorageClass() == SC_Static;
   if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
-    return FD->getStorageClass() == FunctionDecl::Static;
+    return FD->getStorageClass() == SC_Static;
   return false;
 }
 
@@ -375,13 +557,11 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   //   static int a __attribute__((weakref ("v2")));
   // }
   // we reject them
-  if (const DeclContext *Ctx = d->getDeclContext()) {
-    Ctx = Ctx->getLookupContext();
-    if (!isa<TranslationUnitDecl>(Ctx) && !isa<NamespaceDecl>(Ctx) ) {
-      S.Diag(Attr.getLoc(), diag::err_attribute_weakref_not_global_context) <<
-	dyn_cast<NamedDecl>(d)->getNameAsString();
-      return;
-    }
+  const DeclContext *Ctx = d->getDeclContext()->getRedeclContext();
+  if (!Ctx->isFileContext()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_weakref_not_global_context) <<
+        dyn_cast<NamedDecl>(d)->getNameAsString();
+    return;
   }
 
   // The GCC manual says
@@ -424,10 +604,10 @@ static void HandleWeakRefAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     }
     // GCC will accept anything as the argument of weakref. Should we
     // check for an existing decl?
-    d->addAttr(::new (S.Context) AliasAttr(S.Context, Str->getString()));
+    d->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context, Str->getString()));
   }
 
-  d->addAttr(::new (S.Context) WeakRefAttr());
+  d->addAttr(::new (S.Context) WeakRefAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleAliasAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -449,7 +629,7 @@ static void HandleAliasAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   // FIXME: check if target symbol exists in current file
 
-  d->addAttr(::new (S.Context) AliasAttr(S.Context, Str->getString()));
+  d->addAttr(::new (S.Context) AliasAttr(Attr.getLoc(), S.Context, Str->getString()));
 }
 
 static void HandleAlwaysInlineAttr(Decl *d, const AttributeList &Attr,
@@ -466,7 +646,7 @@ static void HandleAlwaysInlineAttr(Decl *d, const AttributeList &Attr,
     return;
   }
 
-  d->addAttr(::new (S.Context) AlwaysInlineAttr());
+  d->addAttr(::new (S.Context) AlwaysInlineAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleMallocAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -479,7 +659,7 @@ static void HandleMallocAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(d)) {
     QualType RetTy = FD->getResultType();
     if (RetTy->isAnyPointerType() || RetTy->isBlockPointerType()) {
-      d->addAttr(::new (S.Context) MallocAttr());
+      d->addAttr(::new (S.Context) MallocAttr(Attr.getLoc(), S.Context));
       return;
     }
   }
@@ -487,39 +667,75 @@ static void HandleMallocAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   S.Diag(Attr.getLoc(), diag::warn_attribute_malloc_pointer_only);
 }
 
-static bool HandleCommonNoReturnAttr(Decl *d, const AttributeList &Attr,
-                                     Sema &S) {
-  // check the attribute arguments.
+static void HandleNoReturnAttr(Decl *d, const AttributeList &Attr, Sema &S) {
+  /* Diagnostics (if any) was emitted by Sema::ProcessFnAttr(). */
+  assert(Attr.isInvalid() == false);
+  d->addAttr(::new (S.Context) NoReturnAttr(Attr.getLoc(), S.Context));
+}
+
+static void HandleAnalyzerNoReturnAttr(Decl *d, const AttributeList &Attr,
+                                       Sema &S) {
+  
+  // The checking path for 'noreturn' and 'analyzer_noreturn' are different
+  // because 'analyzer_noreturn' does not impact the type.
+  
   if (Attr.getNumArgs() != 0) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
-    return false;
+    return;
   }
-
+  
   if (!isFunctionOrMethod(d) && !isa<BlockDecl>(d)) {
     ValueDecl *VD = dyn_cast<ValueDecl>(d);
     if (VD == 0 || (!VD->getType()->isBlockPointerType()
                     && !VD->getType()->isFunctionPointerType())) {
       S.Diag(Attr.getLoc(),
              Attr.isCXX0XAttribute() ? diag::err_attribute_wrong_decl_type
-                                     : diag::warn_attribute_wrong_decl_type)
-        << Attr.getName() << 0 /*function*/;
-      return false;
+             : diag::warn_attribute_wrong_decl_type)
+      << Attr.getName() << 0 /*function*/;
+      return;
     }
   }
-
-  return true;
-}
-
-static void HandleNoReturnAttr(Decl *d, const AttributeList &Attr, Sema &S) {
-  /* Diagnostics (if any) was emitted by Sema::ProcessFnAttr(). */
-  assert(Attr.isInvalid() == false);
-  d->addAttr(::new (S.Context) NoReturnAttr());
+  
+  d->addAttr(::new (S.Context) AnalyzerNoReturnAttr(Attr.getLoc(), S.Context));
 }
 
-static void HandleAnalyzerNoReturnAttr(Decl *d, const AttributeList &Attr,
+// PS3 PPU-specific.
+static void HandleVecReturnAttr(Decl *d, const AttributeList &Attr,
                                        Sema &S) {
-  if (HandleCommonNoReturnAttr(d, Attr, S))
-    d->addAttr(::new (S.Context) AnalyzerNoReturnAttr());
+/*
+  Returning a Vector Class in Registers
+  
+  According to the PPU ABI specifications, a class with a single member of vector type is returned in
+  memory when used as the return value of a function. This results in inefficient code when implementing
+  vector classes. To return the value in a single vector register, add the vecreturn attribute to the class
+  definition. This attribute is also applicable to struct types.
+  
+  Example:
+  
+  struct Vector
+  {
+    __vector float xyzw;
+  } __attribute__((vecreturn));
+  
+  Vector Add(Vector lhs, Vector rhs)
+  {
+    Vector result;
+    result.xyzw = vec_add(lhs.xyzw, rhs.xyzw);
+    return result; // This will be returned in a register
+  }
+*/
+  if (!isa<CXXRecordDecl>(d)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_decl_type)
+      << Attr.getName() << 9 /*class*/;
+    return;
+  }
+
+  if (d->getAttr<VecReturnAttr>()) {
+    S.Diag(Attr.getLoc(), diag::err_repeat_attribute) << "vecreturn";
+    return;
+  }
+
+  d->addAttr(::new (S.Context) VecReturnAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleDependencyAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -545,7 +761,7 @@ static void HandleUnusedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) UnusedAttr());
+  d->addAttr(::new (S.Context) UnusedAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleUsedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -566,7 +782,7 @@ static void HandleUsedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) UsedAttr());
+  d->addAttr(::new (S.Context) UsedAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleConstructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -596,7 +812,7 @@ static void HandleConstructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) ConstructorAttr(priority));
+  d->addAttr(::new (S.Context) ConstructorAttr(Attr.getLoc(), S.Context, priority));
 }
 
 static void HandleDestructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -626,7 +842,7 @@ static void HandleDestructorAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) DestructorAttr(priority));
+  d->addAttr(::new (S.Context) DestructorAttr(Attr.getLoc(), S.Context, priority));
 }
 
 static void HandleDeprecatedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -636,7 +852,7 @@ static void HandleDeprecatedAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) DeprecatedAttr());
+  d->addAttr(::new (S.Context) DeprecatedAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleUnavailableAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -646,7 +862,7 @@ static void HandleUnavailableAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) UnavailableAttr());
+  d->addAttr(::new (S.Context) UnavailableAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleVisibilityAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -667,22 +883,22 @@ static void HandleVisibilityAttr(Decl *d, const AttributeList &Attr, Sema &S) {
   }
 
   llvm::StringRef TypeStr = Str->getString();
-  VisibilityAttr::VisibilityTypes type;
+  VisibilityAttr::VisibilityType type;
 
   if (TypeStr == "default")
-    type = VisibilityAttr::DefaultVisibility;
+    type = VisibilityAttr::Default;
   else if (TypeStr == "hidden")
-    type = VisibilityAttr::HiddenVisibility;
+    type = VisibilityAttr::Hidden;
   else if (TypeStr == "internal")
-    type = VisibilityAttr::HiddenVisibility; // FIXME
+    type = VisibilityAttr::Hidden; // FIXME
   else if (TypeStr == "protected")
-    type = VisibilityAttr::ProtectedVisibility;
+    type = VisibilityAttr::Protected;
   else {
     S.Diag(Attr.getLoc(), diag::warn_attribute_unknown_visibility) << TypeStr;
     return;
   }
 
-  d->addAttr(::new (S.Context) VisibilityAttr(type));
+  d->addAttr(::new (S.Context) VisibilityAttr(Attr.getLoc(), S.Context, type));
 }
 
 static void HandleObjCExceptionAttr(Decl *D, const AttributeList &Attr,
@@ -698,7 +914,7 @@ static void HandleObjCExceptionAttr(Decl *D, const AttributeList &Attr,
     return;
   }
 
-  D->addAttr(::new (S.Context) ObjCExceptionAttr());
+  D->addAttr(::new (S.Context) ObjCExceptionAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleObjCNSObject(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -714,7 +930,7 @@ static void HandleObjCNSObject(Decl *D, const AttributeList &Attr, Sema &S) {
       return;
     }
   }
-  D->addAttr(::new (S.Context) ObjCNSObjectAttr());
+  D->addAttr(::new (S.Context) ObjCNSObjectAttr(Attr.getLoc(), S.Context));
 }
 
 static void
@@ -729,7 +945,7 @@ HandleOverloadableAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  D->addAttr(::new (S.Context) OverloadableAttr());
+  D->addAttr(::new (S.Context) OverloadableAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleBlocksAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -744,7 +960,7 @@ static void HandleBlocksAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  BlocksAttr::BlocksAttrTypes type;
+  BlocksAttr::BlockType type;
   if (Attr.getParameterName()->isStr("byref"))
     type = BlocksAttr::ByRef;
   else {
@@ -753,7 +969,7 @@ static void HandleBlocksAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) BlocksAttr(type));
+  d->addAttr(::new (S.Context) BlocksAttr(Attr.getLoc(), S.Context, type));
 }
 
 static void HandleSentinelAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -846,7 +1062,7 @@ static void HandleSentinelAttr(Decl *d, const AttributeList &Attr, Sema &S) {
       << Attr.getName() << 6 /*function, method or block */;
     return;
   }
-  d->addAttr(::new (S.Context) SentinelAttr(sentinel, nullPos));
+  d->addAttr(::new (S.Context) SentinelAttr(Attr.getLoc(), S.Context, sentinel, nullPos));
 }
 
 static void HandleWarnUnusedResult(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -874,7 +1090,7 @@ static void HandleWarnUnusedResult(Decl *D, const AttributeList &Attr, Sema &S)
       return;
     }
   
-  D->addAttr(::new (S.Context) WarnUnusedResultAttr());
+  D->addAttr(::new (S.Context) WarnUnusedResultAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleWeakAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -898,7 +1114,7 @@ static void HandleWeakAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  D->addAttr(::new (S.Context) WeakAttr());
+  D->addAttr(::new (S.Context) WeakAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleWeakImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -934,7 +1150,7 @@ static void HandleWeakImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  D->addAttr(::new (S.Context) WeakImportAttr());
+  D->addAttr(::new (S.Context) WeakImportAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleReqdWorkGroupSize(Decl *D, const AttributeList &Attr,
@@ -957,7 +1173,8 @@ static void HandleReqdWorkGroupSize(Decl *D, const AttributeList &Attr,
     }
     WGSize[i] = (unsigned) ArgNum.getZExtValue();
   }
-  D->addAttr(::new (S.Context) ReqdWorkGroupSizeAttr(WGSize[0], WGSize[1],
+  D->addAttr(::new (S.Context) ReqdWorkGroupSizeAttr(Attr.getLoc(), S.Context,
+                                                     WGSize[0], WGSize[1],
                                                      WGSize[2]));
 }
 
@@ -991,7 +1208,7 @@ static void HandleSectionAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
   
-  D->addAttr(::new (S.Context) SectionAttr(S.Context, SE->getString()));
+  D->addAttr(::new (S.Context) SectionAttr(Attr.getLoc(), S.Context, SE->getString()));
 }
 
 
@@ -1002,7 +1219,7 @@ static void HandleNothrowAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) NoThrowAttr());
+  d->addAttr(::new (S.Context) NoThrowAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleConstAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1012,7 +1229,7 @@ static void HandleConstAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) ConstAttr());
+  d->addAttr(::new (S.Context) ConstAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandlePureAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1022,7 +1239,7 @@ static void HandlePureAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) PureAttr());
+  d->addAttr(::new (S.Context) PureAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleCleanupAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1080,7 +1297,7 @@ static void HandleCleanupAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) CleanupAttr(FD));
+  d->addAttr(::new (S.Context) CleanupAttr(Attr.getLoc(), S.Context, FD));
 }
 
 /// Handle __attribute__((format_arg((idx)))) attribute based on
@@ -1143,7 +1360,7 @@ static void HandleFormatArgAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) FormatArgAttr(Idx.getZExtValue()));
+  d->addAttr(::new (S.Context) FormatArgAttr(Attr.getLoc(), S.Context, Idx.getZExtValue()));
 }
 
 enum FormatAttrKind {
@@ -1225,7 +1442,7 @@ static void HandleInitPriorityAttr(Decl *d, const AttributeList &Attr,
     Attr.setInvalid();
     return;
   }
-  d->addAttr(::new (S.Context) InitPriorityAttr(prioritynum));
+  d->addAttr(::new (S.Context) InitPriorityAttr(Attr.getLoc(), S.Context, prioritynum));
 }
 
 /// Handle __attribute__((format(type,idx,firstarg))) attributes based on
@@ -1369,7 +1586,8 @@ static void HandleFormatAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) FormatAttr(S.Context, Format, Idx.getZExtValue(),
+  d->addAttr(::new (S.Context) FormatAttr(Attr.getLoc(), S.Context, Format,
+                                          Idx.getZExtValue(),
                                           FirstArg.getZExtValue()));
 }
 
@@ -1438,7 +1656,7 @@ static void HandleTransparentUnionAttr(Decl *d, const AttributeList &Attr,
     }
   }
 
-  RD->addAttr(::new (S.Context) TransparentUnionAttr());
+  RD->addAttr(::new (S.Context) TransparentUnionAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleAnnotateAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1456,7 +1674,7 @@ static void HandleAnnotateAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     S.Diag(ArgExpr->getLocStart(), diag::err_attribute_not_string) <<"annotate";
     return;
   }
-  d->addAttr(::new (S.Context) AnnotateAttr(S.Context, SE->getString()));
+  d->addAttr(::new (S.Context) AnnotateAttr(Attr.getLoc(), S.Context, SE->getString()));
 }
 
 static void HandleAlignedAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -1471,9 +1689,7 @@ static void HandleAlignedAttr(Decl *D, const AttributeList &Attr, Sema &S) {
   //       weaker alignment, rather than being silently ignored.
 
   if (Attr.getNumArgs() == 0) {
-    // FIXME: This should be the target specific maximum alignment.
-    // (For now we just use 128 bits which is the maximum on X86).
-    D->addAttr(::new (S.Context) AlignedAttr(128));
+    D->addAttr(::new (S.Context) AlignedAttr(Attr.getLoc(), S.Context, true, 0));
     return;
   }
 
@@ -1483,10 +1699,11 @@ static void HandleAlignedAttr(Decl *D, const AttributeList &Attr, Sema &S) {
 void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E) {
   if (E->isTypeDependent() || E->isValueDependent()) {
     // Save dependent expressions in the AST to be instantiated.
-    D->addAttr(::new (Context) AlignedAttr(E));
+    D->addAttr(::new (Context) AlignedAttr(AttrLoc, Context, true, E));
     return;
   }
 
+  // FIXME: Cache the number on the Attr object?
   llvm::APSInt Alignment(32);
   if (!E->isIntegerConstantExpr(Alignment, Context)) {
     Diag(AttrLoc, diag::err_attribute_argument_not_int)
@@ -1499,7 +1716,14 @@ void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, Expr *E) {
     return;
   }
 
-  D->addAttr(::new (Context) AlignedAttr(Alignment.getZExtValue() * 8));
+  D->addAttr(::new (Context) AlignedAttr(AttrLoc, Context, true, E));
+}
+
+void Sema::AddAlignedAttr(SourceLocation AttrLoc, Decl *D, TypeSourceInfo *TS) {
+  // FIXME: Cache the number on the Attr object if non-dependent?
+  // FIXME: Perform checking of type validity
+  D->addAttr(::new (Context) AlignedAttr(AttrLoc, Context, false, TS));
+  return;
 }
 
 /// HandleModeAttr - This attribute modifies the width of a decl with primitive
@@ -1686,7 +1910,7 @@ static void HandleNoDebugAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) NoDebugAttr());
+  d->addAttr(::new (S.Context) NoDebugAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleNoInlineAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1702,7 +1926,7 @@ static void HandleNoInlineAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) NoInlineAttr());
+  d->addAttr(::new (S.Context) NoInlineAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleNoInstrumentFunctionAttr(Decl *d, const AttributeList &Attr,
@@ -1719,7 +1943,7 @@ static void HandleNoInstrumentFunctionAttr(Decl *d, const AttributeList &Attr,
     return;
   }
 
-  d->addAttr(::new (S.Context) NoInstrumentFunctionAttr());
+  d->addAttr(::new (S.Context) NoInstrumentFunctionAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleGNUInlineAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1741,7 +1965,7 @@ static void HandleGNUInlineAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) GNUInlineAttr());
+  d->addAttr(::new (S.Context) GNUInlineAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleCallConvAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1751,15 +1975,19 @@ static void HandleCallConvAttr(Decl *d, const AttributeList &Attr, Sema &S) {
 
   switch (Attr.getKind()) {
   case AttributeList::AT_fastcall:
-    d->addAttr(::new (S.Context) FastCallAttr());
+    d->addAttr(::new (S.Context) FastCallAttr(Attr.getLoc(), S.Context));
     return;
   case AttributeList::AT_stdcall:
-    d->addAttr(::new (S.Context) StdCallAttr());
+    d->addAttr(::new (S.Context) StdCallAttr(Attr.getLoc(), S.Context));
     return;
   case AttributeList::AT_thiscall:
-    d->addAttr(::new (S.Context) ThisCallAttr());
+    d->addAttr(::new (S.Context) ThisCallAttr(Attr.getLoc(), S.Context));
+    return;
   case AttributeList::AT_cdecl:
-    d->addAttr(::new (S.Context) CDeclAttr());
+    d->addAttr(::new (S.Context) CDeclAttr(Attr.getLoc(), S.Context));
+    return;
+  case AttributeList::AT_pascal:
+    d->addAttr(::new (S.Context) PascalAttr(Attr.getLoc(), S.Context));
     return;
   default:
     llvm_unreachable("unexpected attribute kind");
@@ -1801,7 +2029,8 @@ static void HandleRegparmAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) RegparmAttr(NumParams.getZExtValue()));
+  d->addAttr(::new (S.Context) RegparmAttr(Attr.getLoc(), S.Context,
+                                           NumParams.getZExtValue()));
 }
 
 static void HandleFinalAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1827,7 +2056,7 @@ static void HandleFinalAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) FinalAttr());
+  d->addAttr(::new (S.Context) FinalAttr(Attr.getLoc(), S.Context));
 }
 
 //===----------------------------------------------------------------------===//
@@ -1853,7 +2082,7 @@ static void HandleBaseCheckAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
   
-  d->addAttr(::new (S.Context) BaseCheckAttr());
+  d->addAttr(::new (S.Context) BaseCheckAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleHidingAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1878,7 +2107,7 @@ static void HandleHidingAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) HidingAttr());
+  d->addAttr(::new (S.Context) HidingAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleOverrideAttr(Decl *d, const AttributeList &Attr, Sema &S) {
@@ -1903,7 +2132,7 @@ static void HandleOverrideAttr(Decl *d, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  d->addAttr(::new (S.Context) OverrideAttr());
+  d->addAttr(::new (S.Context) OverrideAttr(Attr.getLoc(), S.Context));
 }
 
 //===----------------------------------------------------------------------===//
@@ -1939,16 +2168,16 @@ static void HandleNSReturnsRetainedAttr(Decl *d, const AttributeList &Attr,
       assert(0 && "invalid ownership attribute");
       return;
     case AttributeList::AT_cf_returns_not_retained:
-      d->addAttr(::new (S.Context) CFReturnsNotRetainedAttr());
+      d->addAttr(::new (S.Context) CFReturnsNotRetainedAttr(Attr.getLoc(), S.Context));
       return;
     case AttributeList::AT_ns_returns_not_retained:
-      d->addAttr(::new (S.Context) NSReturnsNotRetainedAttr());
+      d->addAttr(::new (S.Context) NSReturnsNotRetainedAttr(Attr.getLoc(), S.Context));
       return;
     case AttributeList::AT_cf_returns_retained:
-      d->addAttr(::new (S.Context) CFReturnsRetainedAttr());
+      d->addAttr(::new (S.Context) CFReturnsRetainedAttr(Attr.getLoc(), S.Context));
       return;
     case AttributeList::AT_ns_returns_retained:
-      d->addAttr(::new (S.Context) NSReturnsRetainedAttr());
+      d->addAttr(::new (S.Context) NSReturnsRetainedAttr(Attr.getLoc(), S.Context));
       return;
   };
 }
@@ -2009,9 +2238,14 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   case AttributeList::AT_mode:        HandleModeAttr        (D, Attr, S); break;
   case AttributeList::AT_malloc:      HandleMallocAttr      (D, Attr, S); break;
   case AttributeList::AT_nonnull:     HandleNonNullAttr     (D, Attr, S); break;
+  case AttributeList::AT_ownership_returns:
+  case AttributeList::AT_ownership_takes:
+  case AttributeList::AT_ownership_holds:
+      HandleOwnershipAttr     (D, Attr, S); break;
   case AttributeList::AT_noreturn:    HandleNoReturnAttr    (D, Attr, S); break;
   case AttributeList::AT_nothrow:     HandleNothrowAttr     (D, Attr, S); break;
   case AttributeList::AT_override:    HandleOverrideAttr    (D, Attr, S); break;
+  case AttributeList::AT_vecreturn:   HandleVecReturnAttr   (D, Attr, S); break;
 
   // Checker-specific.
   case AttributeList::AT_ns_returns_not_retained:
@@ -2063,6 +2297,7 @@ static void ProcessDeclAttribute(Scope *scope, Decl *D,
   case AttributeList::AT_cdecl:
   case AttributeList::AT_fastcall:
   case AttributeList::AT_thiscall:
+  case AttributeList::AT_pascal:
     HandleCallConvAttr(D, Attr, S);
     break;
   default:
@@ -2087,7 +2322,7 @@ void Sema::ProcessDeclAttributeList(Scope *S, Decl *D, const AttributeList *Attr
   // but that looks really pointless. We reject it.
   if (D->hasAttr<WeakRefAttr>() && !D->hasAttr<AliasAttr>()) {
     Diag(AttrList->getLoc(), diag::err_attribute_weakref_without_alias) <<
-	dyn_cast<NamedDecl>(D)->getNameAsString();
+    dyn_cast<NamedDecl>(D)->getNameAsString();
     return;
   }
 }
@@ -2127,8 +2362,9 @@ void Sema::DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, WeakInfo &W) {
   if (W.getAlias()) { // clone decl, impersonate __attribute(weak,alias(...))
     IdentifierInfo *NDId = ND->getIdentifier();
     NamedDecl *NewD = DeclClonePragmaWeak(ND, W.getAlias());
-    NewD->addAttr(::new (Context) AliasAttr(Context, NDId->getName()));
-    NewD->addAttr(::new (Context) WeakAttr());
+    NewD->addAttr(::new (Context) AliasAttr(W.getLocation(), Context,
+                                            NDId->getName()));
+    NewD->addAttr(::new (Context) WeakAttr(W.getLocation(), Context));
     WeakTopLevelDecl.push_back(NewD);
     // FIXME: "hideous" code from Sema::LazilyCreateBuiltin
     // to insert Decl at TU scope, sorry.
@@ -2137,7 +2373,7 @@ void Sema::DeclApplyPragmaWeak(Scope *S, NamedDecl *ND, WeakInfo &W) {
     PushOnScopeChains(NewD, S);
     CurContext = SavedContext;
   } else { // just add weak to existing
-    ND->addAttr(::new (Context) WeakAttr());
+    ND->addAttr(::new (Context) WeakAttr(W.getLocation(), Context));
   }
 }
 
@@ -2179,12 +2415,12 @@ void Sema::ProcessDeclAttributes(Scope *S, Decl *D, const Declarator &PD) {
 ///
 /// The state token we use is the start index of this scope
 /// on the warning stack.
-Action::ParsingDeclStackState Sema::PushParsingDeclaration() {
+Sema::ParsingDeclStackState Sema::PushParsingDeclaration() {
   ParsingDeclDepth++;
   return (ParsingDeclStackState) DelayedDiagnostics.size();
 }
 
-void Sema::PopParsingDeclaration(ParsingDeclStackState S, DeclPtrTy Ctx) {
+void Sema::PopParsingDeclaration(ParsingDeclStackState S, Decl *D) {
   assert(ParsingDeclDepth > 0 && "empty ParsingDeclaration stack");
   ParsingDeclDepth--;
 
@@ -2199,7 +2435,6 @@ void Sema::PopParsingDeclaration(ParsingDeclStackState S, DeclPtrTy Ctx) {
 
   // We only want to actually emit delayed diagnostics when we
   // successfully parsed a decl.
-  Decl *D = Ctx ? Ctx.getAs<Decl>() : 0;
   if (D) {
     // We really do want to start with 0 here.  We get one push for a
     // decl spec and another for each declarator;  in a decl group like:
@@ -2238,7 +2473,7 @@ static bool isDeclDeprecated(Decl *D) {
   return false;
 }
 
-void Sema::HandleDelayedDeprecationCheck(Sema::DelayedDiagnostic &DD,
+void Sema::HandleDelayedDeprecationCheck(DelayedDiagnostic &DD,
                                          Decl *Ctx) {
   if (isDeclDeprecated(Ctx))
     return;
diff --git a/lib/Sema/SemaDeclCXX.cpp b/lib/Sema/SemaDeclCXX.cpp
index bd97df2ce9d1..63acdb5f1cc6 100644
--- a/lib/Sema/SemaDeclCXX.cpp
+++ b/lib/Sema/SemaDeclCXX.cpp
@@ -11,9 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/CXXFieldCollector.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CharUnits.h"
@@ -23,10 +25,11 @@
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeOrdering.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include <map>
 #include <set>
@@ -108,7 +111,7 @@ namespace {
 }
 
 bool
-Sema::SetParamDefaultArgument(ParmVarDecl *Param, ExprArg DefaultArg,
+Sema::SetParamDefaultArgument(ParmVarDecl *Param, Expr *Arg,
                               SourceLocation EqualLoc) {
   if (RequireCompleteType(Param->getLocation(), Param->getType(),
                           diag::err_typecheck_decl_incomplete_type)) {
@@ -116,8 +119,6 @@ Sema::SetParamDefaultArgument(ParmVarDecl *Param, ExprArg DefaultArg,
     return true;
   }
 
-  Expr *Arg = (Expr *)DefaultArg.get();
-
   // C++ [dcl.fct.default]p5
   //   A default argument expression is implicitly converted (clause
   //   4) to the parameter type. The default argument expression has
@@ -128,8 +129,8 @@ Sema::SetParamDefaultArgument(ParmVarDecl *Param, ExprArg DefaultArg,
   InitializationKind Kind = InitializationKind::CreateCopy(Param->getLocation(),
                                                            EqualLoc);
   InitializationSequence InitSeq(*this, Entity, Kind, &Arg, 1);
-  OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                          MultiExprArg(*this, (void**)&Arg, 1));
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
+                                            MultiExprArg(*this, &Arg, 1));
   if (Result.isInvalid())
     return true;
   Arg = Result.takeAs<Expr>();
@@ -139,8 +140,6 @@ Sema::SetParamDefaultArgument(ParmVarDecl *Param, ExprArg DefaultArg,
   // Okay: add the default argument to the parameter
   Param->setDefaultArg(Arg);
 
-  DefaultArg.release();
-
   return false;
 }
 
@@ -148,16 +147,14 @@ Sema::SetParamDefaultArgument(ParmVarDecl *Param, ExprArg DefaultArg,
 /// provided for a function parameter is well-formed. If so, attach it
 /// to the parameter declaration.
 void
-Sema::ActOnParamDefaultArgument(DeclPtrTy param, SourceLocation EqualLoc,
-                                ExprArg defarg) {
-  if (!param || !defarg.get())
+Sema::ActOnParamDefaultArgument(Decl *param, SourceLocation EqualLoc,
+                                Expr *DefaultArg) {
+  if (!param || !DefaultArg)
     return;
 
-  ParmVarDecl *Param = cast<ParmVarDecl>(param.getAs<Decl>());
+  ParmVarDecl *Param = cast<ParmVarDecl>(param);
   UnparsedDefaultArgLocs.erase(Param);
 
-  ExprOwningPtr<Expr> DefaultArg(this, defarg.takeAs<Expr>());
-
   // Default arguments are only permitted in C++
   if (!getLangOptions().CPlusPlus) {
     Diag(EqualLoc, diag::err_param_default_argument)
@@ -167,26 +164,26 @@ Sema::ActOnParamDefaultArgument(DeclPtrTy param, SourceLocation EqualLoc,
   }
 
   // Check that the default argument is well-formed
-  CheckDefaultArgumentVisitor DefaultArgChecker(DefaultArg.get(), this);
-  if (DefaultArgChecker.Visit(DefaultArg.get())) {
+  CheckDefaultArgumentVisitor DefaultArgChecker(DefaultArg, this);
+  if (DefaultArgChecker.Visit(DefaultArg)) {
     Param->setInvalidDecl();
     return;
   }
 
-  SetParamDefaultArgument(Param, move(DefaultArg), EqualLoc);
+  SetParamDefaultArgument(Param, DefaultArg, EqualLoc);
 }
 
 /// ActOnParamUnparsedDefaultArgument - We've seen a default
 /// argument for a function parameter, but we can't parse it yet
 /// because we're inside a class definition. Note that this default
 /// argument will be parsed later.
-void Sema::ActOnParamUnparsedDefaultArgument(DeclPtrTy param,
+void Sema::ActOnParamUnparsedDefaultArgument(Decl *param,
                                              SourceLocation EqualLoc,
                                              SourceLocation ArgLoc) {
   if (!param)
     return;
 
-  ParmVarDecl *Param = cast<ParmVarDecl>(param.getAs<Decl>());
+  ParmVarDecl *Param = cast<ParmVarDecl>(param);
   if (Param)
     Param->setUnparsedDefaultArg();
 
@@ -195,11 +192,11 @@ void Sema::ActOnParamUnparsedDefaultArgument(DeclPtrTy param,
 
 /// ActOnParamDefaultArgumentError - Parsing or semantic analysis of
 /// the default argument for the parameter param failed.
-void Sema::ActOnParamDefaultArgumentError(DeclPtrTy param) {
+void Sema::ActOnParamDefaultArgumentError(Decl *param) {
   if (!param)
     return;
 
-  ParmVarDecl *Param = cast<ParmVarDecl>(param.getAs<Decl>());
+  ParmVarDecl *Param = cast<ParmVarDecl>(param);
 
   Param->setInvalidDecl();
 
@@ -224,7 +221,7 @@ void Sema::CheckExtraCXXDefaultArguments(Declarator &D) {
     if (chunk.Kind == DeclaratorChunk::Function) {
       for (unsigned argIdx = 0, e = chunk.Fun.NumArgs; argIdx != e; ++argIdx) {
         ParmVarDecl *Param =
-          cast<ParmVarDecl>(chunk.Fun.ArgInfo[argIdx].Param.getAs<Decl>());
+          cast<ParmVarDecl>(chunk.Fun.ArgInfo[argIdx].Param);
         if (Param->hasUnparsedDefaultArg()) {
           CachedTokens *Toks = chunk.Fun.ArgInfo[argIdx].DefaultArgTokens;
           Diag(Param->getLocation(), diag::err_param_default_argument_nonfunc)
@@ -412,8 +409,6 @@ void Sema::CheckCXXDefaultArguments(FunctionDecl *FD) {
     for (p = 0; p <= LastMissingDefaultArg; ++p) {
       ParmVarDecl *Param = FD->getParamDecl(p);
       if (Param->hasDefaultArg()) {
-        if (!Param->hasUnparsedDefaultArg())
-          Param->getDefaultArg()->Destroy(Context);
         Param->setDefaultArg(0);
       }
     }
@@ -449,8 +444,9 @@ CXXBaseSpecifier *
 Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
                          SourceRange SpecifierRange,
                          bool Virtual, AccessSpecifier Access,
-                         QualType BaseType,
-                         SourceLocation BaseLoc) {
+                         TypeSourceInfo *TInfo) {
+  QualType BaseType = TInfo->getType();
+
   // C++ [class.union]p1:
   //   A union shall not have base classes.
   if (Class->isUnion()) {
@@ -461,8 +457,10 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
 
   if (BaseType->isDependentType())
     return new (Context) CXXBaseSpecifier(SpecifierRange, Virtual,
-                                Class->getTagKind() == TTK_Class,
-                                Access, BaseType);
+                                          Class->getTagKind() == TTK_Class,
+                                          Access, TInfo);
+
+  SourceLocation BaseLoc = TInfo->getTypeLoc().getBeginLoc();
 
   // Base specifiers must be record types.
   if (!BaseType->isRecordType()) {
@@ -482,8 +480,10 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
   //   defined class.
   if (RequireCompleteType(BaseLoc, BaseType,
                           PDiag(diag::err_incomplete_base_class)
-                            << SpecifierRange))
+                            << SpecifierRange)) {
+    Class->setInvalidDecl();
     return 0;
+  }
 
   // If the base class is polymorphic or isn't empty, the new one is/isn't, too.
   RecordDecl *BaseDecl = BaseType->getAs<RecordType>()->getDecl();
@@ -502,11 +502,14 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
   }
 
   SetClassDeclAttributesFromBase(Class, CXXBaseDecl, Virtual);
+
+  if (BaseDecl->isInvalidDecl())
+    Class->setInvalidDecl();
   
   // Create the base specifier.
   return new (Context) CXXBaseSpecifier(SpecifierRange, Virtual,
-                              Class->getTagKind() == TTK_Class,
-                              Access, BaseType);
+                                        Class->getTagKind() == TTK_Class,
+                                        Access, TInfo);
 }
 
 void Sema::SetClassDeclAttributesFromBase(CXXRecordDecl *Class,
@@ -581,22 +584,22 @@ void Sema::SetClassDeclAttributesFromBase(CXXRecordDecl *Class,
 /// example:
 ///    class foo : public bar, virtual private baz {
 /// 'public bar' and 'virtual private baz' are each base-specifiers.
-Sema::BaseResult
-Sema::ActOnBaseSpecifier(DeclPtrTy classdecl, SourceRange SpecifierRange,
+BaseResult
+Sema::ActOnBaseSpecifier(Decl *classdecl, SourceRange SpecifierRange,
                          bool Virtual, AccessSpecifier Access,
-                         TypeTy *basetype, SourceLocation BaseLoc) {
+                         ParsedType basetype, SourceLocation BaseLoc) {
   if (!classdecl)
     return true;
 
   AdjustDeclIfTemplate(classdecl);
-  CXXRecordDecl *Class = dyn_cast<CXXRecordDecl>(classdecl.getAs<Decl>());
+  CXXRecordDecl *Class = dyn_cast<CXXRecordDecl>(classdecl);
   if (!Class)
     return true;
 
-  QualType BaseType = GetTypeFromParser(basetype);
+  TypeSourceInfo *TInfo = 0;
+  GetTypeFromParser(basetype, &TInfo);
   if (CXXBaseSpecifier *BaseSpec = CheckBaseSpecifier(Class, SpecifierRange,
-                                                      Virtual, Access,
-                                                      BaseType, BaseLoc))
+                                                      Virtual, Access, TInfo))
     return BaseSpec;
 
   return true;
@@ -664,13 +667,13 @@ bool Sema::AttachBaseSpecifiers(CXXRecordDecl *Class, CXXBaseSpecifier **Bases,
 /// ActOnBaseSpecifiers - Attach the given base specifiers to the
 /// class, after checking whether there are any duplicate base
 /// classes.
-void Sema::ActOnBaseSpecifiers(DeclPtrTy ClassDecl, BaseTy **Bases,
+void Sema::ActOnBaseSpecifiers(Decl *ClassDecl, BaseTy **Bases,
                                unsigned NumBases) {
   if (!ClassDecl || !Bases || !NumBases)
     return;
 
   AdjustDeclIfTemplate(ClassDecl);
-  AttachBaseSpecifiers(cast<CXXRecordDecl>(ClassDecl.getAs<Decl>()),
+  AttachBaseSpecifiers(cast<CXXRecordDecl>(ClassDecl),
                        (CXXBaseSpecifier**)(Bases), NumBases);
 }
 
@@ -719,7 +722,7 @@ bool Sema::IsDerivedFrom(QualType Derived, QualType Base, CXXBasePaths &Paths) {
 }
 
 void Sema::BuildBasePathArray(const CXXBasePaths &Paths, 
-                              CXXBaseSpecifierArray &BasePathArray) {
+                              CXXCastPath &BasePathArray) {
   assert(BasePathArray.empty() && "Base path array must be empty!");
   assert(Paths.isRecordingPaths() && "Must record paths!");
   
@@ -738,14 +741,14 @@ void Sema::BuildBasePathArray(const CXXBasePaths &Paths,
 
   // Now add all bases.
   for (unsigned I = Start, E = Path.size(); I != E; ++I)
-    BasePathArray.push_back(Path[I].Base);
+    BasePathArray.push_back(const_cast<CXXBaseSpecifier*>(Path[I].Base));
 }
 
 /// \brief Determine whether the given base path includes a virtual
 /// base class.
-bool Sema::BasePathInvolvesVirtualBase(const CXXBaseSpecifierArray &BasePath) {
-  for (CXXBaseSpecifierArray::iterator B = BasePath.begin(), 
-                                    BEnd = BasePath.end();
+bool Sema::BasePathInvolvesVirtualBase(const CXXCastPath &BasePath) {
+  for (CXXCastPath::const_iterator B = BasePath.begin(), 
+                                BEnd = BasePath.end();
        B != BEnd; ++B)
     if ((*B)->isVirtual())
       return true;
@@ -767,7 +770,7 @@ Sema::CheckDerivedToBaseConversion(QualType Derived, QualType Base,
                                    unsigned AmbigiousBaseConvID,
                                    SourceLocation Loc, SourceRange Range,
                                    DeclarationName Name,
-                                   CXXBaseSpecifierArray *BasePath) {
+                                   CXXCastPath *BasePath) {
   // First, determine whether the path from Derived to Base is
   // ambiguous. This is slightly more expensive than checking whether
   // the Derived to Base conversion exists, because here we need to
@@ -825,7 +828,7 @@ Sema::CheckDerivedToBaseConversion(QualType Derived, QualType Base,
 bool
 Sema::CheckDerivedToBaseConversion(QualType Derived, QualType Base,
                                    SourceLocation Loc, SourceRange Range,
-                                   CXXBaseSpecifierArray *BasePath,
+                                   CXXCastPath *BasePath,
                                    bool IgnoreAccess) {
   return CheckDerivedToBaseConversion(Derived, Base,
                                       IgnoreAccess ? 0
@@ -872,30 +875,31 @@ std::string Sema::getAmbiguousPathsDisplayString(CXXBasePaths &Paths) {
 //===----------------------------------------------------------------------===//
 
 /// ActOnAccessSpecifier - Parsed an access specifier followed by a colon.
-Sema::DeclPtrTy
-Sema::ActOnAccessSpecifier(AccessSpecifier Access,
-                           SourceLocation ASLoc, SourceLocation ColonLoc) {
+Decl *Sema::ActOnAccessSpecifier(AccessSpecifier Access,
+                                 SourceLocation ASLoc,
+                                 SourceLocation ColonLoc) {
   assert(Access != AS_none && "Invalid kind for syntactic access specifier!");
-  AccessSpecDecl* ASDecl = AccessSpecDecl::Create(Context, Access, CurContext,
+  AccessSpecDecl *ASDecl = AccessSpecDecl::Create(Context, Access, CurContext,
                                                   ASLoc, ColonLoc);
   CurContext->addHiddenDecl(ASDecl);
-  return DeclPtrTy::make(ASDecl);
+  return ASDecl;
 }
 
 /// ActOnCXXMemberDeclarator - This is invoked when a C++ class member
 /// declarator is parsed. 'AS' is the access specifier, 'BW' specifies the
 /// bitfield width if there is one and 'InitExpr' specifies the initializer if
 /// any.
-Sema::DeclPtrTy
+Decl *
 Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                                MultiTemplateParamsArg TemplateParameterLists,
                                ExprTy *BW, ExprTy *InitExpr, bool IsDefinition,
                                bool Deleted) {
   const DeclSpec &DS = D.getDeclSpec();
-  DeclarationName Name = GetNameForDeclarator(D);
+  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
+  SourceLocation Loc = NameInfo.getLoc();
   Expr *BitWidth = static_cast<Expr*>(BW);
   Expr *Init = static_cast<Expr*>(InitExpr);
-  SourceLocation Loc = D.getIdentifierLoc();
 
   assert(isa<CXXRecordDecl>(CurContext));
   assert(!DS.isFriendSpecified());
@@ -905,7 +909,7 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
     isFunc = true;
   else if (D.getNumTypeObjects() == 0 &&
            D.getDeclSpec().getTypeSpecType() == DeclSpec::TST_typename) {
-    QualType TDType = GetTypeFromParser(DS.getTypeRep());
+    QualType TDType = GetTypeFromParser(DS.getRepAsType());
     isFunc = TDType->isFunctionType();
   }
 
@@ -952,11 +956,9 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                          AS);
     assert(Member && "HandleField never returns null");
   } else {
-    Member = HandleDeclarator(S, D, move(TemplateParameterLists), IsDefinition)
-               .getAs<Decl>();
+    Member = HandleDeclarator(S, D, move(TemplateParameterLists), IsDefinition);
     if (!Member) {
-      if (BitWidth) DeleteExpr(BitWidth);
-      return DeclPtrTy();
+      return 0;
     }
 
     // Non-instance-fields can't have a bitfield.
@@ -980,7 +982,6 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
           << BitWidth->getSourceRange();
       }
 
-      DeleteExpr(BitWidth);
       BitWidth = 0;
       Member->setInvalidDecl();
     }
@@ -996,15 +997,15 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
   assert((Name || isInstField) && "No identifier for non-field ?");
 
   if (Init)
-    AddInitializerToDecl(DeclPtrTy::make(Member), ExprArg(*this, Init), false);
+    AddInitializerToDecl(Member, Init, false);
   if (Deleted) // FIXME: Source location is not very good.
-    SetDeclDeleted(DeclPtrTy::make(Member), D.getSourceRange().getBegin());
+    SetDeclDeleted(Member, D.getSourceRange().getBegin());
 
   if (isInstField) {
     FieldCollector->Add(cast<FieldDecl>(Member));
-    return DeclPtrTy();
+    return 0;
   }
-  return DeclPtrTy::make(Member);
+  return Member;
 }
 
 /// \brief Find the direct and/or virtual base specifiers that
@@ -1053,12 +1054,12 @@ static bool FindBaseInitializer(Sema &SemaRef,
 }
 
 /// ActOnMemInitializer - Handle a C++ member initializer.
-Sema::MemInitResult
-Sema::ActOnMemInitializer(DeclPtrTy ConstructorD,
+MemInitResult
+Sema::ActOnMemInitializer(Decl *ConstructorD,
                           Scope *S,
                           CXXScopeSpec &SS,
                           IdentifierInfo *MemberOrBase,
-                          TypeTy *TemplateTypeTy,
+                          ParsedType TemplateTypeTy,
                           SourceLocation IdLoc,
                           SourceLocation LParenLoc,
                           ExprTy **Args, unsigned NumArgs,
@@ -1070,7 +1071,7 @@ Sema::ActOnMemInitializer(DeclPtrTy ConstructorD,
   AdjustDeclIfTemplate(ConstructorD);
 
   CXXConstructorDecl *Constructor
-    = dyn_cast<CXXConstructorDecl>(ConstructorD.getAs<Decl>());
+    = dyn_cast<CXXConstructorDecl>(ConstructorD);
   if (!Constructor) {
     // The user wrote a constructor initializer on a function that is
     // not a C++ constructor. Ignore the error for now, because we may
@@ -1148,7 +1149,7 @@ Sema::ActOnMemInitializer(DeclPtrTy ConstructorD,
           CorrectTypo(R, S, &SS, ClassDecl, 0, CTC_NoKeywords) && 
           R.isSingleResult()) {
         if (FieldDecl *Member = R.getAsSingle<FieldDecl>()) {
-          if (Member->getDeclContext()->getLookupContext()->Equals(ClassDecl)) {
+          if (Member->getDeclContext()->getRedeclContext()->Equals(ClassDecl)) {
             // We have found a non-static data member with a similar
             // name to what was typed; complain and initialize that
             // member.
@@ -1259,7 +1260,7 @@ static bool InitExprContainsUninitializedFields(const Stmt *S,
   return false;
 }
 
-Sema::MemInitResult
+MemInitResult
 Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
                              unsigned NumArgs, SourceLocation IdLoc,
                              SourceLocation LParenLoc,
@@ -1285,15 +1286,12 @@ Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
   for (unsigned i = 0; i < NumArgs; i++)
     HasDependentArg |= Args[i]->isTypeDependent();
 
-  QualType FieldType = Member->getType();
-  if (const ArrayType *Array = Context.getAsArrayType(FieldType))
-    FieldType = Array->getElementType();
-  if (FieldType->isDependentType() || HasDependentArg) {
+  if (Member->getType()->isDependentType() || HasDependentArg) {
     // Can't check initialization for a member of dependent type or when
     // any of the arguments are type-dependent expressions.
-    OwningExprResult Init
-      = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                          RParenLoc));
+    Expr *Init
+      = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
+                                    RParenLoc);
 
     // Erase any temporaries within this evaluation context; we're not
     // going to track them in the AST, since we'll be rebuilding the
@@ -1304,7 +1302,7 @@ Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
     
     return new (Context) CXXBaseOrMemberInitializer(Context, Member, IdLoc,
                                                     LParenLoc, 
-                                                    Init.takeAs<Expr>(),
+                                                    Init,
                                                     RParenLoc);
     
   }
@@ -1320,16 +1318,16 @@ Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
   
   InitializationSequence InitSeq(*this, MemberEntity, Kind, Args, NumArgs);
   
-  OwningExprResult MemberInit =
+  ExprResult MemberInit =
     InitSeq.Perform(*this, MemberEntity, Kind, 
-                    MultiExprArg(*this, (void**)Args, NumArgs), 0);
+                    MultiExprArg(*this, Args, NumArgs), 0);
   if (MemberInit.isInvalid())
     return true;
   
   // C++0x [class.base.init]p7:
   //   The initialization of each base and member constitutes a 
   //   full-expression.
-  MemberInit = MaybeCreateCXXExprWithTemporaries(move(MemberInit));
+  MemberInit = MaybeCreateCXXExprWithTemporaries(MemberInit.get());
   if (MemberInit.isInvalid())
     return true;
   
@@ -1345,22 +1343,21 @@ Sema::BuildMemberInitializer(FieldDecl *Member, Expr **Args,
     for (unsigned I = 0; I != NumArgs; ++I)
       Args[I]->Retain();
 
-    OwningExprResult Init
-      = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
-                                          RParenLoc));
+    Expr *Init = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
+                                             RParenLoc);
     return new (Context) CXXBaseOrMemberInitializer(Context, Member, IdLoc,
                                                     LParenLoc, 
-                                                    Init.takeAs<Expr>(),
+                                                    Init,
                                                     RParenLoc);
   }
 
   return new (Context) CXXBaseOrMemberInitializer(Context, Member, IdLoc,
                                                   LParenLoc, 
-                                                  MemberInit.takeAs<Expr>(),
+                                                  MemberInit.get(),
                                                   RParenLoc);
 }
 
-Sema::MemInitResult
+MemInitResult
 Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
                            Expr **Args, unsigned NumArgs, 
                            SourceLocation LParenLoc, SourceLocation RParenLoc, 
@@ -1413,7 +1410,7 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   if (Dependent) {
     // Can't check initialization for a base of dependent type or when
     // any of the arguments are type-dependent expressions.
-    OwningExprResult BaseInit
+    ExprResult BaseInit
       = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
                                           RParenLoc));
 
@@ -1452,16 +1449,16 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   
   InitializationSequence InitSeq(*this, BaseEntity, Kind, Args, NumArgs);
   
-  OwningExprResult BaseInit =
+  ExprResult BaseInit =
     InitSeq.Perform(*this, BaseEntity, Kind, 
-                    MultiExprArg(*this, (void**)Args, NumArgs), 0);
+                    MultiExprArg(*this, Args, NumArgs), 0);
   if (BaseInit.isInvalid())
     return true;
   
   // C++0x [class.base.init]p7:
   //   The initialization of each base and member constitutes a 
   //   full-expression.
-  BaseInit = MaybeCreateCXXExprWithTemporaries(move(BaseInit));
+  BaseInit = MaybeCreateCXXExprWithTemporaries(BaseInit.get());
   if (BaseInit.isInvalid())
     return true;
 
@@ -1477,7 +1474,7 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
     for (unsigned I = 0; I != NumArgs; ++I)
       Args[I]->Retain();
 
-    OwningExprResult Init
+    ExprResult Init
       = Owned(new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
                                           RParenLoc));
     return new (Context) CXXBaseOrMemberInitializer(Context, BaseTInfo,
@@ -1512,7 +1509,7 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     = InitializedEntity::InitializeBase(SemaRef.Context, BaseSpec,
                                         IsInheritedVirtualBase);
 
-  Sema::OwningExprResult BaseInit(SemaRef);
+  ExprResult BaseInit;
   
   switch (ImplicitInitKind) {
   case IIK_Default: {
@@ -1520,7 +1517,7 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
       = InitializationKind::CreateDefault(Constructor->getLocation());
     InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, 0, 0);
     BaseInit = InitSeq.Perform(SemaRef, InitEntity, InitKind,
-                               Sema::MultiExprArg(SemaRef, 0, 0));
+                               MultiExprArg(SemaRef, 0, 0));
     break;
   }
 
@@ -1536,10 +1533,12 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     QualType ArgTy = 
       SemaRef.Context.getQualifiedType(BaseSpec->getType().getUnqualifiedType(), 
                                        ParamType.getQualifiers());
-    SemaRef.ImpCastExprToType(CopyCtorArg, ArgTy, 
-                              CastExpr::CK_UncheckedDerivedToBase,
-                              /*isLvalue=*/true, 
-                              CXXBaseSpecifierArray(BaseSpec));
+
+    CXXCastPath BasePath;
+    BasePath.push_back(BaseSpec);
+    SemaRef.ImpCastExprToType(CopyCtorArg, ArgTy,
+                              CK_UncheckedDerivedToBase,
+                              VK_LValue, &BasePath);
 
     InitializationKind InitKind
       = InitializationKind::CreateDirect(Constructor->getLocation(),
@@ -1547,16 +1546,18 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, 
                                    &CopyCtorArg, 1);
     BaseInit = InitSeq.Perform(SemaRef, InitEntity, InitKind,
-                               Sema::MultiExprArg(SemaRef, 
-                                                  (void**)&CopyCtorArg, 1));
+                               MultiExprArg(&CopyCtorArg, 1));
     break;
   }
 
   case IIK_Move:
     assert(false && "Unhandled initializer kind!");
   }
+
+  if (BaseInit.isInvalid())
+    return true;
       
-  BaseInit = SemaRef.MaybeCreateCXXExprWithTemporaries(move(BaseInit));
+  BaseInit = SemaRef.MaybeCreateCXXExprWithTemporaries(BaseInit.get());
   if (BaseInit.isInvalid())
     return true;
         
@@ -1596,8 +1597,8 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
                               Sema::LookupMemberName);
     MemberLookup.addDecl(Field, AS_public);
     MemberLookup.resolveKind();
-    Sema::OwningExprResult CopyCtorArg 
-      = SemaRef.BuildMemberReferenceExpr(SemaRef.Owned(MemberExprBase),
+    ExprResult CopyCtorArg 
+      = SemaRef.BuildMemberReferenceExpr(MemberExprBase,
                                          ParamType, Loc,
                                          /*IsArrow=*/false,
                                          SS,
@@ -1628,19 +1629,19 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
         = VarDecl::Create(SemaRef.Context, SemaRef.CurContext, Loc,
                           IterationVarName, SizeType,
                         SemaRef.Context.getTrivialTypeSourceInfo(SizeType, Loc),
-                          VarDecl::None, VarDecl::None);
+                          SC_None, SC_None);
       IndexVariables.push_back(IterationVar);
       
       // Create a reference to the iteration variable.
-      Sema::OwningExprResult IterationVarRef
+      ExprResult IterationVarRef
         = SemaRef.BuildDeclRefExpr(IterationVar, SizeType, Loc);
       assert(!IterationVarRef.isInvalid() &&
              "Reference to invented variable cannot fail!");
       
       // Subscript the array with this iteration variable.
-      CopyCtorArg = SemaRef.CreateBuiltinArraySubscriptExpr(move(CopyCtorArg),
+      CopyCtorArg = SemaRef.CreateBuiltinArraySubscriptExpr(CopyCtorArg.take(),
                                                             Loc,
-                                                          move(IterationVarRef),
+                                                        IterationVarRef.take(),
                                                             Loc);
       if (CopyCtorArg.isInvalid())
         return true;
@@ -1667,10 +1668,10 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     InitializationSequence InitSeq(SemaRef, Entities.back(), InitKind,
                                    &CopyCtorArgE, 1);
     
-    Sema::OwningExprResult MemberInit
+    ExprResult MemberInit
       = InitSeq.Perform(SemaRef, Entities.back(), InitKind, 
-                        Sema::MultiExprArg(SemaRef, (void**)&CopyCtorArgE, 1));
-    MemberInit = SemaRef.MaybeCreateCXXExprWithTemporaries(move(MemberInit));
+                        MultiExprArg(&CopyCtorArgE, 1));
+    MemberInit = SemaRef.MaybeCreateCXXExprWithTemporaries(MemberInit.get());
     if (MemberInit.isInvalid())
       return true;
 
@@ -1693,17 +1694,19 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
       InitializationKind::CreateDefault(Loc);
     
     InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, 0, 0);
-    Sema::OwningExprResult MemberInit = 
-      InitSeq.Perform(SemaRef, InitEntity, InitKind, 
-                      Sema::MultiExprArg(SemaRef, 0, 0));
-    MemberInit = SemaRef.MaybeCreateCXXExprWithTemporaries(move(MemberInit));
+    ExprResult MemberInit = 
+      InitSeq.Perform(SemaRef, InitEntity, InitKind, MultiExprArg());
+    if (MemberInit.isInvalid())
+      return true;
+
+    MemberInit = SemaRef.MaybeCreateCXXExprWithTemporaries(MemberInit.get());
     if (MemberInit.isInvalid())
       return true;
     
     CXXMemberInit =
       new (SemaRef.Context) CXXBaseOrMemberInitializer(SemaRef.Context,
                                                        Field, Loc, Loc,
-                                                      MemberInit.takeAs<Expr>(),
+                                                       MemberInit.get(),
                                                        Loc);
     return false;
   }
@@ -1797,6 +1800,9 @@ static bool CollectFieldInitializer(BaseAndFieldInfo &Info,
           // Once we've initialized a field of an anonymous union, the union
           // field in the class is also initialized, so exit immediately.
           return false;
+        } else if ((*FA)->isAnonymousStructOrUnion()) {
+          if (CollectFieldInitializer(Info, Top, *FA))
+            return true;
         }
       }
 
@@ -2147,7 +2153,7 @@ bool CheckRedundantUnionInit(Sema &S,
 }
 
 /// ActOnMemInitializers - Handle the member initializers for a constructor.
-void Sema::ActOnMemInitializers(DeclPtrTy ConstructorDecl,
+void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
                                 SourceLocation ColonLoc,
                                 MemInitTy **meminits, unsigned NumMemInits,
                                 bool AnyErrors) {
@@ -2157,7 +2163,7 @@ void Sema::ActOnMemInitializers(DeclPtrTy ConstructorDecl,
   AdjustDeclIfTemplate(ConstructorDecl);
 
   CXXConstructorDecl *Constructor
-    = dyn_cast<CXXConstructorDecl>(ConstructorDecl.getAs<Decl>());
+    = dyn_cast<CXXConstructorDecl>(ConstructorDecl);
 
   if (!Constructor) {
     Diag(ColonLoc, diag::err_only_constructors_take_base_inits);
@@ -2292,35 +2298,30 @@ Sema::MarkBaseAndMemberDestructorsReferenced(SourceLocation Location,
   }
 }
 
-void Sema::ActOnDefaultCtorInitializers(DeclPtrTy CDtorDecl) {
+void Sema::ActOnDefaultCtorInitializers(Decl *CDtorDecl) {
   if (!CDtorDecl)
     return;
 
   if (CXXConstructorDecl *Constructor
-      = dyn_cast<CXXConstructorDecl>(CDtorDecl.getAs<Decl>()))
+      = dyn_cast<CXXConstructorDecl>(CDtorDecl))
     SetBaseOrMemberInitializers(Constructor, 0, 0, /*AnyErrors=*/false);
 }
 
 bool Sema::RequireNonAbstractType(SourceLocation Loc, QualType T,
-                                  unsigned DiagID, AbstractDiagSelID SelID,
-                                  const CXXRecordDecl *CurrentRD) {
+                                  unsigned DiagID, AbstractDiagSelID SelID) {
   if (SelID == -1)
-    return RequireNonAbstractType(Loc, T,
-                                  PDiag(DiagID), CurrentRD);
+    return RequireNonAbstractType(Loc, T, PDiag(DiagID));
   else
-    return RequireNonAbstractType(Loc, T,
-                                  PDiag(DiagID) << SelID, CurrentRD);
+    return RequireNonAbstractType(Loc, T, PDiag(DiagID) << SelID);
 }
 
 bool Sema::RequireNonAbstractType(SourceLocation Loc, QualType T,
-                                  const PartialDiagnostic &PD,
-                                  const CXXRecordDecl *CurrentRD) {
+                                  const PartialDiagnostic &PD) {
   if (!getLangOptions().CPlusPlus)
     return false;
 
   if (const ArrayType *AT = Context.getAsArrayType(T))
-    return RequireNonAbstractType(Loc, AT->getElementType(), PD,
-                                  CurrentRD);
+    return RequireNonAbstractType(Loc, AT->getElementType(), PD);
 
   if (const PointerType *PT = T->getAs<PointerType>()) {
     // Find the innermost pointer type.
@@ -2328,7 +2329,7 @@ bool Sema::RequireNonAbstractType(SourceLocation Loc, QualType T,
       PT = T;
 
     if (const ArrayType *AT = Context.getAsArrayType(PT->getPointeeType()))
-      return RequireNonAbstractType(Loc, AT->getElementType(), PD, CurrentRD);
+      return RequireNonAbstractType(Loc, AT->getElementType(), PD);
   }
 
   const RecordType *RT = T->getAs<RecordType>();
@@ -2337,22 +2338,27 @@ bool Sema::RequireNonAbstractType(SourceLocation Loc, QualType T,
 
   const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
 
-  if (CurrentRD && CurrentRD != RD)
-    return false;
-
-  // FIXME: is this reasonable?  It matches current behavior, but....
-  if (!RD->getDefinition())
+  // We can't answer whether something is abstract until it has a
+  // definition.  If it's currently being defined, we'll walk back
+  // over all the declarations when we have a full definition.
+  const CXXRecordDecl *Def = RD->getDefinition();
+  if (!Def || Def->isBeingDefined())
     return false;
 
   if (!RD->isAbstract())
     return false;
 
   Diag(Loc, PD) << RD->getDeclName();
+  DiagnoseAbstractType(RD);
 
-  // Check if we've already emitted the list of pure virtual functions for this
-  // class.
+  return true;
+}
+
+void Sema::DiagnoseAbstractType(const CXXRecordDecl *RD) {
+  // Check if we've already emitted the list of pure virtual functions
+  // for this class.
   if (PureVirtualClassDiagSet && PureVirtualClassDiagSet->count(RD))
-    return true;
+    return;
 
   CXXFinalOverriderMap FinalOverriders;
   RD->getFinalOverriders(FinalOverriders);
@@ -2392,69 +2398,168 @@ bool Sema::RequireNonAbstractType(SourceLocation Loc, QualType T,
   if (!PureVirtualClassDiagSet)
     PureVirtualClassDiagSet.reset(new RecordDeclSetTy);
   PureVirtualClassDiagSet->insert(RD);
-
-  return true;
 }
 
 namespace {
-  class AbstractClassUsageDiagnoser
-    : public DeclVisitor<AbstractClassUsageDiagnoser, bool> {
-    Sema &SemaRef;
-    CXXRecordDecl *AbstractClass;
+struct AbstractUsageInfo {
+  Sema &S;
+  CXXRecordDecl *Record;
+  CanQualType AbstractType;
+  bool Invalid;
+
+  AbstractUsageInfo(Sema &S, CXXRecordDecl *Record)
+    : S(S), Record(Record),
+      AbstractType(S.Context.getCanonicalType(
+                   S.Context.getTypeDeclType(Record))),
+      Invalid(false) {}
+
+  void DiagnoseAbstractType() {
+    if (Invalid) return;
+    S.DiagnoseAbstractType(Record);
+    Invalid = true;
+  }
 
-    bool VisitDeclContext(const DeclContext *DC) {
-      bool Invalid = false;
+  void CheckType(const NamedDecl *D, TypeLoc TL, Sema::AbstractDiagSelID Sel);
+};
 
-      for (CXXRecordDecl::decl_iterator I = DC->decls_begin(),
-           E = DC->decls_end(); I != E; ++I)
-        Invalid |= Visit(*I);
+struct CheckAbstractUsage {
+  AbstractUsageInfo &Info;
+  const NamedDecl *Ctx;
 
-      return Invalid;
+  CheckAbstractUsage(AbstractUsageInfo &Info, const NamedDecl *Ctx)
+    : Info(Info), Ctx(Ctx) {}
+
+  void Visit(TypeLoc TL, Sema::AbstractDiagSelID Sel) {
+    switch (TL.getTypeLocClass()) {
+#define ABSTRACT_TYPELOC(CLASS, PARENT)
+#define TYPELOC(CLASS, PARENT) \
+    case TypeLoc::CLASS: Check(cast<CLASS##TypeLoc>(TL), Sel); break;
+#include "clang/AST/TypeLocNodes.def"
     }
+  }
 
-  public:
-    AbstractClassUsageDiagnoser(Sema& SemaRef, CXXRecordDecl *ac)
-      : SemaRef(SemaRef), AbstractClass(ac) {
-        Visit(SemaRef.Context.getTranslationUnitDecl());
+  void Check(FunctionProtoTypeLoc TL, Sema::AbstractDiagSelID Sel) {
+    Visit(TL.getResultLoc(), Sema::AbstractReturnType);
+    for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I) {
+      TypeSourceInfo *TSI = TL.getArg(I)->getTypeSourceInfo();
+      if (TSI) Visit(TSI->getTypeLoc(), Sema::AbstractParamType);
     }
+  }
 
-    bool VisitFunctionDecl(const FunctionDecl *FD) {
-      if (FD->isThisDeclarationADefinition()) {
-        // No need to do the check if we're in a definition, because it requires
-        // that the return/param types are complete.
-        // because that requires
-        return VisitDeclContext(FD);
-      }
+  void Check(ArrayTypeLoc TL, Sema::AbstractDiagSelID Sel) {
+    Visit(TL.getElementLoc(), Sema::AbstractArrayType);
+  }
 
-      // Check the return type.
-      QualType RTy = FD->getType()->getAs<FunctionType>()->getResultType();
-      bool Invalid =
-        SemaRef.RequireNonAbstractType(FD->getLocation(), RTy,
-                                       diag::err_abstract_type_in_decl,
-                                       Sema::AbstractReturnType,
-                                       AbstractClass);
-
-      for (FunctionDecl::param_const_iterator I = FD->param_begin(),
-           E = FD->param_end(); I != E; ++I) {
-        const ParmVarDecl *VD = *I;
-        Invalid |=
-          SemaRef.RequireNonAbstractType(VD->getLocation(),
-                                         VD->getOriginalType(),
-                                         diag::err_abstract_type_in_decl,
-                                         Sema::AbstractParamType,
-                                         AbstractClass);
-      }
+  void Check(TemplateSpecializationTypeLoc TL, Sema::AbstractDiagSelID Sel) {
+    // Visit the type parameters from a permissive context.
+    for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I) {
+      TemplateArgumentLoc TAL = TL.getArgLoc(I);
+      if (TAL.getArgument().getKind() == TemplateArgument::Type)
+        if (TypeSourceInfo *TSI = TAL.getTypeSourceInfo())
+          Visit(TSI->getTypeLoc(), Sema::AbstractNone);
+      // TODO: other template argument types?
+    }
+  }
 
-      return Invalid;
+  // Visit pointee types from a permissive context.
+#define CheckPolymorphic(Type) \
+  void Check(Type TL, Sema::AbstractDiagSelID Sel) { \
+    Visit(TL.getNextTypeLoc(), Sema::AbstractNone); \
+  }
+  CheckPolymorphic(PointerTypeLoc)
+  CheckPolymorphic(ReferenceTypeLoc)
+  CheckPolymorphic(MemberPointerTypeLoc)
+  CheckPolymorphic(BlockPointerTypeLoc)
+
+  /// Handle all the types we haven't given a more specific
+  /// implementation for above.
+  void Check(TypeLoc TL, Sema::AbstractDiagSelID Sel) {
+    // Every other kind of type that we haven't called out already
+    // that has an inner type is either (1) sugar or (2) contains that
+    // inner type in some way as a subobject.
+    if (TypeLoc Next = TL.getNextTypeLoc())
+      return Visit(Next, Sel);
+
+    // If there's no inner type and we're in a permissive context,
+    // don't diagnose.
+    if (Sel == Sema::AbstractNone) return;
+
+    // Check whether the type matches the abstract type.
+    QualType T = TL.getType();
+    if (T->isArrayType()) {
+      Sel = Sema::AbstractArrayType;
+      T = Info.S.Context.getBaseElementType(T);
     }
+    CanQualType CT = T->getCanonicalTypeUnqualified().getUnqualifiedType();
+    if (CT != Info.AbstractType) return;
 
-    bool VisitDecl(const Decl* D) {
-      if (const DeclContext *DC = dyn_cast<DeclContext>(D))
-        return VisitDeclContext(DC);
+    // It matched; do some magic.
+    if (Sel == Sema::AbstractArrayType) {
+      Info.S.Diag(Ctx->getLocation(), diag::err_array_of_abstract_type)
+        << T << TL.getSourceRange();
+    } else {
+      Info.S.Diag(Ctx->getLocation(), diag::err_abstract_type_in_decl)
+        << Sel << T << TL.getSourceRange();
+    }
+    Info.DiagnoseAbstractType();
+  }
+};
 
-      return false;
+void AbstractUsageInfo::CheckType(const NamedDecl *D, TypeLoc TL,
+                                  Sema::AbstractDiagSelID Sel) {
+  CheckAbstractUsage(*this, D).Visit(TL, Sel);
+}
+
+}
+
+/// Check for invalid uses of an abstract type in a method declaration.
+static void CheckAbstractClassUsage(AbstractUsageInfo &Info,
+                                    CXXMethodDecl *MD) {
+  // No need to do the check on definitions, which require that
+  // the return/param types be complete.
+  if (MD->isThisDeclarationADefinition())
+    return;
+
+  // For safety's sake, just ignore it if we don't have type source
+  // information.  This should never happen for non-implicit methods,
+  // but...
+  if (TypeSourceInfo *TSI = MD->getTypeSourceInfo())
+    Info.CheckType(MD, TSI->getTypeLoc(), Sema::AbstractNone);
+}
+
+/// Check for invalid uses of an abstract type within a class definition.
+static void CheckAbstractClassUsage(AbstractUsageInfo &Info,
+                                    CXXRecordDecl *RD) {
+  for (CXXRecordDecl::decl_iterator
+         I = RD->decls_begin(), E = RD->decls_end(); I != E; ++I) {
+    Decl *D = *I;
+    if (D->isImplicit()) continue;
+
+    // Methods and method templates.
+    if (isa<CXXMethodDecl>(D)) {
+      CheckAbstractClassUsage(Info, cast<CXXMethodDecl>(D));
+    } else if (isa<FunctionTemplateDecl>(D)) {
+      FunctionDecl *FD = cast<FunctionTemplateDecl>(D)->getTemplatedDecl();
+      CheckAbstractClassUsage(Info, cast<CXXMethodDecl>(FD));
+
+    // Fields and static variables.
+    } else if (isa<FieldDecl>(D)) {
+      FieldDecl *FD = cast<FieldDecl>(D);
+      if (TypeSourceInfo *TSI = FD->getTypeSourceInfo())
+        Info.CheckType(FD, TSI->getTypeLoc(), Sema::AbstractFieldType);
+    } else if (isa<VarDecl>(D)) {
+      VarDecl *VD = cast<VarDecl>(D);
+      if (TypeSourceInfo *TSI = VD->getTypeSourceInfo())
+        Info.CheckType(VD, TSI->getTypeLoc(), Sema::AbstractVariableType);
+
+    // Nested classes and class templates.
+    } else if (isa<CXXRecordDecl>(D)) {
+      CheckAbstractClassUsage(Info, cast<CXXRecordDecl>(D));
+    } else if (isa<ClassTemplateDecl>(D)) {
+      CheckAbstractClassUsage(Info,
+                             cast<ClassTemplateDecl>(D)->getTemplatedDecl());
     }
-  };
+  }
 }
 
 /// \brief Perform semantic checks on a class definition that has been
@@ -2539,8 +2644,10 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
     }
   }
 
-  if (Record->isAbstract() && !Record->isInvalidDecl())
-    (void)AbstractClassUsageDiagnoser(*this, Record);
+  if (Record->isAbstract() && !Record->isInvalidDecl()) {
+    AbstractUsageInfo Info(*this, Record);
+    CheckAbstractClassUsage(Info, Record);
+  }
   
   // If this is not an aggregate type and has no user-declared constructor,
   // complain about any non-static data members of reference or const scalar
@@ -2571,7 +2678,7 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
 }
 
 void Sema::ActOnFinishCXXMemberSpecification(Scope* S, SourceLocation RLoc,
-                                             DeclPtrTy TagDecl,
+                                             Decl *TagDecl,
                                              SourceLocation LBrac,
                                              SourceLocation RBrac,
                                              AttributeList *AttrList) {
@@ -2581,11 +2688,12 @@ void Sema::ActOnFinishCXXMemberSpecification(Scope* S, SourceLocation RLoc,
   AdjustDeclIfTemplate(TagDecl);
 
   ActOnFields(S, RLoc, TagDecl,
-              (DeclPtrTy*)FieldCollector->getCurFields(),
+              // strict aliasing violation!
+              reinterpret_cast<Decl**>(FieldCollector->getCurFields()),
               FieldCollector->getCurNumFields(), LBrac, RBrac, AttrList);
 
   CheckCompletedCXXClass(
-                        dyn_cast_or_null<CXXRecordDecl>(TagDecl.getAs<Decl>()));
+                        dyn_cast_or_null<CXXRecordDecl>(TagDecl));
 }
 
 namespace {
@@ -2682,8 +2790,7 @@ void Sema::AddImplicitlyDeclaredMembersToClass(CXXRecordDecl *ClassDecl) {
   }
 }
 
-void Sema::ActOnReenterTemplateScope(Scope *S, DeclPtrTy TemplateD) {
-  Decl *D = TemplateD.getAs<Decl>();
+void Sema::ActOnReenterTemplateScope(Scope *S, Decl *D) {
   if (!D)
     return;
   
@@ -2701,20 +2808,20 @@ void Sema::ActOnReenterTemplateScope(Scope *S, DeclPtrTy TemplateD) {
        Param != ParamEnd; ++Param) {
     NamedDecl *Named = cast<NamedDecl>(*Param);
     if (Named->getDeclName()) {
-      S->AddDecl(DeclPtrTy::make(Named));
+      S->AddDecl(Named);
       IdResolver.AddDecl(Named);
     }
   }
 }
 
-void Sema::ActOnStartDelayedMemberDeclarations(Scope *S, DeclPtrTy RecordD) {
+void Sema::ActOnStartDelayedMemberDeclarations(Scope *S, Decl *RecordD) {
   if (!RecordD) return;
   AdjustDeclIfTemplate(RecordD);
-  CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordD.getAs<Decl>());
+  CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordD);
   PushDeclContext(S, Record);
 }
 
-void Sema::ActOnFinishDelayedMemberDeclarations(Scope *S, DeclPtrTy RecordD) {
+void Sema::ActOnFinishDelayedMemberDeclarations(Scope *S, Decl *RecordD) {
   if (!RecordD) return;
   PopDeclContext();
 }
@@ -2727,7 +2834,7 @@ void Sema::ActOnFinishDelayedMemberDeclarations(Scope *S, DeclPtrTy RecordD) {
 /// Method declaration as if we had just parsed the qualified method
 /// name. However, it should not bring the parameters into scope;
 /// that will be performed by ActOnDelayedCXXMethodParameter.
-void Sema::ActOnStartDelayedCXXMethodDeclaration(Scope *S, DeclPtrTy MethodD) {
+void Sema::ActOnStartDelayedCXXMethodDeclaration(Scope *S, Decl *MethodD) {
 }
 
 /// ActOnDelayedCXXMethodParameter - We've already started a delayed
@@ -2735,18 +2842,18 @@ void Sema::ActOnStartDelayedCXXMethodDeclaration(Scope *S, DeclPtrTy MethodD) {
 /// function parameter into scope for use in parsing later parts of
 /// the method declaration. For example, we could see an
 /// ActOnParamDefaultArgument event for this parameter.
-void Sema::ActOnDelayedCXXMethodParameter(Scope *S, DeclPtrTy ParamD) {
+void Sema::ActOnDelayedCXXMethodParameter(Scope *S, Decl *ParamD) {
   if (!ParamD)
     return;
 
-  ParmVarDecl *Param = cast<ParmVarDecl>(ParamD.getAs<Decl>());
+  ParmVarDecl *Param = cast<ParmVarDecl>(ParamD);
 
   // If this parameter has an unparsed default argument, clear it out
   // to make way for the parsed default argument.
   if (Param->hasUnparsedDefaultArg())
     Param->setDefaultArg(0);
 
-  S->AddDecl(DeclPtrTy::make(Param));
+  S->AddDecl(Param);
   if (Param->getDeclName())
     IdResolver.AddDecl(Param);
 }
@@ -2757,13 +2864,13 @@ void Sema::ActOnDelayedCXXMethodParameter(Scope *S, DeclPtrTy ParamD) {
 /// ActOnStartOfFunctionDef action later (not necessarily
 /// immediately!) for this method, if it was also defined inside the
 /// class body.
-void Sema::ActOnFinishDelayedCXXMethodDeclaration(Scope *S, DeclPtrTy MethodD) {
+void Sema::ActOnFinishDelayedCXXMethodDeclaration(Scope *S, Decl *MethodD) {
   if (!MethodD)
     return;
 
   AdjustDeclIfTemplate(MethodD);
 
-  FunctionDecl *Method = cast<FunctionDecl>(MethodD.getAs<Decl>());
+  FunctionDecl *Method = cast<FunctionDecl>(MethodD);
 
   // Now that we have our default arguments, check the constructor
   // again. It could produce additional diagnostics or affect whether
@@ -2784,7 +2891,7 @@ void Sema::ActOnFinishDelayedCXXMethodDeclaration(Scope *S, DeclPtrTy MethodD) {
 /// will be updated to reflect a well-formed type for the constructor and
 /// returned.
 QualType Sema::CheckConstructorDeclarator(Declarator &D, QualType R,
-                                          FunctionDecl::StorageClass &SC) {
+                                          StorageClass &SC) {
   bool isVirtual = D.getDeclSpec().isVirtualSpecified();
 
   // C++ [class.ctor]p3:
@@ -2799,13 +2906,13 @@ QualType Sema::CheckConstructorDeclarator(Declarator &D, QualType R,
         << SourceRange(D.getIdentifierLoc());
     D.setInvalidType();
   }
-  if (SC == FunctionDecl::Static) {
+  if (SC == SC_Static) {
     if (!D.isInvalidType())
       Diag(D.getIdentifierLoc(), diag::err_constructor_cannot_be)
         << "static" << SourceRange(D.getDeclSpec().getStorageClassSpecLoc())
         << SourceRange(D.getIdentifierLoc());
     D.setInvalidType();
-    SC = FunctionDecl::None;
+    SC = SC_None;
   }
 
   DeclaratorChunk::FunctionTypeInfo &FTI = D.getTypeObject(0).Fun;
@@ -2879,8 +2986,9 @@ void Sema::CheckConstructor(CXXConstructorDecl *Constructor) {
     ClassDecl->addedConstructor(Context, Constructor);
 }
 
-/// CheckDestructor - Checks a fully-formed destructor for well-formedness, 
-/// issuing any diagnostics required. Returns true on error.
+/// CheckDestructor - Checks a fully-formed destructor definition for
+/// well-formedness, issuing any diagnostics required.  Returns true
+/// on error.
 bool Sema::CheckDestructor(CXXDestructorDecl *Destructor) {
   CXXRecordDecl *RD = Destructor->getParent();
   
@@ -2911,7 +3019,7 @@ static inline bool
 FTIHasSingleVoidArgument(DeclaratorChunk::FunctionTypeInfo &FTI) {
   return (FTI.NumArgs == 1 && !FTI.isVariadic && FTI.ArgInfo[0].Ident == 0 &&
           FTI.ArgInfo[0].Param &&
-          FTI.ArgInfo[0].Param.getAs<ParmVarDecl>()->getType()->isVoidType());
+          cast<ParmVarDecl>(FTI.ArgInfo[0].Param)->getType()->isVoidType());
 }
 
 /// CheckDestructorDeclarator - Called by ActOnDeclarator to check
@@ -2921,7 +3029,7 @@ FTIHasSingleVoidArgument(DeclaratorChunk::FunctionTypeInfo &FTI) {
 /// will be updated to reflect a well-formed type for the destructor and
 /// returned.
 QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
-                                         FunctionDecl::StorageClass& SC) {
+                                         StorageClass& SC) {
   // C++ [class.dtor]p1:
   //   [...] A typedef-name that names a class is a class-name
   //   (7.1.3); however, a typedef-name that names a class shall not
@@ -2940,14 +3048,14 @@ QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
   //   destructor can be invoked for a const, volatile or const
   //   volatile object. A destructor shall not be declared const,
   //   volatile or const volatile (9.3.2).
-  if (SC == FunctionDecl::Static) {
+  if (SC == SC_Static) {
     if (!D.isInvalidType())
       Diag(D.getIdentifierLoc(), diag::err_destructor_cannot_be)
         << "static" << SourceRange(D.getDeclSpec().getStorageClassSpecLoc())
         << SourceRange(D.getIdentifierLoc())
         << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
     
-    SC = FunctionDecl::None;
+    SC = SC_None;
   }
   if (D.getDeclSpec().hasTypeSpecifier() && !D.isInvalidType()) {
     // Destructors don't have return types, but the parser will
@@ -3015,18 +3123,18 @@ QualType Sema::CheckDestructorDeclarator(Declarator &D, QualType R,
 /// false. Either way, the type @p R will be updated to reflect a
 /// well-formed type for the conversion operator.
 void Sema::CheckConversionDeclarator(Declarator &D, QualType &R,
-                                     FunctionDecl::StorageClass& SC) {
+                                     StorageClass& SC) {
   // C++ [class.conv.fct]p1:
   //   Neither parameter types nor return type can be specified. The
   //   type of a conversion function (8.3.5) is "function taking no
   //   parameter returning conversion-type-id."
-  if (SC == FunctionDecl::Static) {
+  if (SC == SC_Static) {
     if (!D.isInvalidType())
       Diag(D.getIdentifierLoc(), diag::err_conv_function_not_member)
         << "static" << SourceRange(D.getDeclSpec().getStorageClassSpecLoc())
         << SourceRange(D.getIdentifierLoc());
     D.setInvalidType();
-    SC = FunctionDecl::None;
+    SC = SC_None;
   }
 
   QualType ConvType = GetTypeFromParser(D.getName().ConversionFunctionId);
@@ -3106,7 +3214,7 @@ void Sema::CheckConversionDeclarator(Declarator &D, QualType &R,
 /// the declaration of the given C++ conversion function. This routine
 /// is responsible for recording the conversion function in the C++
 /// class, if possible.
-Sema::DeclPtrTy Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
+Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
   assert(Conversion && "Expected to receive a conversion function declaration");
 
   CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Conversion->getDeclContext());
@@ -3147,10 +3255,10 @@ Sema::DeclPtrTy Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
       if (ClassDecl->replaceConversion(
                                    ConversionTemplate->getPreviousDeclaration(),
                                        ConversionTemplate))
-        return DeclPtrTy::make(ConversionTemplate);
+        return ConversionTemplate;
     } else if (ClassDecl->replaceConversion(Conversion->getPreviousDeclaration(),
                                             Conversion))
-      return DeclPtrTy::make(Conversion);
+      return Conversion;
     assert(Conversion->isInvalidDecl() && "Conversion should not get here.");
   } else if (FunctionTemplateDecl *ConversionTemplate
                = Conversion->getDescribedFunctionTemplate())
@@ -3158,28 +3266,36 @@ Sema::DeclPtrTy Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
   else 
     ClassDecl->addConversionFunction(Conversion);
 
-  return DeclPtrTy::make(Conversion);
+  return Conversion;
 }
 
 //===----------------------------------------------------------------------===//
 // Namespace Handling
 //===----------------------------------------------------------------------===//
 
+
+
 /// ActOnStartNamespaceDef - This is called at the start of a namespace
 /// definition.
-Sema::DeclPtrTy Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
-                                             SourceLocation IdentLoc,
-                                             IdentifierInfo *II,
-                                             SourceLocation LBrace,
-                                             AttributeList *AttrList) {
-  NamespaceDecl *Namespc =
-      NamespaceDecl::Create(Context, CurContext, IdentLoc, II);
+Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
+                                   SourceLocation InlineLoc,
+                                   SourceLocation IdentLoc,
+                                   IdentifierInfo *II,
+                                   SourceLocation LBrace,
+                                   AttributeList *AttrList) {
+  // anonymous namespace starts at its left brace
+  NamespaceDecl *Namespc = NamespaceDecl::Create(Context, CurContext,
+    (II ? IdentLoc : LBrace) , II);
   Namespc->setLBracLoc(LBrace);
+  Namespc->setInline(InlineLoc.isValid());
 
   Scope *DeclRegionScope = NamespcScope->getParent();
 
   ProcessDeclAttributeList(DeclRegionScope, Namespc, AttrList);
 
+  if (const VisibilityAttr *attr = Namespc->getAttr<VisibilityAttr>())
+    PushVisibilityAttr(attr);
+
   if (II) {
     // C++ [namespace.def]p2:
     // The identifier in an original-namespace-definition shall not have been
@@ -3194,15 +3310,25 @@ Sema::DeclPtrTy Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
 
     if (NamespaceDecl *OrigNS = dyn_cast_or_null<NamespaceDecl>(PrevDecl)) {
       // This is an extended namespace definition.
+      if (Namespc->isInline() != OrigNS->isInline()) {
+        // inline-ness must match
+        Diag(Namespc->getLocation(), diag::err_inline_namespace_mismatch)
+          << Namespc->isInline();
+        Diag(OrigNS->getLocation(), diag::note_previous_definition);
+        Namespc->setInvalidDecl();
+        // Recover by ignoring the new namespace's inline status.
+        Namespc->setInline(OrigNS->isInline());
+      }
+
       // Attach this namespace decl to the chain of extended namespace
       // definitions.
       OrigNS->setNextNamespace(Namespc);
       Namespc->setOriginalNamespace(OrigNS->getOriginalNamespace());
 
       // Remove the previous declaration from the scope.
-      if (DeclRegionScope->isDeclScope(DeclPtrTy::make(OrigNS))) {
+      if (DeclRegionScope->isDeclScope(OrigNS)) {
         IdResolver.RemoveDecl(OrigNS);
-        DeclRegionScope->RemoveDecl(DeclPtrTy::make(OrigNS));
+        DeclRegionScope->RemoveDecl(OrigNS);
       }
     } else if (PrevDecl) {
       // This is an invalid name redefinition.
@@ -3212,15 +3338,15 @@ Sema::DeclPtrTy Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
       Namespc->setInvalidDecl();
       // Continue on to push Namespc as current DeclContext and return it.
     } else if (II->isStr("std") && 
-               CurContext->getLookupContext()->isTranslationUnit()) {
+               CurContext->getRedeclContext()->isTranslationUnit()) {
       // This is the first "real" definition of the namespace "std", so update
       // our cache of the "std" namespace to point at this definition.
-      if (StdNamespace) {
+      if (NamespaceDecl *StdNS = getStdNamespace()) {
         // We had already defined a dummy namespace "std". Link this new 
         // namespace definition to the dummy namespace "std".
-        StdNamespace->setNextNamespace(Namespc);
-        StdNamespace->setLocation(IdentLoc);
-        Namespc->setOriginalNamespace(StdNamespace->getOriginalNamespace());
+        StdNS->setNextNamespace(Namespc);
+        StdNS->setLocation(IdentLoc);
+        Namespc->setOriginalNamespace(StdNS->getOriginalNamespace());
       }
       
       // Make our StdNamespace cache point at the first real definition of the
@@ -3235,7 +3361,7 @@ Sema::DeclPtrTy Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
 
     // Link the anonymous namespace into its parent.
     NamespaceDecl *PrevDecl;
-    DeclContext *Parent = CurContext->getLookupContext();
+    DeclContext *Parent = CurContext->getRedeclContext();
     if (TranslationUnitDecl *TU = dyn_cast<TranslationUnitDecl>(Parent)) {
       PrevDecl = TU->getAnonymousNamespace();
       TU->setAnonymousNamespace(Namespc);
@@ -3251,6 +3377,16 @@ Sema::DeclPtrTy Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
       assert(!PrevDecl->getNextNamespace());
       Namespc->setOriginalNamespace(PrevDecl->getOriginalNamespace());
       PrevDecl->setNextNamespace(Namespc);
+
+      if (Namespc->isInline() != PrevDecl->isInline()) {
+        // inline-ness must match
+        Diag(Namespc->getLocation(), diag::err_inline_namespace_mismatch)
+          << Namespc->isInline();
+        Diag(PrevDecl->getLocation(), diag::note_previous_definition);
+        Namespc->setInvalidDecl();
+        // Recover by ignoring the new namespace's inline status.
+        Namespc->setInline(PrevDecl->isInline());
+      }
     }
 
     CurContext->addDecl(Namespc);
@@ -3292,7 +3428,7 @@ Sema::DeclPtrTy Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
   // for the namespace has the declarations that showed up in that particular
   // namespace definition.
   PushDeclContext(NamespcScope, Namespc);
-  return DeclPtrTy::make(Namespc);
+  return Namespc;
 }
 
 /// getNamespaceDecl - Returns the namespace a decl represents. If the decl
@@ -3305,30 +3441,41 @@ static inline NamespaceDecl *getNamespaceDecl(NamedDecl *D) {
 
 /// ActOnFinishNamespaceDef - This callback is called after a namespace is
 /// exited. Decl is the DeclTy returned by ActOnStartNamespaceDef.
-void Sema::ActOnFinishNamespaceDef(DeclPtrTy D, SourceLocation RBrace) {
-  Decl *Dcl = D.getAs<Decl>();
+void Sema::ActOnFinishNamespaceDef(Decl *Dcl, SourceLocation RBrace) {
   NamespaceDecl *Namespc = dyn_cast_or_null<NamespaceDecl>(Dcl);
   assert(Namespc && "Invalid parameter, expected NamespaceDecl");
   Namespc->setRBracLoc(RBrace);
   PopDeclContext();
+  if (Namespc->hasAttr<VisibilityAttr>())
+    PopPragmaVisibility();
+}
+
+CXXRecordDecl *Sema::getStdBadAlloc() const {
+  return cast_or_null<CXXRecordDecl>(
+                                  StdBadAlloc.get(Context.getExternalSource()));
+}
+
+NamespaceDecl *Sema::getStdNamespace() const {
+  return cast_or_null<NamespaceDecl>(
+                                 StdNamespace.get(Context.getExternalSource()));
 }
 
 /// \brief Retrieve the special "std" namespace, which may require us to 
 /// implicitly define the namespace.
-NamespaceDecl *Sema::getStdNamespace() {
+NamespaceDecl *Sema::getOrCreateStdNamespace() {
   if (!StdNamespace) {
     // The "std" namespace has not yet been defined, so build one implicitly.
     StdNamespace = NamespaceDecl::Create(Context, 
                                          Context.getTranslationUnitDecl(),
                                          SourceLocation(),
                                          &PP.getIdentifierTable().get("std"));
-    StdNamespace->setImplicit(true);
+    getStdNamespace()->setImplicit(true);
   }
   
-  return StdNamespace;
+  return getStdNamespace();
 }
 
-Sema::DeclPtrTy Sema::ActOnUsingDirective(Scope *S,
+Decl *Sema::ActOnUsingDirective(Scope *S,
                                           SourceLocation UsingLoc,
                                           SourceLocation NamespcLoc,
                                           CXXScopeSpec &SS,
@@ -3349,7 +3496,7 @@ Sema::DeclPtrTy Sema::ActOnUsingDirective(Scope *S,
   LookupResult R(*this, NamespcName, IdentLoc, LookupNamespaceName);
   LookupParsedName(R, S, &SS);
   if (R.isAmbiguous())
-    return DeclPtrTy();
+    return 0;
 
   if (R.empty()) {
     // Allow "using namespace std;" or "using namespace ::std;" even if 
@@ -3357,7 +3504,7 @@ Sema::DeclPtrTy Sema::ActOnUsingDirective(Scope *S,
     if ((!Qualifier || Qualifier->getKind() == NestedNameSpecifier::Global) &&
         NamespcName->isStr("std")) {
       Diag(IdentLoc, diag::ext_using_undefined_std);
-      R.addDecl(getStdNamespace());
+      R.addDecl(getOrCreateStdNamespace());
       R.resolveKind();
     } 
     // Otherwise, attempt typo correction.
@@ -3416,7 +3563,7 @@ Sema::DeclPtrTy Sema::ActOnUsingDirective(Scope *S,
 
   // FIXME: We ignore attributes for now.
   delete AttrList;
-  return DeclPtrTy::make(UDir);
+  return UDir;
 }
 
 void Sema::PushUsingDirective(Scope *S, UsingDirectiveDecl *UDir) {
@@ -3428,11 +3575,11 @@ void Sema::PushUsingDirective(Scope *S, UsingDirectiveDecl *UDir) {
   else
     // Otherwise it is block-sope. using-directives will affect lookup
     // only to the end of scope.
-    S->PushUsingDirective(DeclPtrTy::make(UDir));
+    S->PushUsingDirective(UDir);
 }
 
 
-Sema::DeclPtrTy Sema::ActOnUsingDeclaration(Scope *S,
+Decl *Sema::ActOnUsingDeclaration(Scope *S,
                                             AccessSpecifier AS,
                                             bool HasUsingKeyword,
                                             SourceLocation UsingLoc,
@@ -3457,22 +3604,23 @@ Sema::DeclPtrTy Sema::ActOnUsingDeclaration(Scope *S,
 
     Diag(Name.getSourceRange().getBegin(), diag::err_using_decl_constructor)
       << SS.getRange();
-    return DeclPtrTy();
+    return 0;
       
   case UnqualifiedId::IK_DestructorName:
     Diag(Name.getSourceRange().getBegin(), diag::err_using_decl_destructor)
       << SS.getRange();
-    return DeclPtrTy();
+    return 0;
       
   case UnqualifiedId::IK_TemplateId:
     Diag(Name.getSourceRange().getBegin(), diag::err_using_decl_template_id)
       << SourceRange(Name.TemplateId->LAngleLoc, Name.TemplateId->RAngleLoc);
-    return DeclPtrTy();
+    return 0;
   }
-  
-  DeclarationName TargetName = GetNameFromUnqualifiedId(Name);
+
+  DeclarationNameInfo TargetNameInfo = GetNameFromUnqualifiedId(Name);
+  DeclarationName TargetName = TargetNameInfo.getName();
   if (!TargetName)
-    return DeclPtrTy();
+    return 0;
 
   // Warn about using declarations.
   // TODO: store that the declaration was written without 'using' and
@@ -3486,14 +3634,13 @@ Sema::DeclPtrTy Sema::ActOnUsingDeclaration(Scope *S,
   }
 
   NamedDecl *UD = BuildUsingDeclaration(S, AS, UsingLoc, SS,
-                                        Name.getSourceRange().getBegin(),
-                                        TargetName, AttrList,
+                                        TargetNameInfo, AttrList,
                                         /* IsInstantiation */ false,
                                         IsTypeName, TypenameLoc);
   if (UD)
     PushOnScopeChains(UD, S, /*AddToContext*/ false);
 
-  return DeclPtrTy::make(UD);
+  return UD;
 }
 
 /// \brief Determine whether a using declaration considers the given
@@ -3717,7 +3864,7 @@ void Sema::HideUsingShadowDecl(Scope *S, UsingShadowDecl *Shadow) {
 
   // ...and the scope, if applicable...
   if (S) {
-    S->RemoveDecl(DeclPtrTy::make(static_cast<Decl*>(Shadow)));
+    S->RemoveDecl(Shadow);
     IdResolver.RemoveDecl(Shadow);
   }
 
@@ -3736,13 +3883,13 @@ void Sema::HideUsingShadowDecl(Scope *S, UsingShadowDecl *Shadow) {
 NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
                                        SourceLocation UsingLoc,
                                        CXXScopeSpec &SS,
-                                       SourceLocation IdentLoc,
-                                       DeclarationName Name,
+                                       const DeclarationNameInfo &NameInfo,
                                        AttributeList *AttrList,
                                        bool IsInstantiation,
                                        bool IsTypeName,
                                        SourceLocation TypenameLoc) {
   assert(!SS.isInvalid() && "Invalid CXXScopeSpec.");
+  SourceLocation IdentLoc = NameInfo.getLoc();
   assert(IdentLoc.isValid() && "Invalid TargetName location.");
 
   // FIXME: We ignore attributes for now.
@@ -3754,7 +3901,7 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
   }
 
   // Do the redeclaration lookup in the current scope.
-  LookupResult Previous(*this, Name, IdentLoc, LookupUsingDeclName,
+  LookupResult Previous(*this, NameInfo, LookupUsingDeclName,
                         ForRedeclaration);
   Previous.setHideTags(false);
   if (S) {
@@ -3793,15 +3940,15 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
       D = UnresolvedUsingTypenameDecl::Create(Context, CurContext,
                                               UsingLoc, TypenameLoc,
                                               SS.getRange(), NNS,
-                                              IdentLoc, Name);
+                                              IdentLoc, NameInfo.getName());
     } else {
       D = UnresolvedUsingValueDecl::Create(Context, CurContext,
-                                           UsingLoc, SS.getRange(), NNS,
-                                           IdentLoc, Name);
+                                           UsingLoc, SS.getRange(),
+                                           NNS, NameInfo);
     }
   } else {
-    D = UsingDecl::Create(Context, CurContext, IdentLoc,
-                          SS.getRange(), UsingLoc, NNS, Name,
+    D = UsingDecl::Create(Context, CurContext,
+                          SS.getRange(), UsingLoc, NNS, NameInfo,
                           IsTypeName);
   }
   D->setAccess(AS);
@@ -3817,7 +3964,7 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
 
   // Look up the target name.
 
-  LookupResult R(*this, Name, IdentLoc, LookupOrdinaryName);
+  LookupResult R(*this, NameInfo, LookupOrdinaryName);
 
   // Unlike most lookups, we don't always want to hide tag
   // declarations: tag names are visible through the using declaration
@@ -3830,7 +3977,7 @@ NamedDecl *Sema::BuildUsingDeclaration(Scope *S, AccessSpecifier AS,
 
   if (R.empty()) {
     Diag(IdentLoc, diag::err_no_member) 
-      << Name << LookupContext << SS.getRange();
+      << NameInfo.getName() << LookupContext << SS.getRange();
     UD->setInvalidDecl();
     return UD;
   }
@@ -3894,7 +4041,7 @@ bool Sema::CheckUsingDeclRedeclaration(SourceLocation UsingLoc,
   //   allowed.
   //
   // That's in non-member contexts.
-  if (!CurContext->getLookupContext()->isRecord())
+  if (!CurContext->getRedeclContext()->isRecord())
     return false;
 
   NestedNameSpecifier *Qual
@@ -4069,7 +4216,7 @@ bool Sema::CheckUsingDeclQualifier(SourceLocation UsingLoc,
   return true;
 }
 
-Sema::DeclPtrTy Sema::ActOnNamespaceAliasDef(Scope *S,
+Decl *Sema::ActOnNamespaceAliasDef(Scope *S,
                                              SourceLocation NamespaceLoc,
                                              SourceLocation AliasLoc,
                                              IdentifierInfo *Alias,
@@ -4096,18 +4243,18 @@ Sema::DeclPtrTy Sema::ActOnNamespaceAliasDef(Scope *S,
       // declaration to maintain better source information.
       if (!R.isAmbiguous() && !R.empty() &&
           AD->getNamespace()->Equals(getNamespaceDecl(R.getFoundDecl())))
-        return DeclPtrTy();
+        return 0;
     }
 
     unsigned DiagID = isa<NamespaceDecl>(PrevDecl) ? diag::err_redefinition :
       diag::err_redefinition_different_kind;
     Diag(AliasLoc, DiagID) << Alias;
     Diag(PrevDecl->getLocation(), diag::note_previous_definition);
-    return DeclPtrTy();
+    return 0;
   }
 
   if (R.isAmbiguous())
-    return DeclPtrTy();
+    return 0;
 
   if (R.empty()) {
     if (DeclarationName Corrected = CorrectTypo(R, S, &SS, 0, false, 
@@ -4135,7 +4282,7 @@ Sema::DeclPtrTy Sema::ActOnNamespaceAliasDef(Scope *S,
     
     if (R.empty()) {
       Diag(NamespaceLoc, diag::err_expected_namespace_name) << SS.getRange();
-      return DeclPtrTy();
+      return 0;
     }
   }
 
@@ -4146,7 +4293,7 @@ Sema::DeclPtrTy Sema::ActOnNamespaceAliasDef(Scope *S,
                                IdentLoc, R.getFoundDecl());
 
   PushOnScopeChains(AliasDecl, S);
-  return DeclPtrTy::make(AliasDecl);
+  return AliasDecl;
 }
 
 namespace {
@@ -4242,9 +4389,9 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
     = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
   DeclarationName Name
     = Context.DeclarationNames.getCXXConstructorName(ClassType);
+  DeclarationNameInfo NameInfo(Name, ClassDecl->getLocation());
   CXXConstructorDecl *DefaultCon
-    = CXXConstructorDecl::Create(Context, ClassDecl,
-                                 ClassDecl->getLocation(), Name,
+    = CXXConstructorDecl::Create(Context, ClassDecl, NameInfo,
                                  Context.getFunctionType(Context.VoidTy,
                                                          0, 0, false, 0,
                                        ExceptSpec.hasExceptionSpecification(),
@@ -4348,9 +4495,9 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
     = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
   DeclarationName Name
     = Context.DeclarationNames.getCXXDestructorName(ClassType);
+  DeclarationNameInfo NameInfo(Name, ClassDecl->getLocation());
   CXXDestructorDecl *Destructor
-    = CXXDestructorDecl::Create(Context, ClassDecl,
-                                ClassDecl->getLocation(), Name, Ty,
+    = CXXDestructorDecl::Create(Context, ClassDecl, NameInfo, Ty,
                                 /*isInline=*/true,
                                 /*isImplicitlyDeclared=*/true);
   Destructor->setAccess(AS_public);
@@ -4426,13 +4573,10 @@ void Sema::DefineImplicitDestructor(SourceLocation CurrentLocation,
 /// \param Depth Internal parameter recording the depth of the recursion.
 ///
 /// \returns A statement or a loop that copies the expressions.
-static Sema::OwningStmtResult
+static StmtResult
 BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T, 
-                      Sema::OwningExprResult To, Sema::OwningExprResult From,
+                      Expr *To, Expr *From,
                       bool CopyingBaseSubobject, unsigned Depth = 0) {
-  typedef Sema::OwningStmtResult OwningStmtResult;
-  typedef Sema::OwningExprResult OwningExprResult;
-  
   // C++0x [class.copy]p30:
   //   Each subobject is assigned in the manner appropriate to its type:
   //
@@ -4489,21 +4633,21 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
                                                T.getTypePtr()));
     
     // Create the reference to operator=.
-    OwningExprResult OpEqualRef
-      = S.BuildMemberReferenceExpr(move(To), T, Loc, /*isArrow=*/false, SS, 
+    ExprResult OpEqualRef
+      = S.BuildMemberReferenceExpr(To, T, Loc, /*isArrow=*/false, SS, 
                                    /*FirstQualifierInScope=*/0, OpLookup, 
                                    /*TemplateArgs=*/0,
                                    /*SuppressQualifierCheck=*/true);
     if (OpEqualRef.isInvalid())
-      return S.StmtError();
+      return StmtError();
     
     // Build the call to the assignment operator.
-    Expr *FromE = From.takeAs<Expr>();
-    OwningExprResult Call = S.BuildCallToMemberFunction(/*Scope=*/0, 
+
+    ExprResult Call = S.BuildCallToMemberFunction(/*Scope=*/0, 
                                                       OpEqualRef.takeAs<Expr>(),
-                                                        Loc, &FromE, 1, 0, Loc);
+                                                        Loc, &From, 1, 0, Loc);
     if (Call.isInvalid())
-      return S.StmtError();
+      return StmtError();
     
     return S.Owned(Call.takeAs<Stmt>());
   }
@@ -4512,12 +4656,9 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
   //       operator is used.
   const ConstantArrayType *ArrayTy = S.Context.getAsConstantArrayType(T);  
   if (!ArrayTy) {
-    OwningExprResult Assignment = S.CreateBuiltinBinOp(Loc, 
-                                                       BinaryOperator::Assign,
-                                                       To.takeAs<Expr>(),
-                                                       From.takeAs<Expr>());
+    ExprResult Assignment = S.CreateBuiltinBinOp(Loc, BO_Assign, To, From);
     if (Assignment.isInvalid())
-      return S.StmtError();
+      return StmtError();
     
     return S.Owned(Assignment.takeAs<Stmt>());
   }
@@ -4543,11 +4684,11 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
   VarDecl *IterationVar = VarDecl::Create(S.Context, S.CurContext, Loc,
                                           IterationVarName, SizeType,
                             S.Context.getTrivialTypeSourceInfo(SizeType, Loc),
-                                          VarDecl::None, VarDecl::None);
+                                          SC_None, SC_None);
   
   // Initialize the iteration variable to zero.
   llvm::APInt Zero(S.Context.getTypeSize(SizeType), 0);
-  IterationVar->setInit(new (S.Context) IntegerLiteral(Zero, SizeType, Loc));
+  IterationVar->setInit(IntegerLiteral::Create(S.Context, Zero, SizeType, Loc));
 
   // Create a reference to the iteration variable; we'll use this several
   // times throughout.
@@ -4561,43 +4702,37 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
   // Create the comparison against the array bound.
   llvm::APInt Upper = ArrayTy->getSize();
   Upper.zextOrTrunc(S.Context.getTypeSize(SizeType));
-  OwningExprResult Comparison
-    = S.Owned(new (S.Context) BinaryOperator(IterationVarRef->Retain(),
-                           new (S.Context) IntegerLiteral(Upper, SizeType, Loc),
-                                    BinaryOperator::NE, S.Context.BoolTy, Loc));
+  Expr *Comparison
+    = new (S.Context) BinaryOperator(IterationVarRef->Retain(),
+                           IntegerLiteral::Create(S.Context,
+                                                  Upper, SizeType, Loc),
+                                                  BO_NE, S.Context.BoolTy, Loc);
   
   // Create the pre-increment of the iteration variable.
-  OwningExprResult Increment
-    = S.Owned(new (S.Context) UnaryOperator(IterationVarRef->Retain(),
-                                            UnaryOperator::PreInc,
-                                            SizeType, Loc));
+  Expr *Increment
+    = new (S.Context) UnaryOperator(IterationVarRef->Retain(),
+                                    UO_PreInc,
+                                    SizeType, Loc);
   
   // Subscript the "from" and "to" expressions with the iteration variable.
-  From = S.CreateBuiltinArraySubscriptExpr(move(From), Loc,
-                                           S.Owned(IterationVarRef->Retain()),
-                                           Loc);
-  To = S.CreateBuiltinArraySubscriptExpr(move(To), Loc,
-                                         S.Owned(IterationVarRef->Retain()),
-                                         Loc);
-  assert(!From.isInvalid() && "Builtin subscripting can't fail!");
-  assert(!To.isInvalid() && "Builtin subscripting can't fail!");
+  From = AssertSuccess(S.CreateBuiltinArraySubscriptExpr(From, Loc,
+                                                         IterationVarRef, Loc));
+  To = AssertSuccess(S.CreateBuiltinArraySubscriptExpr(To, Loc,
+                                                       IterationVarRef, Loc));
   
   // Build the copy for an individual element of the array.
-  OwningStmtResult Copy = BuildSingleCopyAssign(S, Loc, 
+  StmtResult Copy = BuildSingleCopyAssign(S, Loc, 
                                                 ArrayTy->getElementType(),
-                                                move(To), move(From), 
+                                                To, From, 
                                                 CopyingBaseSubobject, Depth+1);
-  if (Copy.isInvalid()) {
-    InitStmt->Destroy(S.Context);
-    return S.StmtError();
-  }
+  if (Copy.isInvalid())
+    return StmtError();
   
   // Construct the loop that copies all elements of this array.
-  return S.ActOnForStmt(Loc, Loc, S.Owned(InitStmt), 
+  return S.ActOnForStmt(Loc, Loc, InitStmt, 
                         S.MakeFullExpr(Comparison),
-                        Sema::DeclPtrTy(), 
-                        S.MakeFullExpr(Increment),
-                        Loc, move(Copy));
+                        0, S.MakeFullExpr(Increment),
+                        Loc, Copy.take());
 }
 
 /// \brief Determine whether the given class has a copy assignment operator 
@@ -4747,8 +4882,9 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   //   An implicitly-declared copy assignment operator is an inline public
   //   member of its class.
   DeclarationName Name = Context.DeclarationNames.getCXXOperatorName(OO_Equal);
+  DeclarationNameInfo NameInfo(Name, ClassDecl->getLocation());
   CXXMethodDecl *CopyAssignment
-    = CXXMethodDecl::Create(Context, ClassDecl, ClassDecl->getLocation(), Name,
+    = CXXMethodDecl::Create(Context, ClassDecl, NameInfo,
                             Context.getFunctionType(RetType, &ArgType, 1,
                                                     false, 0,
                                          ExceptSpec.hasExceptionSpecification(),
@@ -4757,7 +4893,7 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
                                                     ExceptSpec.data(),
                                                     FunctionType::ExtInfo()),
                             /*TInfo=*/0, /*isStatic=*/false,
-                            /*StorageClassAsWritten=*/FunctionDecl::None,
+                            /*StorageClassAsWritten=*/SC_None,
                             /*isInline=*/true);
   CopyAssignment->setAccess(AS_public);
   CopyAssignment->setImplicit();
@@ -4769,8 +4905,8 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
                                                ClassDecl->getLocation(),
                                                /*Id=*/0,
                                                ArgType, /*TInfo=*/0,
-                                               VarDecl::None,
-                                               VarDecl::None, 0);
+                                               SC_None,
+                                               SC_None, 0);
   CopyAssignment->setParams(&FromParam, 1);
   
   // Note that we have added this copy-assignment operator.
@@ -4814,7 +4950,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   //   which they were declared in the class definition.
   
   // The statements that form the synthesized function body.
-  ASTOwningVector<&ActionBase::DeleteStmt> Statements(*this);
+  ASTOwningVector<Stmt*> Statements(*this);
   
   // The parameter for the "other" object, which we are copying from.
   ParmVarDecl *Other = CopyAssignOperator->getParamDecl(0);
@@ -4854,30 +4990,32 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
       continue;
     }
 
+    CXXCastPath BasePath;
+    BasePath.push_back(Base);
+
     // Construct the "from" expression, which is an implicit cast to the
     // appropriately-qualified base type.
     Expr *From = OtherRef->Retain();
     ImpCastExprToType(From, Context.getQualifiedType(BaseType, OtherQuals),
-                      CastExpr::CK_UncheckedDerivedToBase, /*isLvalue=*/true, 
-                      CXXBaseSpecifierArray(Base));
+                      CK_UncheckedDerivedToBase,
+                      VK_LValue, &BasePath);
 
     // Dereference "this".
-    OwningExprResult To = CreateBuiltinUnaryOp(Loc, UnaryOperator::Deref,
-                                               Owned(This->Retain()));
+    ExprResult To = CreateBuiltinUnaryOp(Loc, UO_Deref, This);
     
     // Implicitly cast "this" to the appropriately-qualified base type.
     Expr *ToE = To.takeAs<Expr>();
     ImpCastExprToType(ToE, 
                       Context.getCVRQualifiedType(BaseType,
                                       CopyAssignOperator->getTypeQualifiers()),
-                      CastExpr::CK_UncheckedDerivedToBase, 
-                      /*isLvalue=*/true, CXXBaseSpecifierArray(Base));
+                      CK_UncheckedDerivedToBase, 
+                      VK_LValue, &BasePath);
     To = Owned(ToE);
 
     // Build the copy.
-    OwningStmtResult Copy = BuildSingleCopyAssign(*this, Loc, BaseType,
-                                                  move(To), Owned(From),
-                                                /*CopyingBaseSubobject=*/true);
+    StmtResult Copy = BuildSingleCopyAssign(*this, Loc, BaseType,
+                                            To.get(), From,
+                                            /*CopyingBaseSubobject=*/true);
     if (Copy.isInvalid()) {
       Diag(CurrentLocation, diag::note_member_synthesized_at) 
         << CXXCopyAssignment << Context.getTagDeclType(ClassDecl);
@@ -4934,12 +5072,10 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
                               LookupMemberName);
     MemberLookup.addDecl(*Field);
     MemberLookup.resolveKind();
-    OwningExprResult From = BuildMemberReferenceExpr(Owned(OtherRef->Retain()),
-                                                     OtherRefType,
+    ExprResult From = BuildMemberReferenceExpr(OtherRef, OtherRefType,
                                                      Loc, /*IsArrow=*/false,
                                                      SS, 0, MemberLookup, 0);
-    OwningExprResult To = BuildMemberReferenceExpr(Owned(This->Retain()),
-                                                   This->getType(),
+    ExprResult To = BuildMemberReferenceExpr(This, This->getType(),
                                                    Loc, /*IsArrow=*/true,
                                                    SS, 0, MemberLookup, 0);
     assert(!From.isInvalid() && "Implicit field reference cannot fail");
@@ -4967,8 +5103,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
       }
           
       // Take the address of the field references for "from" and "to".
-      From = CreateBuiltinUnaryOp(Loc, UnaryOperator::AddrOf, move(From));
-      To = CreateBuiltinUnaryOp(Loc, UnaryOperator::AddrOf, move(To));
+      From = CreateBuiltinUnaryOp(Loc, UO_AddrOf, From.get());
+      To = CreateBuiltinUnaryOp(Loc, UO_AddrOf, To.get());
           
       bool NeedsCollectableMemCpy = 
           (BaseType->isRecordType() && 
@@ -5016,22 +5152,22 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
         assert(BuiltinMemCpyRef && "Builtin reference cannot fail");
       }
           
-      ASTOwningVector<&ActionBase::DeleteExpr> CallArgs(*this);
+      ASTOwningVector<Expr*> CallArgs(*this);
       CallArgs.push_back(To.takeAs<Expr>());
       CallArgs.push_back(From.takeAs<Expr>());
-      CallArgs.push_back(new (Context) IntegerLiteral(Size, SizeType, Loc));
+      CallArgs.push_back(IntegerLiteral::Create(Context, Size, SizeType, Loc));
       llvm::SmallVector<SourceLocation, 4> Commas; // FIXME: Silly
       Commas.push_back(Loc);
       Commas.push_back(Loc);
-      OwningExprResult Call = ExprError();
+      ExprResult Call = ExprError();
       if (NeedsCollectableMemCpy)
         Call = ActOnCallExpr(/*Scope=*/0,
-                             Owned(CollectableMemCpyRef->Retain()),
+                             CollectableMemCpyRef,
                              Loc, move_arg(CallArgs), 
                              Commas.data(), Loc);
       else
         Call = ActOnCallExpr(/*Scope=*/0,
-                             Owned(BuiltinMemCpyRef->Retain()),
+                             BuiltinMemCpyRef,
                              Loc, move_arg(CallArgs), 
                              Commas.data(), Loc);
           
@@ -5041,8 +5177,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     }
     
     // Build the copy of this field.
-    OwningStmtResult Copy = BuildSingleCopyAssign(*this, Loc, FieldType, 
-                                                  move(To), move(From),
+    StmtResult Copy = BuildSingleCopyAssign(*this, Loc, FieldType, 
+                                                  To.get(), From.get(),
                                               /*CopyingBaseSubobject=*/false);
     if (Copy.isInvalid()) {
       Diag(CurrentLocation, diag::note_member_synthesized_at) 
@@ -5057,10 +5193,9 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
 
   if (!Invalid) {
     // Add a "return *this;"
-    OwningExprResult ThisObj = CreateBuiltinUnaryOp(Loc, UnaryOperator::Deref,
-                                                    Owned(This->Retain()));
+    ExprResult ThisObj = CreateBuiltinUnaryOp(Loc, UO_Deref, This);
     
-    OwningStmtResult Return = ActOnReturnStmt(Loc, move(ThisObj));
+    StmtResult Return = ActOnReturnStmt(Loc, ThisObj.get());
     if (Return.isInvalid())
       Invalid = true;
     else {
@@ -5079,7 +5214,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     return;
   }
   
-  OwningStmtResult Body = ActOnCompoundStmt(Loc, Loc, move_arg(Statements),
+  StmtResult Body = ActOnCompoundStmt(Loc, Loc, move_arg(Statements),
                                             /*isStmtExpr=*/false);
   assert(!Body.isInvalid() && "Compound statement creation cannot fail");
   CopyAssignOperator->setBody(Body.takeAs<Stmt>());
@@ -5220,9 +5355,9 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   DeclarationName Name
     = Context.DeclarationNames.getCXXConstructorName(
                                            Context.getCanonicalType(ClassType));
+  DeclarationNameInfo NameInfo(Name, ClassDecl->getLocation());
   CXXConstructorDecl *CopyConstructor
-    = CXXConstructorDecl::Create(Context, ClassDecl,
-                                 ClassDecl->getLocation(), Name,
+    = CXXConstructorDecl::Create(Context, ClassDecl, NameInfo,
                                  Context.getFunctionType(Context.VoidTy,
                                                          &ArgType, 1,
                                                          false, 0,
@@ -5248,8 +5383,8 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
                                                ClassDecl->getLocation(),
                                                /*IdentifierInfo=*/0,
                                                ArgType, /*TInfo=*/0,
-                                               VarDecl::None,
-                                               VarDecl::None, 0);
+                                               SC_None,
+                                               SC_None, 0);
   CopyConstructor->setParams(&FromParam, 1);
   if (Scope *S = getScopeForContext(ClassDecl))
     PushOnScopeChains(CopyConstructor, S, false);
@@ -5288,12 +5423,12 @@ void Sema::DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
   CopyConstructor->setUsed();
 }
 
-Sema::OwningExprResult
+ExprResult
 Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             CXXConstructorDecl *Constructor,
                             MultiExprArg ExprArgs,
                             bool RequiresZeroInit,
-                            CXXConstructExpr::ConstructionKind ConstructKind) {
+                            unsigned ConstructKind) {
   bool Elidable = false;
 
   // C++0x [class.copy]p34:
@@ -5309,6 +5444,7 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
   if (Constructor->isCopyConstructor() && ExprArgs.size() >= 1) {
     Expr *SubExpr = ((Expr **)ExprArgs.get())[0];
     Elidable = SubExpr->isTemporaryObject() &&
+      ConstructKind == CXXConstructExpr::CK_Complete &&
       Context.hasSameUnqualifiedType(SubExpr->getType(), 
                            Context.getTypeDeclType(Constructor->getParent()));
   }
@@ -5320,27 +5456,28 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
 
 /// BuildCXXConstructExpr - Creates a complete call to a constructor,
 /// including handling of its default argument expressions.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             CXXConstructorDecl *Constructor, bool Elidable,
                             MultiExprArg ExprArgs,
                             bool RequiresZeroInit,
-                            CXXConstructExpr::ConstructionKind ConstructKind) {
+                            unsigned ConstructKind) {
   unsigned NumExprs = ExprArgs.size();
   Expr **Exprs = (Expr **)ExprArgs.release();
 
   MarkDeclarationReferenced(ConstructLoc, Constructor);
   return Owned(CXXConstructExpr::Create(Context, DeclInitType, ConstructLoc,
                                         Constructor, Elidable, Exprs, NumExprs, 
-                                        RequiresZeroInit, ConstructKind));
+                                        RequiresZeroInit,
+              static_cast<CXXConstructExpr::ConstructionKind>(ConstructKind)));
 }
 
 bool Sema::InitializeVarWithConstructor(VarDecl *VD,
                                         CXXConstructorDecl *Constructor,
                                         MultiExprArg Exprs) {
-  OwningExprResult TempResult =
+  ExprResult TempResult =
     BuildCXXConstructExpr(VD->getLocation(), VD->getType(), Constructor,
-                          move(Exprs));
+                          move(Exprs), false, CXXConstructExpr::CK_Complete);
   if (TempResult.isInvalid())
     return true;
 
@@ -5362,19 +5499,21 @@ void Sema::FinalizeVarWithDestructor(VarDecl *VD, const RecordType *Record) {
                           PDiag(diag::err_access_dtor_var)
                             << VD->getDeclName()
                             << VD->getType());
+
+    if (!VD->isInvalidDecl() && VD->hasGlobalStorage())
+      Diag(VD->getLocation(), diag::warn_global_destructor);
   }
 }
 
 /// AddCXXDirectInitializerToDecl - This action is called immediately after
 /// ActOnDeclarator, when a C++ direct initializer is present.
 /// e.g: "int x(1);"
-void Sema::AddCXXDirectInitializerToDecl(DeclPtrTy Dcl,
+void Sema::AddCXXDirectInitializerToDecl(Decl *RealDecl,
                                          SourceLocation LParenLoc,
                                          MultiExprArg Exprs,
                                          SourceLocation *CommaLocs,
                                          SourceLocation RParenLoc) {
   assert(Exprs.size() != 0 && Exprs.get() && "missing expressions");
-  Decl *RealDecl = Dcl.getAs<Decl>();
 
   // If there is no declaration, there was an error parsing it.  Just ignore
   // the initializer.
@@ -5402,9 +5541,6 @@ void Sema::AddCXXDirectInitializerToDecl(DeclPtrTy Dcl,
   // The form of initialization (using parentheses or '=') is generally
   // insignificant, but does matter when the entity being initialized has a
   // class type.
-  QualType DeclInitType = VDecl->getType();
-  if (const ArrayType *Array = Context.getAsArrayType(DeclInitType))
-    DeclInitType = Context.getBaseElementType(Array);
 
   if (!VDecl->getType()->isDependentType() &&
       RequireCompleteType(VDecl->getLocation(), VDecl->getType(),
@@ -5428,6 +5564,25 @@ void Sema::AddCXXDirectInitializerToDecl(DeclPtrTy Dcl,
     return;
   }
 
+  // C++ [class.static.data]p4
+  //   If a static data member is of const integral or const
+  //   enumeration type, its declaration in the class definition can
+  //   specify a constant-initializer which shall be an integral
+  //   constant expression (5.19). In that case, the member can appear
+  //   in integral constant expressions. The member shall still be
+  //   defined in a namespace scope if it is used in the program and the
+  //   namespace scope definition shall not contain an initializer.
+  //
+  // We already performed a redefinition check above, but for static
+  // data members we also need to check whether there was an in-class
+  // declaration with an initializer.
+  const VarDecl* PrevInit = 0;
+  if (VDecl->isStaticDataMember() && VDecl->getAnyInitializer(PrevInit)) {
+    Diag(VDecl->getLocation(), diag::err_redefinition) << VDecl->getDeclName();
+    Diag(PrevInit->getLocation(), diag::note_previous_definition);
+    return;
+  } 
+
   // If either the declaration has a dependent type or if any of the
   // expressions is type-dependent, we represent the initialization
   // via a ParenListExpr for later use during template instantiation.
@@ -5454,17 +5609,25 @@ void Sema::AddCXXDirectInitializerToDecl(DeclPtrTy Dcl,
                                        LParenLoc, RParenLoc);
   
   InitializationSequence InitSeq(*this, Entity, Kind, 
-                                 (Expr**)Exprs.get(), Exprs.size());
-  OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind, move(Exprs));
+                                 Exprs.get(), Exprs.size());
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, move(Exprs));
   if (Result.isInvalid()) {
     VDecl->setInvalidDecl();
     return;
   }
   
-  Result = MaybeCreateCXXExprWithTemporaries(move(Result));
+  Result = MaybeCreateCXXExprWithTemporaries(Result.get());
   VDecl->setInit(Result.takeAs<Expr>());
   VDecl->setCXXDirectInitializer(true);
 
+    if (!VDecl->isInvalidDecl() &&
+        !VDecl->getDeclContext()->isDependentContext() &&
+        VDecl->hasGlobalStorage() &&
+        !VDecl->getInit()->isConstantInitializer(Context,
+                                        VDecl->getType()->isReferenceType()))
+      Diag(VDecl->getLocation(), diag::warn_global_constructor)
+        << VDecl->getInit()->getSourceRange();
+
   if (const RecordType *Record = VDecl->getType()->getAs<RecordType>())
     FinalizeVarWithDestructor(VDecl, Record);
 }
@@ -5478,7 +5641,7 @@ bool
 Sema::CompleteConstructorCall(CXXConstructorDecl *Constructor,
                               MultiExprArg ArgsPtr,
                               SourceLocation Loc,                                    
-                     ASTOwningVector<&ActionBase::DeleteExpr> &ConvertedArgs) {
+                              ASTOwningVector<Expr*> &ConvertedArgs) {
   // FIXME: This duplicates a lot of code from Sema::ConvertArgumentsForCall.
   unsigned NumArgs = ArgsPtr.size();
   Expr **Args = (Expr **)ArgsPtr.get();
@@ -5508,7 +5671,7 @@ Sema::CompleteConstructorCall(CXXConstructorDecl *Constructor,
 static inline bool
 CheckOperatorNewDeleteDeclarationScope(Sema &SemaRef, 
                                        const FunctionDecl *FnDecl) {
-  const DeclContext *DC = FnDecl->getDeclContext()->getLookupContext();
+  const DeclContext *DC = FnDecl->getDeclContext()->getRedeclContext();
   if (isa<NamespaceDecl>(DC)) {
     return SemaRef.Diag(FnDecl->getLocation(), 
                         diag::err_operator_new_delete_declared_in_namespace)
@@ -5516,7 +5679,7 @@ CheckOperatorNewDeleteDeclarationScope(Sema &SemaRef,
   }
   
   if (isa<TranslationUnitDecl>(DC) && 
-      FnDecl->getStorageClass() == FunctionDecl::Static) {
+      FnDecl->getStorageClass() == SC_Static) {
     return SemaRef.Diag(FnDecl->getLocation(),
                         diag::err_operator_new_delete_declared_static)
       << FnDecl->getDeclName();
@@ -5622,18 +5785,6 @@ CheckOperatorDeleteDeclaration(Sema &SemaRef, const FunctionDecl *FnDecl) {
                                  diag::err_operator_delete_param_type))
     return true;
 
-  QualType FirstParamType = FnDecl->getParamDecl(0)->getType();
-  if (FirstParamType->isDependentType())
-    return SemaRef.Diag(FnDecl->getLocation(),
-                        diag::err_operator_delete_dependent_param_type)
-    << FnDecl->getDeclName() << SemaRef.Context.VoidPtrTy;
-
-  if (SemaRef.Context.getCanonicalType(FirstParamType) != 
-      SemaRef.Context.VoidPtrTy)
-    return SemaRef.Diag(FnDecl->getLocation(),
-                        diag::err_operator_delete_param_type)
-      << FnDecl->getDeclName() << SemaRef.Context.VoidPtrTy;
-  
   return false;
 }
 
@@ -5891,7 +6042,7 @@ FinishedParams:
 /// by Lang/StrSize. LBraceLoc, if valid, provides the location of
 /// the '{' brace. Otherwise, this linkage specification does not
 /// have any braces.
-Sema::DeclPtrTy Sema::ActOnStartLinkageSpecification(Scope *S,
+Decl *Sema::ActOnStartLinkageSpecification(Scope *S,
                                                      SourceLocation ExternLoc,
                                                      SourceLocation LangLoc,
                                                      llvm::StringRef Lang,
@@ -5903,7 +6054,7 @@ Sema::DeclPtrTy Sema::ActOnStartLinkageSpecification(Scope *S,
     Language = LinkageSpecDecl::lang_cxx;
   else {
     Diag(LangLoc, diag::err_bad_language);
-    return DeclPtrTy();
+    return 0;
   }
 
   // FIXME: Add all the various semantics of linkage specifications
@@ -5913,15 +6064,15 @@ Sema::DeclPtrTy Sema::ActOnStartLinkageSpecification(Scope *S,
                                                LBraceLoc.isValid());
   CurContext->addDecl(D);
   PushDeclContext(S, D);
-  return DeclPtrTy::make(D);
+  return D;
 }
 
-/// ActOnFinishLinkageSpecification - Completely the definition of
+/// ActOnFinishLinkageSpecification - Complete the definition of
 /// the C++ linkage specification LinkageSpec. If RBraceLoc is
 /// valid, it's the position of the closing '}' brace in a linkage
 /// specification that uses braces.
-Sema::DeclPtrTy Sema::ActOnFinishLinkageSpecification(Scope *S,
-                                                      DeclPtrTy LinkageSpec,
+Decl *Sema::ActOnFinishLinkageSpecification(Scope *S,
+                                                      Decl *LinkageSpec,
                                                       SourceLocation RBraceLoc) {
   if (LinkageSpec)
     PopDeclContext();
@@ -5983,9 +6134,30 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
                              AbstractVariableType))
     Invalid = true;
 
+  // Only the non-fragile NeXT runtime currently supports C++ catches
+  // of ObjC types, and no runtime supports catching ObjC types by value.
+  if (!Invalid && getLangOptions().ObjC1) {
+    QualType T = ExDeclType;
+    if (const ReferenceType *RT = T->getAs<ReferenceType>())
+      T = RT->getPointeeType();
+
+    if (T->isObjCObjectType()) {
+      Diag(Loc, diag::err_objc_object_catch);
+      Invalid = true;
+    } else if (T->isObjCObjectPointerType()) {
+      if (!getLangOptions().NeXTRuntime) {
+        Diag(Loc, diag::err_objc_pointer_cxx_catch_gnu);
+        Invalid = true;
+      } else if (!getLangOptions().ObjCNonFragileABI) {
+        Diag(Loc, diag::err_objc_pointer_cxx_catch_fragile);
+        Invalid = true;
+      }
+    }
+  }
+
   VarDecl *ExDecl = VarDecl::Create(Context, CurContext, Loc,
-                                    Name, ExDeclType, TInfo, VarDecl::None,
-                                    VarDecl::None);
+                                    Name, ExDeclType, TInfo, SC_None,
+                                    SC_None);
   ExDecl->setExceptionVariable(true);
   
   if (!Invalid) {
@@ -6005,8 +6177,8 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
       InitializationKind Kind = InitializationKind::CreateCopy(Loc, 
                                                                SourceLocation());
       InitializationSequence InitSeq(*this, Entity, Kind, &ExDeclRef, 1);
-      OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind, 
-                                    MultiExprArg(*this, (void**)&ExDeclRef, 1));
+      ExprResult Result = InitSeq.Perform(*this, Entity, Kind, 
+                                         MultiExprArg(*this, &ExDeclRef, 1));
       if (Result.isInvalid())
         Invalid = true;
       else 
@@ -6022,7 +6194,7 @@ VarDecl *Sema::BuildExceptionDeclaration(Scope *S, QualType ExDeclType,
 
 /// ActOnExceptionDeclarator - Parsed the exception-declarator in a C++ catch
 /// handler.
-Sema::DeclPtrTy Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
+Decl *Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
   QualType ExDeclType = TInfo->getType();
 
@@ -6033,7 +6205,7 @@ Sema::DeclPtrTy Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
                                              ForRedeclaration)) {
     // The scope should be freshly made just for us. There is just no way
     // it contains any previous declaration.
-    assert(!S->isDeclScope(DeclPtrTy::make(PrevDecl)));
+    assert(!S->isDeclScope(PrevDecl));
     if (PrevDecl->isTemplateParameter()) {
       // Maybe we will complain about the shadowed template parameter.
       DiagnoseTemplateParameterShadow(D.getIdentifierLoc(), PrevDecl);
@@ -6061,22 +6233,20 @@ Sema::DeclPtrTy Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) {
     CurContext->addDecl(ExDecl);
 
   ProcessDeclAttributes(S, ExDecl, D);
-  return DeclPtrTy::make(ExDecl);
+  return ExDecl;
 }
 
-Sema::DeclPtrTy Sema::ActOnStaticAssertDeclaration(SourceLocation AssertLoc,
-                                                   ExprArg assertexpr,
-                                                   ExprArg assertmessageexpr) {
-  Expr *AssertExpr = (Expr *)assertexpr.get();
-  StringLiteral *AssertMessage =
-    cast<StringLiteral>((Expr *)assertmessageexpr.get());
+Decl *Sema::ActOnStaticAssertDeclaration(SourceLocation AssertLoc,
+                                         Expr *AssertExpr,
+                                         Expr *AssertMessageExpr_) {
+  StringLiteral *AssertMessage = cast<StringLiteral>(AssertMessageExpr_);
 
   if (!AssertExpr->isTypeDependent() && !AssertExpr->isValueDependent()) {
     llvm::APSInt Value(32);
     if (!AssertExpr->isIntegerConstantExpr(Value, Context)) {
       Diag(AssertLoc, diag::err_static_assert_expression_is_not_constant) <<
         AssertExpr->getSourceRange();
-      return DeclPtrTy();
+      return 0;
     }
 
     if (Value == 0) {
@@ -6085,13 +6255,11 @@ Sema::DeclPtrTy Sema::ActOnStaticAssertDeclaration(SourceLocation AssertLoc,
     }
   }
 
-  assertexpr.release();
-  assertmessageexpr.release();
   Decl *Decl = StaticAssertDecl::Create(Context, CurContext, AssertLoc,
                                         AssertExpr, AssertMessage);
 
   CurContext->addDecl(Decl);
-  return DeclPtrTy::make(Decl);
+  return Decl;
 }
 
 /// \brief Perform semantic analysis of the given friend type declaration.
@@ -6167,7 +6335,7 @@ FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation FriendLoc,
 /// We permit this as a special case; if there are any template
 /// parameters present at all, require proper matching, i.e.
 ///   template <> template <class T> friend class A<int>::B;
-Sema::DeclPtrTy Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
+Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
                                           MultiTemplateParamsArg TempParams) {
   SourceLocation Loc = DS.getSourceRange().getBegin();
 
@@ -6181,7 +6349,7 @@ Sema::DeclPtrTy Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   TypeSourceInfo *TSI = GetTypeForDeclarator(TheDeclarator, S);
   QualType T = TSI->getType();
   if (TheDeclarator.isInvalidType())
-    return DeclPtrTy();
+    return 0;
 
   // This is definitely an error in C++98.  It's probably meant to
   // be forbidden in C++0x, too, but the specification is just
@@ -6200,7 +6368,7 @@ Sema::DeclPtrTy Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   if (TempParams.size() && !T->isElaboratedTypeSpecifier()) {
     Diag(Loc, diag::err_tagless_friend_type_template)
       << DS.getSourceRange();
-    return DeclPtrTy();
+    return 0;
   }
   
   // C++98 [class.friend]p1: A friend of a class is a function
@@ -6225,18 +6393,17 @@ Sema::DeclPtrTy Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
     D = CheckFriendTypeDecl(DS.getFriendSpecLoc(), TSI);
   
   if (!D)
-    return DeclPtrTy();
+    return 0;
   
   D->setAccess(AS_public);
   CurContext->addDecl(D);
 
-  return DeclPtrTy::make(D);
+  return D;
 }
 
-Sema::DeclPtrTy
-Sema::ActOnFriendFunctionDecl(Scope *S,
-                              Declarator &D,
-                              bool IsDefinition,
+Decl *Sema::ActOnFriendFunctionDecl(Scope *S,
+                                         Declarator &D,
+                                         bool IsDefinition,
                               MultiTemplateParamsArg TemplateParams) {
   const DeclSpec &DS = D.getDeclSpec();
 
@@ -6262,7 +6429,7 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
 
     // It might be worthwhile to try to recover by creating an
     // appropriate declaration.
-    return DeclPtrTy();
+    return 0;
   }
 
   // C++ [namespace.memdef]p3
@@ -6281,7 +6448,8 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
   //    namespace scope are not considered.
 
   CXXScopeSpec &ScopeQual = D.getCXXScopeSpec();
-  DeclarationName Name = GetNameForDeclarator(D);
+  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
   assert(Name);
 
   // The context we found the declaration in, or in which we should
@@ -6291,14 +6459,14 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
   // FIXME: handle local classes
 
   // Recover from invalid scope qualifiers as if they just weren't there.
-  LookupResult Previous(*this, Name, D.getIdentifierLoc(), LookupOrdinaryName,
+  LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
                         ForRedeclaration);
   if (!ScopeQual.isInvalid() && ScopeQual.isSet()) {
     DC = computeDeclContext(ScopeQual);
 
     // FIXME: handle dependent contexts
-    if (!DC) return DeclPtrTy();
-    if (RequireCompleteDeclContext(ScopeQual, DC)) return DeclPtrTy();
+    if (!DC) return 0;
+    if (RequireCompleteDeclContext(ScopeQual, DC)) return 0;
 
     LookupQualifiedName(Previous, DC);
 
@@ -6308,7 +6476,8 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
     LookupResult::Filter F = Previous.makeFilter();
     while (F.hasNext()) {
       NamedDecl *D = F.next();
-      if (!D->getDeclContext()->getLookupContext()->Equals(DC))
+      if (!DC->InEnclosingNamespaceSetOf(
+              D->getDeclContext()->getRedeclContext()))
         F.erase();
     }
     F.done();
@@ -6316,7 +6485,7 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
     if (Previous.empty()) {
       D.setInvalidType();
       Diag(Loc, diag::err_qualified_friend_not_found) << Name << T;
-      return DeclPtrTy();
+      return 0;
     }
 
     // C++ [class.friend]p1: A friend of a class is a function or
@@ -6368,7 +6537,7 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
       Diag(Loc, diag::err_introducing_special_friend) <<
         (D.getName().getKind() == UnqualifiedId::IK_ConstructorName ? 0 :
          D.getName().getKind() == UnqualifiedId::IK_DestructorName ? 1 : 2);
-      return DeclPtrTy();
+      return 0;
     }
   }
 
@@ -6377,7 +6546,7 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
                                           move(TemplateParams),
                                           IsDefinition,
                                           Redeclaration);
-  if (!ND) return DeclPtrTy();
+  if (!ND) return 0;
 
   assert(ND->getDeclContext() == DC);
   assert(ND->getLexicalDeclContext() == CurContext);
@@ -6389,7 +6558,7 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
   // Also update the scope-based lookup if the target context's
   // lookup context is in lexical scope.
   if (!CurContext->isDependentContext()) {
-    DC = DC->getLookupContext();
+    DC = DC->getRedeclContext();
     DC->makeDeclVisibleInContext(ND, /* Recoverable=*/ false);
     if (Scope *EnclosingScope = getScopeForDeclContext(S, DC))
       PushOnScopeChains(ND, EnclosingScope, /*AddToContext=*/ false);
@@ -6401,13 +6570,12 @@ Sema::ActOnFriendFunctionDecl(Scope *S,
   FrD->setAccess(AS_public);
   CurContext->addDecl(FrD);
 
-  return DeclPtrTy::make(ND);
+  return ND;
 }
 
-void Sema::SetDeclDeleted(DeclPtrTy dcl, SourceLocation DelLoc) {
-  AdjustDeclIfTemplate(dcl);
+void Sema::SetDeclDeleted(Decl *Dcl, SourceLocation DelLoc) {
+  AdjustDeclIfTemplate(Dcl);
 
-  Decl *Dcl = dcl.getAs<Decl>();
   FunctionDecl *Fn = dyn_cast<FunctionDecl>(Dcl);
   if (!Fn) {
     Diag(DelLoc, diag::err_deleted_non_function);
@@ -6575,9 +6743,8 @@ bool Sema::CheckPureMethod(CXXMethodDecl *Method, SourceRange InitRange) {
 /// After this method is called, according to [C++ 3.4.1p13], if 'Dcl' is a
 /// static data member of class X, names should be looked up in the scope of
 /// class X.
-void Sema::ActOnCXXEnterDeclInitializer(Scope *S, DeclPtrTy Dcl) {
+void Sema::ActOnCXXEnterDeclInitializer(Scope *S, Decl *D) {
   // If there is no declaration, there was an error parsing it.
-  Decl *D = Dcl.getAs<Decl>();
   if (D == 0) return;
 
   // We should only get called for declarations with scope specifiers, like:
@@ -6587,10 +6754,9 @@ void Sema::ActOnCXXEnterDeclInitializer(Scope *S, DeclPtrTy Dcl) {
 }
 
 /// ActOnCXXExitDeclInitializer - Invoked after we are finished parsing an
-/// initializer for the out-of-line declaration 'Dcl'.
-void Sema::ActOnCXXExitDeclInitializer(Scope *S, DeclPtrTy Dcl) {
+/// initializer for the out-of-line declaration 'D'.
+void Sema::ActOnCXXExitDeclInitializer(Scope *S, Decl *D) {
   // If there is no declaration, there was an error parsing it.
-  Decl *D = Dcl.getAs<Decl>();
   if (D == 0) return;
 
   assert(D->isOutOfLine());
@@ -6600,8 +6766,7 @@ void Sema::ActOnCXXExitDeclInitializer(Scope *S, DeclPtrTy Dcl) {
 /// ActOnCXXConditionDeclarationExpr - Parsed a condition declaration of a
 /// C++ if/switch/while/for statement.
 /// e.g: "if (int x = f()) {...}"
-Action::DeclResult
-Sema::ActOnCXXConditionDeclaration(Scope *S, Declarator &D) {
+DeclResult Sema::ActOnCXXConditionDeclaration(Scope *S, Declarator &D) {
   // C++ 6.4p2:
   // The declarator shall not specify a function or an array.
   // The type-specifier-seq shall not contain typedef and shall not declare a
@@ -6624,12 +6789,10 @@ Sema::ActOnCXXConditionDeclaration(Scope *S, Declarator &D) {
     Diag(OwnedTag->getLocation(), diag::err_type_defined_in_condition);
   }
   
-  DeclPtrTy Dcl = ActOnDeclarator(S, D);
+  Decl *Dcl = ActOnDeclarator(S, D);
   if (!Dcl)
     return DeclResult();
 
-  VarDecl *VD = cast<VarDecl>(Dcl.getAs<Decl>());
-  VD->setDeclaredInCondition(true);
   return Dcl;
 }
 
@@ -6775,8 +6938,6 @@ void Sema::MarkVirtualMembersReferenced(SourceLocation Loc,
            e = RD->bases_end(); i != e; ++i) {
     const CXXRecordDecl *Base =
         cast<CXXRecordDecl>(i->getType()->getAs<RecordType>()->getDecl());
-    if (i->isVirtual())
-      continue;
     if (Base->getNumVBases() == 0)
       continue;
     MarkVirtualMembersReferenced(Loc, Base);
@@ -6788,7 +6949,7 @@ void Sema::MarkVirtualMembersReferenced(SourceLocation Loc,
 void Sema::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
   if (!getLangOptions().CPlusPlus)
     return;
-  if (const ObjCInterfaceDecl *OID = ObjCImplementation->getClassInterface()) {
+  if (ObjCInterfaceDecl *OID = ObjCImplementation->getClassInterface()) {
     llvm::SmallVector<ObjCIvarDecl*, 8> ivars;
     CollectIvarsToConstructOrDestruct(OID, ivars);
     if (ivars.empty())
@@ -6805,10 +6966,9 @@ void Sema::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
         InitializationKind::CreateDefault(ObjCImplementation->getLocation());
       
       InitializationSequence InitSeq(*this, InitEntity, InitKind, 0, 0);
-      Sema::OwningExprResult MemberInit = 
-        InitSeq.Perform(*this, InitEntity, InitKind, 
-                        Sema::MultiExprArg(*this, 0, 0));
-      MemberInit = MaybeCreateCXXExprWithTemporaries(move(MemberInit));
+      ExprResult MemberInit = 
+        InitSeq.Perform(*this, InitEntity, InitKind, MultiExprArg());
+      MemberInit = MaybeCreateCXXExprWithTemporaries(MemberInit.get());
       // Note, MemberInit could actually come back empty if no initialization 
       // is required (e.g., because it would call a trivial default constructor)
       if (!MemberInit.get() || MemberInit.isInvalid())
diff --git a/lib/Sema/SemaDeclObjC.cpp b/lib/Sema/SemaDeclObjC.cpp
index 21aeb59a0886..a6902a3e391c 100644
--- a/lib/Sema/SemaDeclObjC.cpp
+++ b/lib/Sema/SemaDeclObjC.cpp
@@ -11,20 +11,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ExternalSemaSource.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/Parse/DeclSpec.h"
+#include "clang/Sema/DeclSpec.h"
+#include "llvm/ADT/DenseSet.h"
+
 using namespace clang;
 
 /// ActOnStartOfObjCMethodDef - This routine sets up parameters; invisible
 /// and user declared, in the method definition's AST.
-void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, DeclPtrTy D) {
+void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
   assert(getCurMethodDecl() == 0 && "Method parsing confused");
-  ObjCMethodDecl *MDecl = dyn_cast_or_null<ObjCMethodDecl>(D.getAs<Decl>());
+  ObjCMethodDecl *MDecl = dyn_cast_or_null<ObjCMethodDecl>(D);
 
   // If we don't have a valid method decl, simply return.
   if (!MDecl)
@@ -32,10 +36,10 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, DeclPtrTy D) {
 
   // Allow the rest of sema to find private method decl implementations.
   if (MDecl->isInstanceMethod())
-    AddInstanceMethodToGlobalPool(MDecl);
+    AddInstanceMethodToGlobalPool(MDecl, true);
   else
-    AddFactoryMethodToGlobalPool(MDecl);
-
+    AddFactoryMethodToGlobalPool(MDecl, true);
+  
   // Allow all of Sema to see that we are entering a method definition.
   PushDeclContext(FnBodyScope, MDecl);
   PushFunctionScope();
@@ -56,11 +60,11 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, DeclPtrTy D) {
       PushOnScopeChains(*PI, FnBodyScope);
 }
 
-Sema::DeclPtrTy Sema::
+Decl *Sema::
 ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
                          IdentifierInfo *ClassName, SourceLocation ClassLoc,
                          IdentifierInfo *SuperName, SourceLocation SuperLoc,
-                         const DeclPtrTy *ProtoRefs, unsigned NumProtoRefs,
+                         Decl * const *ProtoRefs, unsigned NumProtoRefs,
                          const SourceLocation *ProtoLocs, 
                          SourceLocation EndProtoLoc, AttributeList *AttrList) {
   assert(ClassName && "Missing class identifier");
@@ -84,11 +88,14 @@ ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
 
       // Return the previous class interface.
       // FIXME: don't leak the objects passed in!
-      return DeclPtrTy::make(IDecl);
+      return IDecl;
     } else {
       IDecl->setLocation(AtInterfaceLoc);
       IDecl->setForwardDecl(false);
       IDecl->setClassLoc(ClassLoc);
+      // If the forward decl was in a PCH, we need to write it again in a
+      // dependent AST file.
+      IDecl->setChangedSinceDeserialization(true);
       
       // Since this ObjCInterfaceDecl was created by a forward declaration,
       // we now add it to the DeclContext since it wasn't added before
@@ -176,7 +183,7 @@ ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
     IDecl->setLocEnd(ClassLoc);
   }
 
-  /// Check then save referenced protocols.
+  // Check then save referenced protocols.
   if (NumProtoRefs) {
     IDecl->setProtocolList((ObjCProtocolDecl**)ProtoRefs, NumProtoRefs,
                            ProtoLocs, Context);
@@ -184,16 +191,16 @@ ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
   }
 
   CheckObjCDeclScope(IDecl);
-  return DeclPtrTy::make(IDecl);
+  return IDecl;
 }
 
 /// ActOnCompatiblityAlias - this action is called after complete parsing of
 /// @compatibility_alias declaration. It sets up the alias relationships.
-Sema::DeclPtrTy Sema::ActOnCompatiblityAlias(SourceLocation AtLoc,
-                                             IdentifierInfo *AliasName,
-                                             SourceLocation AliasLocation,
-                                             IdentifierInfo *ClassName,
-                                             SourceLocation ClassLocation) {
+Decl *Sema::ActOnCompatiblityAlias(SourceLocation AtLoc,
+                                        IdentifierInfo *AliasName,
+                                        SourceLocation AliasLocation,
+                                        IdentifierInfo *ClassName,
+                                        SourceLocation ClassLocation) {
   // Look for previous declaration of alias name
   NamedDecl *ADecl = LookupSingleName(TUScope, AliasName, AliasLocation,
                                       LookupOrdinaryName, ForRedeclaration);
@@ -203,7 +210,7 @@ Sema::DeclPtrTy Sema::ActOnCompatiblityAlias(SourceLocation AtLoc,
     else
       Diag(AliasLocation, diag::err_conflicting_aliasing_type) << AliasName;
     Diag(ADecl->getLocation(), diag::note_previous_declaration);
-    return DeclPtrTy();
+    return 0;
   }
   // Check for class declaration
   NamedDecl *CDeclU = LookupSingleName(TUScope, ClassName, ClassLocation,
@@ -223,7 +230,7 @@ Sema::DeclPtrTy Sema::ActOnCompatiblityAlias(SourceLocation AtLoc,
     Diag(ClassLocation, diag::warn_undef_interface) << ClassName;
     if (CDeclU)
       Diag(CDeclU->getLocation(), diag::note_previous_declaration);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Everything checked out, instantiate a new alias declaration AST.
@@ -233,7 +240,7 @@ Sema::DeclPtrTy Sema::ActOnCompatiblityAlias(SourceLocation AtLoc,
   if (!CheckObjCDeclScope(AliasDecl))
     PushOnScopeChains(AliasDecl, TUScope);
 
-  return DeclPtrTy::make(AliasDecl);
+  return AliasDecl;
 }
 
 void Sema::CheckForwardProtocolDeclarationForCircularDependency(
@@ -255,11 +262,11 @@ void Sema::CheckForwardProtocolDeclarationForCircularDependency(
   }
 }
 
-Sema::DeclPtrTy
+Decl *
 Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
                                   IdentifierInfo *ProtocolName,
                                   SourceLocation ProtocolLoc,
-                                  const DeclPtrTy *ProtoRefs,
+                                  Decl * const *ProtoRefs,
                                   unsigned NumProtoRefs,
                                   const SourceLocation *ProtoLocs,
                                   SourceLocation EndProtoLoc,
@@ -274,17 +281,19 @@ Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
       Diag(PDecl->getLocation(), diag::note_previous_definition);
       // Just return the protocol we already had.
       // FIXME: don't leak the objects passed in!
-      return DeclPtrTy::make(PDecl);
+      return PDecl;
     }
     ObjCList<ObjCProtocolDecl> PList;
     PList.set((ObjCProtocolDecl *const*)ProtoRefs, NumProtoRefs, Context);
     CheckForwardProtocolDeclarationForCircularDependency(
       ProtocolName, ProtocolLoc, PDecl->getLocation(), PList);
-    PList.Destroy(Context);
 
     // Make sure the cached decl gets a valid start location.
     PDecl->setLocation(AtProtoInterfaceLoc);
     PDecl->setForwardDecl(false);
+    CurContext->addDecl(PDecl);
+    // Repeat in dependent AST files.
+    PDecl->setChangedSinceDeserialization(true);
   } else {
     PDecl = ObjCProtocolDecl::Create(Context, CurContext,
                                      AtProtoInterfaceLoc,ProtocolName);
@@ -301,7 +310,7 @@ Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
   }
 
   CheckObjCDeclScope(PDecl);
-  return DeclPtrTy::make(PDecl);
+  return PDecl;
 }
 
 /// FindProtocolDeclaration - This routine looks up protocols and
@@ -311,7 +320,7 @@ void
 Sema::FindProtocolDeclaration(bool WarnOnDeclarations,
                               const IdentifierLocPair *ProtocolId,
                               unsigned NumProtocols,
-                              llvm::SmallVectorImpl<DeclPtrTy> &Protocols) {
+                              llvm::SmallVectorImpl<Decl *> &Protocols) {
   for (unsigned i = 0; i != NumProtocols; ++i) {
     ObjCProtocolDecl *PDecl = LookupProtocol(ProtocolId[i].first,
                                              ProtocolId[i].second);
@@ -340,7 +349,7 @@ Sema::FindProtocolDeclaration(bool WarnOnDeclarations,
     if (WarnOnDeclarations && PDecl->isForwardDecl())
       Diag(ProtocolId[i].second, diag::warn_undef_protocolref)
         << ProtocolId[i].first;
-    Protocols.push_back(DeclPtrTy::make(PDecl));
+    Protocols.push_back(PDecl);
   }
 }
 
@@ -374,7 +383,7 @@ void Sema::DiagnoseClassExtensionDupMethods(ObjCCategoryDecl *CAT,
 }
 
 /// ActOnForwardProtocolDeclaration - Handle @protocol foo;
-Action::DeclPtrTy
+Decl *
 Sema::ActOnForwardProtocolDeclaration(SourceLocation AtProtocolLoc,
                                       const IdentifierLocPair *IdentList,
                                       unsigned NumElts,
@@ -385,13 +394,18 @@ Sema::ActOnForwardProtocolDeclaration(SourceLocation AtProtocolLoc,
   for (unsigned i = 0; i != NumElts; ++i) {
     IdentifierInfo *Ident = IdentList[i].first;
     ObjCProtocolDecl *PDecl = LookupProtocol(Ident, IdentList[i].second);
+    bool isNew = false;
     if (PDecl == 0) { // Not already seen?
       PDecl = ObjCProtocolDecl::Create(Context, CurContext,
                                        IdentList[i].second, Ident);
-      PushOnScopeChains(PDecl, TUScope);
+      PushOnScopeChains(PDecl, TUScope, false);
+      isNew = true;
     }
-    if (attrList)
+    if (attrList) {
       ProcessDeclAttributeList(TUScope, PDecl, attrList);
+      if (!isNew)
+        PDecl->setChangedSinceDeserialization(true);
+    }
     Protocols.push_back(PDecl);
     ProtoLocs.push_back(IdentList[i].second);
   }
@@ -402,15 +416,15 @@ Sema::ActOnForwardProtocolDeclaration(SourceLocation AtProtocolLoc,
                                     ProtoLocs.data());
   CurContext->addDecl(PDecl);
   CheckObjCDeclScope(PDecl);
-  return DeclPtrTy::make(PDecl);
+  return PDecl;
 }
 
-Sema::DeclPtrTy Sema::
+Decl *Sema::
 ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
                             IdentifierInfo *ClassName, SourceLocation ClassLoc,
                             IdentifierInfo *CategoryName,
                             SourceLocation CategoryLoc,
-                            const DeclPtrTy *ProtoRefs,
+                            Decl * const *ProtoRefs,
                             unsigned NumProtoRefs,
                             const SourceLocation *ProtoLocs,
                             SourceLocation EndProtoLoc) {
@@ -426,7 +440,7 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
                                      ClassLoc, CategoryLoc, CategoryName);
     CDecl->setInvalidDecl();
     Diag(ClassLoc, diag::err_undef_interface) << ClassName;
-    return DeclPtrTy::make(CDecl);
+    return CDecl;
   }
 
   if (!CategoryName && IDecl->getImplementation()) {
@@ -471,18 +485,17 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
     // Protocols in the class extension belong to the class.
     if (CDecl->IsClassExtension())
      IDecl->mergeClassExtensionProtocolList((ObjCProtocolDecl**)ProtoRefs, 
-                                            NumProtoRefs, ProtoLocs,
-                                            Context); 
+                                            NumProtoRefs, Context); 
   }
 
   CheckObjCDeclScope(CDecl);
-  return DeclPtrTy::make(CDecl);
+  return CDecl;
 }
 
 /// ActOnStartCategoryImplementation - Perform semantic checks on the
 /// category implementation declaration and build an ObjCCategoryImplDecl
 /// object.
-Sema::DeclPtrTy Sema::ActOnStartCategoryImplementation(
+Decl *Sema::ActOnStartCategoryImplementation(
                       SourceLocation AtCatImplLoc,
                       IdentifierInfo *ClassName, SourceLocation ClassLoc,
                       IdentifierInfo *CatName, SourceLocation CatLoc) {
@@ -523,10 +536,10 @@ Sema::DeclPtrTy Sema::ActOnStartCategoryImplementation(
   }
 
   CheckObjCDeclScope(CDecl);
-  return DeclPtrTy::make(CDecl);
+  return CDecl;
 }
 
-Sema::DeclPtrTy Sema::ActOnStartClassImplementation(
+Decl *Sema::ActOnStartClassImplementation(
                       SourceLocation AtClassImplLoc,
                       IdentifierInfo *ClassName, SourceLocation ClassLoc,
                       IdentifierInfo *SuperClassname,
@@ -617,7 +630,7 @@ Sema::DeclPtrTy Sema::ActOnStartClassImplementation(
                                    IDecl, SDecl);
 
   if (CheckObjCDeclScope(IMPDecl))
-    return DeclPtrTy::make(IMPDecl);
+    return IMPDecl;
 
   // Check that there is no duplicate implementation of this class.
   if (IDecl->getImplementation()) {
@@ -629,7 +642,7 @@ Sema::DeclPtrTy Sema::ActOnStartClassImplementation(
     IDecl->setImplementation(IMPDecl);
     PushOnScopeChains(IMPDecl, TUScope);
   }
-  return DeclPtrTy::make(IMPDecl);
+  return IMPDecl;
 }
 
 void Sema::CheckImplementationIvars(ObjCImplementationDecl *ImpDecl,
@@ -910,8 +923,9 @@ void Sema::MatchAllMethodDeclarations(const llvm::DenseSet<Selector> &InsMap,
   }
   if (ObjCInterfaceDecl *I = dyn_cast<ObjCInterfaceDecl> (CDecl)) {
     // Check for any implementation of a methods declared in protocol.
-    for (ObjCInterfaceDecl::protocol_iterator PI = I->protocol_begin(),
-         E = I->protocol_end(); PI != E; ++PI)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+          PI = I->all_referenced_protocol_begin(),
+          E = I->all_referenced_protocol_end(); PI != E; ++PI)
       MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
                                  IMPDecl,
                                  (*PI), IncompleteImpl, false);
@@ -957,8 +971,9 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
   // implemented in the implementation class.
 
   if (ObjCInterfaceDecl *I = dyn_cast<ObjCInterfaceDecl> (CDecl)) {
-    for (ObjCInterfaceDecl::protocol_iterator PI = I->protocol_begin(),
-         E = I->protocol_end(); PI != E; ++PI)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+          PI = I->all_referenced_protocol_begin(),
+          E = I->all_referenced_protocol_end(); PI != E; ++PI)
       CheckProtocolMethodDefs(IMPDecl->getLocation(), *PI, IncompleteImpl,
                               InsMap, ClsMap, I);
     // Check class extensions (unnamed categories)
@@ -992,7 +1007,7 @@ void Sema::ImplMethodsVsClassMethods(Scope *S, ObjCImplDecl* IMPDecl,
 }
 
 /// ActOnForwardClassDeclaration -
-Action::DeclPtrTy
+Decl *
 Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
                                    IdentifierInfo **IdentList,
                                    SourceLocation *IdentLocs,
@@ -1053,7 +1068,7 @@ Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
                                                Interfaces.size());
   CurContext->addDecl(CDecl);
   CheckObjCDeclScope(CDecl);
-  return DeclPtrTy::make(CDecl);
+  return CDecl;
 }
 
 
@@ -1062,13 +1077,14 @@ Sema::ActOnForwardClassDeclaration(SourceLocation AtClassLoc,
 /// TODO: Handle protocol list; such as id<p1,p2> in type comparisons
 bool Sema::MatchTwoMethodDeclarations(const ObjCMethodDecl *Method,
                                       const ObjCMethodDecl *PrevMethod,
-                                      bool matchBasedOnSizeAndAlignment) {
+                                      bool matchBasedOnSizeAndAlignment,
+                                      bool matchBasedOnStrictEqulity) {
   QualType T1 = Context.getCanonicalType(Method->getResultType());
   QualType T2 = Context.getCanonicalType(PrevMethod->getResultType());
 
   if (T1 != T2) {
     // The result types are different.
-    if (!matchBasedOnSizeAndAlignment)
+    if (!matchBasedOnSizeAndAlignment || matchBasedOnStrictEqulity)
       return false;
     // Incomplete types don't have a size and alignment.
     if (T1->isIncompleteType() || T2->isIncompleteType())
@@ -1088,7 +1104,7 @@ bool Sema::MatchTwoMethodDeclarations(const ObjCMethodDecl *Method,
     T2 = Context.getCanonicalType((*PrevI)->getType());
     if (T1 != T2) {
       // The result types are different.
-      if (!matchBasedOnSizeAndAlignment)
+      if (!matchBasedOnSizeAndAlignment || matchBasedOnStrictEqulity)
         return false;
       // Incomplete types don't have a size and alignment.
       if (T1->isIncompleteType() || T2->isIncompleteType())
@@ -1101,47 +1117,34 @@ bool Sema::MatchTwoMethodDeclarations(const ObjCMethodDecl *Method,
   return true;
 }
 
-/// \brief Read the contents of the instance and factory method pools
-/// for a given selector from external storage.
+/// \brief Read the contents of the method pool for a given selector from
+/// external storage.
 ///
-/// This routine should only be called once, when neither the instance
-/// nor the factory method pool has an entry for this selector.
-Sema::MethodPool::iterator Sema::ReadMethodPool(Selector Sel,
-                                                bool isInstance) {
+/// This routine should only be called once, when the method pool has no entry
+/// for this selector.
+Sema::GlobalMethodPool::iterator Sema::ReadMethodPool(Selector Sel) {
   assert(ExternalSource && "We need an external AST source");
-  assert(InstanceMethodPool.find(Sel) == InstanceMethodPool.end() &&
-         "Selector data already loaded into the instance method pool");
-  assert(FactoryMethodPool.find(Sel) == FactoryMethodPool.end() &&
-         "Selector data already loaded into the factory method pool");
+  assert(MethodPool.find(Sel) == MethodPool.end() &&
+         "Selector data already loaded into the method pool");
 
   // Read the method list from the external source.
-  std::pair<ObjCMethodList, ObjCMethodList> Methods
-    = ExternalSource->ReadMethodPool(Sel);
+  GlobalMethods Methods = ExternalSource->ReadMethodPool(Sel);
 
-  if (isInstance) {
-    if (Methods.second.Method)
-      FactoryMethodPool[Sel] = Methods.second;
-    return InstanceMethodPool.insert(std::make_pair(Sel, Methods.first)).first;
-  }
-
-  if (Methods.first.Method)
-    InstanceMethodPool[Sel] = Methods.first;
-
-  return FactoryMethodPool.insert(std::make_pair(Sel, Methods.second)).first;
+  return MethodPool.insert(std::make_pair(Sel, Methods)).first;
 }
 
-void Sema::AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method) {
-  llvm::DenseMap<Selector, ObjCMethodList>::iterator Pos
-    = InstanceMethodPool.find(Method->getSelector());
-  if (Pos == InstanceMethodPool.end()) {
-    if (ExternalSource && !FactoryMethodPool.count(Method->getSelector()))
-      Pos = ReadMethodPool(Method->getSelector(), /*isInstance=*/true);
+void Sema::AddMethodToGlobalPool(ObjCMethodDecl *Method, bool impl,
+                                 bool instance) {
+  GlobalMethodPool::iterator Pos = MethodPool.find(Method->getSelector());
+  if (Pos == MethodPool.end()) {
+    if (ExternalSource)
+      Pos = ReadMethodPool(Method->getSelector());
     else
-      Pos = InstanceMethodPool.insert(std::make_pair(Method->getSelector(),
-                                                     ObjCMethodList())).first;
+      Pos = MethodPool.insert(std::make_pair(Method->getSelector(),
+                                             GlobalMethods())).first;
   }
-
-  ObjCMethodList &Entry = Pos->second;
+  Method->setDefined(impl);
+  ObjCMethodList &Entry = instance ? Pos->second.first : Pos->second.second;
   if (Entry.Method == 0) {
     // Haven't seen a method with this selector name yet - add it.
     Entry.Method = Method;
@@ -1152,8 +1155,10 @@ void Sema::AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method) {
   // We've seen a method with this name, see if we have already seen this type
   // signature.
   for (ObjCMethodList *List = &Entry; List; List = List->Next)
-    if (MatchTwoMethodDeclarations(Method, List->Method))
+    if (MatchTwoMethodDeclarations(Method, List->Method)) {
+      List->Method->setDefined(impl);
       return;
+    }
 
   // We have a new signature for an existing method - add it.
   // This is extremely rare. Only 1% of Cocoa selectors are "overloaded".
@@ -1161,102 +1166,65 @@ void Sema::AddInstanceMethodToGlobalPool(ObjCMethodDecl *Method) {
   Entry.Next = new (Mem) ObjCMethodList(Method, Entry.Next);
 }
 
-// FIXME: Finish implementing -Wno-strict-selector-match.
-ObjCMethodDecl *Sema::LookupInstanceMethodInGlobalPool(Selector Sel,
-                                                       SourceRange R,
-                                                       bool warn) {
-  llvm::DenseMap<Selector, ObjCMethodList>::iterator Pos
-    = InstanceMethodPool.find(Sel);
-  if (Pos == InstanceMethodPool.end()) {
-    if (ExternalSource && !FactoryMethodPool.count(Sel))
-      Pos = ReadMethodPool(Sel, /*isInstance=*/true);
+ObjCMethodDecl *Sema::LookupMethodInGlobalPool(Selector Sel, SourceRange R,
+                                               bool receiverIdOrClass,
+                                               bool warn, bool instance) {
+  GlobalMethodPool::iterator Pos = MethodPool.find(Sel);
+  if (Pos == MethodPool.end()) {
+    if (ExternalSource)
+      Pos = ReadMethodPool(Sel);
     else
       return 0;
   }
 
-  ObjCMethodList &MethList = Pos->second;
-  bool issueWarning = false;
+  ObjCMethodList &MethList = instance ? Pos->second.first : Pos->second.second;
 
-  if (MethList.Method && MethList.Next) {
-    for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next)
-      // This checks if the methods differ by size & alignment.
-      if (!MatchTwoMethodDeclarations(MethList.Method, Next->Method, true))
-        issueWarning = warn;
-  }
-  if (issueWarning && (MethList.Method && MethList.Next)) {
-    Diag(R.getBegin(), diag::warn_multiple_method_decl) << Sel << R;
-    Diag(MethList.Method->getLocStart(), diag::note_using)
-      << MethList.Method->getSourceRange();
-    for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next)
-      Diag(Next->Method->getLocStart(), diag::note_also_found)
-        << Next->Method->getSourceRange();
-  }
-  return MethList.Method;
-}
+  bool strictSelectorMatch = receiverIdOrClass && warn &&
+    (Diags.getDiagnosticLevel(diag::warn_strict_multiple_method_decl) != 
+      Diagnostic::Ignored);
+  if (warn && MethList.Method && MethList.Next) {
+    bool issueWarning = false;
+    if (strictSelectorMatch)
+      for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next) {
+        // This checks if the methods differ in type mismatch.
+        if (!MatchTwoMethodDeclarations(MethList.Method, Next->Method, false, true))
+          issueWarning = true;
+      }
 
-void Sema::AddFactoryMethodToGlobalPool(ObjCMethodDecl *Method) {
-  llvm::DenseMap<Selector, ObjCMethodList>::iterator Pos
-    = FactoryMethodPool.find(Method->getSelector());
-  if (Pos == FactoryMethodPool.end()) {
-    if (ExternalSource && !InstanceMethodPool.count(Method->getSelector()))
-      Pos = ReadMethodPool(Method->getSelector(), /*isInstance=*/false);
-    else
-      Pos = FactoryMethodPool.insert(std::make_pair(Method->getSelector(),
-                                                    ObjCMethodList())).first;
-  }
+    if (!issueWarning)
+      for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next) {
+        // This checks if the methods differ by size & alignment.
+        if (!MatchTwoMethodDeclarations(MethList.Method, Next->Method, true))
+          issueWarning = true;
+      }
 
-  ObjCMethodList &FirstMethod = Pos->second;
-  if (!FirstMethod.Method) {
-    // Haven't seen a method with this selector name yet - add it.
-    FirstMethod.Method = Method;
-    FirstMethod.Next = 0;
-  } else {
-    // We've seen a method with this name, now check the type signature(s).
-    bool match = MatchTwoMethodDeclarations(Method, FirstMethod.Method);
-
-    for (ObjCMethodList *Next = FirstMethod.Next; !match && Next;
-         Next = Next->Next)
-      match = MatchTwoMethodDeclarations(Method, Next->Method);
-
-    if (!match) {
-      // We have a new signature for an existing method - add it.
-      // This is extremely rare. Only 1% of Cocoa selectors are "overloaded".
-      ObjCMethodList *Mem = BumpAlloc.Allocate<ObjCMethodList>();
-      ObjCMethodList *OMI = new (Mem) ObjCMethodList(Method, FirstMethod.Next);
-      FirstMethod.Next = OMI;
+    if (issueWarning) {
+      if (strictSelectorMatch)
+        Diag(R.getBegin(), diag::warn_strict_multiple_method_decl) << Sel << R;
+      else
+        Diag(R.getBegin(), diag::warn_multiple_method_decl) << Sel << R;
+      Diag(MethList.Method->getLocStart(), diag::note_using)
+        << MethList.Method->getSourceRange();
+      for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next)
+        Diag(Next->Method->getLocStart(), diag::note_also_found)
+          << Next->Method->getSourceRange();
     }
   }
+  return MethList.Method;
 }
 
-ObjCMethodDecl *Sema::LookupFactoryMethodInGlobalPool(Selector Sel,
-                                                      SourceRange R) {
-  llvm::DenseMap<Selector, ObjCMethodList>::iterator Pos
-    = FactoryMethodPool.find(Sel);
-  if (Pos == FactoryMethodPool.end()) {
-    if (ExternalSource && !InstanceMethodPool.count(Sel))
-      Pos = ReadMethodPool(Sel, /*isInstance=*/false);
-    else
-      return 0;
-  }
+ObjCMethodDecl *Sema::LookupImplementedMethodInGlobalPool(Selector Sel) {
+  GlobalMethodPool::iterator Pos = MethodPool.find(Sel);
+  if (Pos == MethodPool.end())
+    return 0;
 
-  ObjCMethodList &MethList = Pos->second;
-  bool issueWarning = false;
+  GlobalMethods &Methods = Pos->second;
 
-  if (MethList.Method && MethList.Next) {
-    for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next)
-      // This checks if the methods differ by size & alignment.
-      if (!MatchTwoMethodDeclarations(MethList.Method, Next->Method, true))
-        issueWarning = true;
-  }
-  if (issueWarning && (MethList.Method && MethList.Next)) {
-    Diag(R.getBegin(), diag::warn_multiple_method_decl) << Sel << R;
-    Diag(MethList.Method->getLocStart(), diag::note_using)
-      << MethList.Method->getSourceRange();
-    for (ObjCMethodList *Next = MethList.Next; Next; Next = Next->Next)
-      Diag(Next->Method->getLocStart(), diag::note_also_found)
-        << Next->Method->getSourceRange();
-  }
-  return MethList.Method;
+  if (Methods.first.Method && Methods.first.Method->isDefined())
+    return Methods.first.Method;
+  if (Methods.second.Method && Methods.second.Method->isDefined())
+    return Methods.second.Method;
+  return 0;
 }
 
 /// CompareMethodParamsInBaseAndSuper - This routine compares methods with
@@ -1322,12 +1290,10 @@ void Sema::DiagnoseDuplicateIvars(ObjCInterfaceDecl *ID,
 // Note: For class/category implemenations, allMethods/allProperties is
 // always null.
 void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
-                      DeclPtrTy classDecl,
-                      DeclPtrTy *allMethods, unsigned allNum,
-                      DeclPtrTy *allProperties, unsigned pNum,
+                      Decl *ClassDecl,
+                      Decl **allMethods, unsigned allNum,
+                      Decl **allProperties, unsigned pNum,
                       DeclGroupPtrTy *allTUVars, unsigned tuvNum) {
-  Decl *ClassDecl = classDecl.getAs<Decl>();
-
   // FIXME: If we don't have a ClassDecl, we have an error. We should consider
   // always passing in a decl. If the decl has an error, isInvalidDecl()
   // should be true.
@@ -1356,7 +1322,7 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
 
   for (unsigned i = 0; i < allNum; i++ ) {
     ObjCMethodDecl *Method =
-      cast_or_null<ObjCMethodDecl>(allMethods[i].getAs<Decl>());
+      cast_or_null<ObjCMethodDecl>(allMethods[i]);
 
     if (!Method) continue;  // Already issued a diagnostic.
     if (Method->isInstanceMethod()) {
@@ -1403,14 +1369,14 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
     // Compares properties declared in this class to those of its
     // super class.
     ComparePropertiesInBaseAndSuper(I);
-    CompareProperties(I, DeclPtrTy::make(I));
+    CompareProperties(I, I);
   } else if (ObjCCategoryDecl *C = dyn_cast<ObjCCategoryDecl>(ClassDecl)) {
     // Categories are used to extend the class by declaring new methods.
     // By the same token, they are also used to add new properties. No
     // need to compare the added property to those in the class.
 
     // Compare protocol properties with those in category
-    CompareProperties(C, DeclPtrTy::make(C));
+    CompareProperties(C, C);
     if (C->IsClassExtension())
       DiagnoseClassExtensionDupMethods(C, C->getClassInterface());
   }
@@ -1432,6 +1398,7 @@ void Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
         DefaultSynthesizeProperties(S, IC, IDecl);
       ImplMethodsVsClassMethods(S, IC, IDecl);
       AtomicPropertySetterGetterRules(IC, IDecl);
+  
       if (LangOpts.ObjCNonFragileABI2)
         while (IDecl->getSuperClass()) {
           DiagnoseDuplicateIvars(IDecl, IDecl->getSuperClass());
@@ -1491,19 +1458,20 @@ CvtQTToAstBitMask(ObjCDeclSpec::ObjCDeclQualifier PQTVal) {
 }
 
 static inline
-bool containsInvalidMethodImplAttribute(const AttributeList *A) {
+bool containsInvalidMethodImplAttribute(const AttrVec &A) {
   // The 'ibaction' attribute is allowed on method definitions because of
   // how the IBAction macro is used on both method declarations and definitions.
   // If the method definitions contains any other attributes, return true.
-  while (A && A->getKind() == AttributeList::AT_IBAction)
-    A = A->getNext();
-  return A != NULL;
+  for (AttrVec::const_iterator i = A.begin(), e = A.end(); i != e; ++i)
+    if ((*i)->getKind() != attr::IBAction)
+      return true;
+  return false;
 }
 
-Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
+Decl *Sema::ActOnMethodDeclaration(
     SourceLocation MethodLoc, SourceLocation EndLoc,
-    tok::TokenKind MethodType, DeclPtrTy classDecl,
-    ObjCDeclSpec &ReturnQT, TypeTy *ReturnType,
+    tok::TokenKind MethodType, Decl *ClassDecl,
+    ObjCDeclSpec &ReturnQT, ParsedType ReturnType,
     Selector Sel,
     // optional arguments. The number of types/arguments is obtained
     // from the Sel.getNumArgs().
@@ -1511,13 +1479,11 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
     DeclaratorChunk::ParamInfo *CParamInfo, unsigned CNumArgs, // c-style args
     AttributeList *AttrList, tok::ObjCKeywordKind MethodDeclKind,
     bool isVariadic) {
-  Decl *ClassDecl = classDecl.getAs<Decl>();
-
   // Make sure we can establish a context for the method.
   if (!ClassDecl) {
     Diag(MethodLoc, diag::error_missing_method_context);
-    getLabelMap().clear();
-    return DeclPtrTy();
+    getCurFunction()->LabelMap.clear();
+    return 0;
   }
   QualType resultDeclType;
 
@@ -1530,7 +1496,7 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
     if (resultDeclType->isObjCObjectType()) {
       Diag(MethodLoc, diag::err_object_cannot_be_passed_returned_by_value)
         << 0 << resultDeclType;
-      return DeclPtrTy();
+      return 0;
     }
   } else // get the type for "id".
     resultDeclType = Context.getObjCIdType();
@@ -1540,7 +1506,7 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
                            ResultTInfo,
                            cast<DeclContext>(ClassDecl),
                            MethodType == tok::minus, isVariadic,
-                           false,
+                           false, false,
                            MethodDeclKind == tok::objc_optional ?
                            ObjCMethodDecl::Optional :
                            ObjCMethodDecl::Required);
@@ -1563,7 +1529,7 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
     ParmVarDecl* Param
       = ParmVarDecl::Create(Context, ObjCMethod, ArgInfo[i].NameLoc,
                             ArgInfo[i].Name, ArgType, DI,
-                            VarDecl::None, VarDecl::None, 0);
+                            SC_None, SC_None, 0);
 
     if (ArgType->isObjCObjectType()) {
       Diag(ArgInfo[i].NameLoc,
@@ -1582,7 +1548,7 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
   }
 
   for (unsigned i = 0, e = CNumArgs; i != e; ++i) {
-    ParmVarDecl *Param = CParamInfo[i].Param.getAs<ParmVarDecl>();
+    ParmVarDecl *Param = cast<ParmVarDecl>(CParamInfo[i].Param);
     QualType ArgType = Param->getType();
     if (ArgType.isNull())
       ArgType = Context.getObjCIdType();
@@ -1596,7 +1562,8 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
       Param->setInvalidDecl();
     }
     Param->setDeclContext(ObjCMethod);
-    IdResolver.RemoveDecl(Param);
+    if (Param->getDeclName())
+      IdResolver.RemoveDecl(Param);
     Params.push_back(Param);
   }
   
@@ -1626,7 +1593,8 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
     }
     InterfaceMD = ImpDecl->getClassInterface()->getMethod(Sel,
                                                    MethodType == tok::minus);
-    if (containsInvalidMethodImplAttribute(AttrList))
+    if (ObjCMethod->hasAttrs() &&
+        containsInvalidMethodImplAttribute(ObjCMethod->getAttrs()))
       Diag(EndLoc, diag::warn_attribute_method_def);
   } else if (ObjCCategoryImplDecl *CatImpDecl =
              dyn_cast<ObjCCategoryImplDecl>(ClassDecl)) {
@@ -1637,7 +1605,8 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
       PrevMethod = CatImpDecl->getClassMethod(Sel);
       CatImpDecl->addClassMethod(ObjCMethod);
     }
-    if (containsInvalidMethodImplAttribute(AttrList))
+    if (ObjCMethod->hasAttrs() &&
+        containsInvalidMethodImplAttribute(ObjCMethod->getAttrs()))
       Diag(EndLoc, diag::warn_attribute_method_def);
   }
   if (PrevMethod) {
@@ -1649,14 +1618,16 @@ Sema::DeclPtrTy Sema::ActOnMethodDeclaration(
 
   // If the interface declared this method, and it was deprecated there,
   // mark it deprecated here.
-  if (InterfaceMD && InterfaceMD->hasAttr<DeprecatedAttr>())
-    ObjCMethod->addAttr(::new (Context) DeprecatedAttr());
+  if (InterfaceMD)
+   if (Attr *DA = InterfaceMD->getAttr<DeprecatedAttr>())
+    ObjCMethod->addAttr(::new (Context) DeprecatedAttr(DA->getLocation(),
+                                                       Context));
 
-  return DeclPtrTy::make(ObjCMethod);
+  return ObjCMethod;
 }
 
 bool Sema::CheckObjCDeclScope(Decl *D) {
-  if (isa<TranslationUnitDecl>(CurContext->getLookupContext()))
+  if (isa<TranslationUnitDecl>(CurContext->getRedeclContext()))
     return false;
 
   Diag(D->getLocation(), diag::err_objc_decls_may_only_appear_in_global_scope);
@@ -1667,9 +1638,9 @@ bool Sema::CheckObjCDeclScope(Decl *D) {
 
 /// Called whenever @defs(ClassName) is encountered in the source.  Inserts the
 /// instance variables of ClassName into Decls.
-void Sema::ActOnDefs(Scope *S, DeclPtrTy TagD, SourceLocation DeclStart,
+void Sema::ActOnDefs(Scope *S, Decl *TagD, SourceLocation DeclStart,
                      IdentifierInfo *ClassName,
-                     llvm::SmallVectorImpl<DeclPtrTy> &Decls) {
+                     llvm::SmallVectorImpl<Decl*> &Decls) {
   // Check that ClassName is a valid class
   ObjCInterfaceDecl *Class = getObjCInterfaceDecl(ClassName, DeclStart);
   if (!Class) {
@@ -1682,25 +1653,25 @@ void Sema::ActOnDefs(Scope *S, DeclPtrTy TagD, SourceLocation DeclStart,
   }
 
   // Collect the instance variables
-  llvm::SmallVector<FieldDecl*, 32> RecFields;
-  Context.CollectObjCIvars(Class, RecFields);
+  llvm::SmallVector<ObjCIvarDecl*, 32> Ivars;
+  Context.DeepCollectObjCIvars(Class, true, Ivars);
   // For each ivar, create a fresh ObjCAtDefsFieldDecl.
-  for (unsigned i = 0; i < RecFields.size(); i++) {
-    FieldDecl* ID = RecFields[i];
-    RecordDecl *Record = dyn_cast<RecordDecl>(TagD.getAs<Decl>());
+  for (unsigned i = 0; i < Ivars.size(); i++) {
+    FieldDecl* ID = cast<FieldDecl>(Ivars[i]);
+    RecordDecl *Record = dyn_cast<RecordDecl>(TagD);
     Decl *FD = ObjCAtDefsFieldDecl::Create(Context, Record, ID->getLocation(),
                                            ID->getIdentifier(), ID->getType(),
                                            ID->getBitWidth());
-    Decls.push_back(Sema::DeclPtrTy::make(FD));
+    Decls.push_back(FD);
   }
 
   // Introduce all of these fields into the appropriate scope.
-  for (llvm::SmallVectorImpl<DeclPtrTy>::iterator D = Decls.begin();
+  for (llvm::SmallVectorImpl<Decl*>::iterator D = Decls.begin();
        D != Decls.end(); ++D) {
-    FieldDecl *FD = cast<FieldDecl>(D->getAs<Decl>());
+    FieldDecl *FD = cast<FieldDecl>(*D);
     if (getLangOptions().CPlusPlus)
       PushOnScopeChains(cast<FieldDecl>(FD), S);
-    else if (RecordDecl *Record = dyn_cast<RecordDecl>(TagD.getAs<Decl>()))
+    else if (RecordDecl *Record = dyn_cast<RecordDecl>(TagD))
       Record->addDecl(FD);
   }
 }
@@ -1735,7 +1706,7 @@ VarDecl *Sema::BuildObjCExceptionDecl(TypeSourceInfo *TInfo,
   }
   
   VarDecl *New = VarDecl::Create(Context, CurContext, NameLoc, Name, T, TInfo,
-                                 VarDecl::None, VarDecl::None);
+                                 SC_None, SC_None);
   New->setExceptionVariable(true);
   
   if (Invalid)
@@ -1743,7 +1714,7 @@ VarDecl *Sema::BuildObjCExceptionDecl(TypeSourceInfo *TInfo,
   return New;
 }
 
-Sema::DeclPtrTy Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
+Decl *Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
   const DeclSpec &DS = D.getDeclSpec();
   
   // We allow the "register" storage class on exception variables because
@@ -1788,7 +1759,7 @@ Sema::DeclPtrTy Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
   }
   
   // Add the parameter declaration into this scope.
-  S->AddDecl(DeclPtrTy::make(New));
+  S->AddDecl(New);
   if (D.getIdentifier())
     IdResolver.AddDecl(New);
   
@@ -1796,43 +1767,18 @@ Sema::DeclPtrTy Sema::ActOnObjCExceptionDecl(Scope *S, Declarator &D) {
   
   if (New->hasAttr<BlocksAttr>())
     Diag(New->getLocation(), diag::err_block_on_nonlocal);
-  return DeclPtrTy::make(New);
+  return New;
 }
 
 /// CollectIvarsToConstructOrDestruct - Collect those ivars which require
 /// initialization.
-void Sema::CollectIvarsToConstructOrDestruct(const ObjCInterfaceDecl *OI,
+void Sema::CollectIvarsToConstructOrDestruct(ObjCInterfaceDecl *OI,
                                 llvm::SmallVectorImpl<ObjCIvarDecl*> &Ivars) {
-  for (ObjCInterfaceDecl::ivar_iterator I = OI->ivar_begin(),
-       E = OI->ivar_end(); I != E; ++I) {
-    ObjCIvarDecl *Iv = (*I);
+  for (ObjCIvarDecl *Iv = OI->all_declared_ivar_begin(); Iv; 
+       Iv= Iv->getNextIvar()) {
     QualType QT = Context.getBaseElementType(Iv->getType());
     if (QT->isRecordType())
-      Ivars.push_back(*I);
-  }
-  
-  // Find ivars to construct/destruct in class extension.
-  for (const ObjCCategoryDecl *CDecl = OI->getFirstClassExtension(); CDecl;
-      CDecl = CDecl->getNextClassExtension()) {
-    for (ObjCCategoryDecl::ivar_iterator I = CDecl->ivar_begin(),
-         E = CDecl->ivar_end(); I != E; ++I) {
-      ObjCIvarDecl *Iv = (*I);
-      QualType QT = Context.getBaseElementType(Iv->getType());
-      if (QT->isRecordType())
-        Ivars.push_back(*I);
-    }
-  }
-  
-  // Also add any ivar defined in this class's implementation.  This
-  // includes synthesized ivars.
-  if (ObjCImplementationDecl *ImplDecl = OI->getImplementation()) {
-    for (ObjCImplementationDecl::ivar_iterator I = ImplDecl->ivar_begin(),
-         E = ImplDecl->ivar_end(); I != E; ++I) {
-      ObjCIvarDecl *Iv = (*I);
-      QualType QT = Context.getBaseElementType(Iv->getType());
-      if (QT->isRecordType())
-        Ivars.push_back(*I);
-    }
+      Ivars.push_back(Iv);
   }
 }
 
@@ -1849,3 +1795,15 @@ void ObjCImplementationDecl::setIvarInitializers(ASTContext &C,
   }
 }
 
+void Sema::DiagnoseUseOfUnimplementedSelectors() {
+  if (ReferencedSelectors.empty())
+    return;
+  for (llvm::DenseMap<Selector, SourceLocation>::iterator S = 
+        ReferencedSelectors.begin(),
+       E = ReferencedSelectors.end(); S != E; ++S) {
+    Selector Sel = (*S).first;
+    if (!LookupImplementedMethodInGlobalPool(Sel))
+      Diag((*S).second, diag::warn_unimplemented_selector) << Sel;
+  }
+  return;
+}
diff --git a/lib/Sema/SemaExceptionSpec.cpp b/lib/Sema/SemaExceptionSpec.cpp
index 34a479ae2a08..c902e7787096 100644
--- a/lib/Sema/SemaExceptionSpec.cpp
+++ b/lib/Sema/SemaExceptionSpec.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
+#include "clang/Sema/SemaInternal.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
@@ -261,6 +261,14 @@ bool Sema::CheckEquivalentExceptionSpec(const PartialDiagnostic &DiagID,
 
   bool OldAny = !Old->hasExceptionSpec() || Old->hasAnyExceptionSpec();
   bool NewAny = !New->hasExceptionSpec() || New->hasAnyExceptionSpec();
+  if (getLangOptions().Microsoft) {
+    // Treat throw(whatever) as throw(...) to be compatible with MS headers.
+    if (New->hasExceptionSpec() && New->getNumExceptions() > 0)
+      NewAny = true;
+    if (Old->hasExceptionSpec() && Old->getNumExceptions() > 0)
+      OldAny = true;
+  }
+
   if (OldAny && NewAny)
     return false;
   if (OldAny || NewAny) {
diff --git a/lib/Sema/SemaExpr.cpp b/lib/Sema/SemaExpr.cpp
index 5f46a977b12c..80b465230e14 100644
--- a/lib/Sema/SemaExpr.cpp
+++ b/lib/Sema/SemaExpr.cpp
@@ -11,10 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
-#include "Lookup.h"
-#include "AnalysisBasedWarnings.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/AnalysisBasedWarnings.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclObjC.h"
@@ -29,11 +29,14 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/Preprocessor.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Designator.h"
-#include "clang/Parse/Scope.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Designator.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/Template.h"
 using namespace clang;
+using namespace sema;
 
 
 /// \brief Determine whether the use of this declaration is valid, and
@@ -59,7 +62,7 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc) {
 
   // See if the decl is unavailable
   if (D->getAttr<UnavailableAttr>()) {
-    Diag(Loc, diag::warn_unavailable) << D->getDeclName();
+    Diag(Loc, diag::err_unavailable) << D->getDeclName();
     Diag(D->getLocation(), diag::note_unavailable_here) << 0;
   }
 
@@ -192,7 +195,7 @@ void Sema::DefaultFunctionArrayConversion(Expr *&E) {
 
   if (Ty->isFunctionType())
     ImpCastExprToType(E, Context.getPointerType(Ty),
-                      CastExpr::CK_FunctionToPointerDecay);
+                      CK_FunctionToPointerDecay);
   else if (Ty->isArrayType()) {
     // In C90 mode, arrays only promote to pointers if the array expression is
     // an lvalue.  The relevant legalese is C90 6.2.2.1p3: "an lvalue that has
@@ -208,7 +211,7 @@ void Sema::DefaultFunctionArrayConversion(Expr *&E) {
     if (getLangOptions().C99 || getLangOptions().CPlusPlus ||
         E->isLvalue(Context) == Expr::LV_Valid)
       ImpCastExprToType(E, Context.getArrayDecayedType(Ty),
-                        CastExpr::CK_ArrayToPointerDecay);
+                        CK_ArrayToPointerDecay);
   }
 }
 
@@ -229,7 +232,7 @@ void Sema::DefaultFunctionArrayLvalueConversion(Expr *&E) {
     //   If the lvalue has qualified type, the value has the unqualified
     //   version of the type of the lvalue; otherwise, the value has the
     //   type of the lvalue.
-    ImpCastExprToType(E, Ty.getUnqualifiedType(), CastExpr::CK_NoOp);
+    ImpCastExprToType(E, Ty.getUnqualifiedType(), CK_NoOp);
   }
 }
 
@@ -258,12 +261,12 @@ Expr *Sema::UsualUnaryConversions(Expr *&Expr) {
   //   other types are unchanged by the integer promotions.
   QualType PTy = Context.isPromotableBitField(Expr);
   if (!PTy.isNull()) {
-    ImpCastExprToType(Expr, PTy, CastExpr::CK_IntegralCast);
+    ImpCastExprToType(Expr, PTy, CK_IntegralCast);
     return Expr;
   }
   if (Ty->isPromotableIntegerType()) {
     QualType PT = Context.getPromotedIntegerType(Ty);
-    ImpCastExprToType(Expr, PT, CastExpr::CK_IntegralCast);
+    ImpCastExprToType(Expr, PT, CK_IntegralCast);
     return Expr;
   }
 
@@ -281,7 +284,7 @@ void Sema::DefaultArgumentPromotion(Expr *&Expr) {
   // If this is a 'float' (CVR qualified or typedef) promote to double.
   if (Ty->isSpecificBuiltinType(BuiltinType::Float))
     return ImpCastExprToType(Expr, Context.DoubleTy,
-                             CastExpr::CK_FloatingCast);
+                             CK_FloatingCast);
 
   UsualUnaryConversions(Expr);
 }
@@ -355,8 +358,8 @@ QualType Sema::UsualArithmeticConversions(Expr *&lhsExpr, Expr *&rhsExpr,
 
   QualType destType = Context.UsualArithmeticConversionsType(lhs, rhs);
   if (!isCompAssign)
-    ImpCastExprToType(lhsExpr, destType, CastExpr::CK_Unknown);
-  ImpCastExprToType(rhsExpr, destType, CastExpr::CK_Unknown);
+    ImpCastExprToType(lhsExpr, destType, CK_Unknown);
+  ImpCastExprToType(rhsExpr, destType, CK_Unknown);
   return destType;
 }
 
@@ -371,7 +374,7 @@ QualType Sema::UsualArithmeticConversions(Expr *&lhsExpr, Expr *&rhsExpr,
 /// multiple tokens.  However, the common case is that StringToks points to one
 /// string.
 ///
-Action::OwningExprResult
+ExprResult
 Sema::ActOnStringLiteral(const Token *StringToks, unsigned NumStringToks) {
   assert(NumStringToks && "Must have at least one string!");
 
@@ -459,13 +462,20 @@ static bool ShouldSnapshotBlockValueReference(Sema &S, BlockScopeInfo *CurBlock,
 }
 
 
+ExprResult
+Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, SourceLocation Loc,
+                       const CXXScopeSpec *SS) {
+  DeclarationNameInfo NameInfo(D->getDeclName(), Loc);
+  return BuildDeclRefExpr(D, Ty, NameInfo, SS);
+}
 
 /// BuildDeclRefExpr - Build a DeclRefExpr.
-Sema::OwningExprResult
-Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, SourceLocation Loc,
+ExprResult
+Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty,
+                       const DeclarationNameInfo &NameInfo,
                        const CXXScopeSpec *SS) {
   if (Context.getCanonicalType(Ty) == Context.UndeducedAutoTy) {
-    Diag(Loc,
+    Diag(NameInfo.getLoc(),
          diag::err_auto_variable_cannot_appear_in_own_initializer)
       << D->getDeclName();
     return ExprError();
@@ -479,7 +489,8 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, SourceLocation Loc,
     } else if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(CurContext)) {
       if (const FunctionDecl *FD = MD->getParent()->isLocalClass()) {
         if (VD->hasLocalStorage() && VD->getDeclContext() != CurContext) {
-          Diag(Loc, diag::err_reference_to_local_var_in_enclosing_function)
+          Diag(NameInfo.getLoc(),
+               diag::err_reference_to_local_var_in_enclosing_function)
             << D->getIdentifier() << FD->getDeclName();
           Diag(D->getLocation(), diag::note_local_variable_declared_here)
             << D->getIdentifier();
@@ -489,12 +500,12 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, SourceLocation Loc,
     }
   }
 
-  MarkDeclarationReferenced(Loc, D);
+  MarkDeclarationReferenced(NameInfo.getLoc(), D);
 
   return Owned(DeclRefExpr::Create(Context,
                               SS? (NestedNameSpecifier *)SS->getScopeRep() : 0,
                                    SS? SS->getRange() : SourceRange(),
-                                   D, Loc, Ty));
+                                   D, NameInfo, Ty));
 }
 
 /// \brief Given a field that represents a member of an anonymous
@@ -535,7 +546,7 @@ VarDecl *Sema::BuildAnonymousStructUnionMemberPath(FieldDecl *Field,
   return BaseObject;
 }
 
-Sema::OwningExprResult
+ExprResult
 Sema::BuildAnonymousStructUnionMemberReference(SourceLocation Loc,
                                                FieldDecl *Field,
                                                Expr *BaseObjectExpr,
@@ -553,7 +564,6 @@ Sema::BuildAnonymousStructUnionMemberReference(SourceLocation Loc,
   if (BaseObject) {
     // BaseObject is an anonymous struct/union variable (and is,
     // therefore, not part of another non-anonymous record).
-    if (BaseObjectExpr) BaseObjectExpr->Destroy(Context);
     MarkDeclarationReferenced(Loc, BaseObject);
     BaseObjectExpr = new (Context) DeclRefExpr(BaseObject,BaseObject->getType(),
                                                SourceLocation());
@@ -640,7 +650,7 @@ Sema::BuildAnonymousStructUnionMemberReference(SourceLocation Loc,
   return Owned(Result);
 }
 
-/// Decomposes the given name into a DeclarationName, its location, and
+/// Decomposes the given name into a DeclarationNameInfo, its location, and
 /// possibly a list of template arguments.
 ///
 /// If this produces template arguments, it is permitted to call
@@ -652,8 +662,7 @@ Sema::BuildAnonymousStructUnionMemberReference(SourceLocation Loc,
 static void DecomposeUnqualifiedId(Sema &SemaRef,
                                    const UnqualifiedId &Id,
                                    TemplateArgumentListInfo &Buffer,
-                                   DeclarationName &Name,
-                                   SourceLocation &NameLoc,
+                                   DeclarationNameInfo &NameInfo,
                              const TemplateArgumentListInfo *&TemplateArgs) {
   if (Id.getKind() == UnqualifiedId::IK_TemplateId) {
     Buffer.setLAngleLoc(Id.TemplateId->LAngleLoc);
@@ -665,15 +674,12 @@ static void DecomposeUnqualifiedId(Sema &SemaRef,
     SemaRef.translateTemplateArguments(TemplateArgsPtr, Buffer);
     TemplateArgsPtr.release();
 
-    TemplateName TName =
-      Sema::TemplateTy::make(Id.TemplateId->Template).getAsVal<TemplateName>();
-
-    Name = SemaRef.Context.getNameForTemplate(TName);
-    NameLoc = Id.TemplateId->TemplateNameLoc;
+    TemplateName TName = Id.TemplateId->Template.get();
+    SourceLocation TNameLoc = Id.TemplateId->TemplateNameLoc;
+    NameInfo = SemaRef.Context.getNameForTemplate(TName, TNameLoc);
     TemplateArgs = &Buffer;
   } else {
-    Name = SemaRef.GetNameFromUnqualifiedId(Id);
-    NameLoc = Id.StartLocation;
+    NameInfo = SemaRef.GetNameFromUnqualifiedId(Id);
     TemplateArgs = 0;
   }
 }
@@ -700,23 +706,6 @@ static bool IsFullyFormedScope(Sema &SemaRef, CXXRecordDecl *Record) {
   return true;
 }
 
-/// Determines whether we can lookup this id-expression now or whether
-/// we have to wait until template instantiation is complete.
-static bool IsDependentIdExpression(Sema &SemaRef, const CXXScopeSpec &SS) {
-  DeclContext *DC = SemaRef.computeDeclContext(SS, false);
-
-  // If the qualifier scope isn't computable, it's definitely dependent.
-  if (!DC) return true;
-
-  // If the qualifier scope doesn't name a record, we can always look into it.
-  if (!isa<CXXRecordDecl>(DC)) return false;
-
-  // We can't look into record types unless they're fully-formed.
-  if (!IsFullyFormedScope(SemaRef, cast<CXXRecordDecl>(DC))) return true;
-
-  return false;
-}
-
 /// Determines if the given class is provably not derived from all of
 /// the prospective base classes.
 static bool IsProvablyNotDerivedFrom(Sema &SemaRef,
@@ -905,25 +894,30 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
         // TODO: fixit for inserting 'Base<T>::' in the other cases.
         // Actually quite difficult!
         if (isInstance) {
-          Diag(R.getNameLoc(), diagnostic) << Name
-            << FixItHint::CreateInsertion(R.getNameLoc(), "this->");
-
           UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(
               CallsUndergoingInstantiation.back()->getCallee());
-          CXXMethodDecl *DepMethod = cast<CXXMethodDecl>(
+          CXXMethodDecl *DepMethod = cast_or_null<CXXMethodDecl>(
               CurMethod->getInstantiatedFromMemberFunction());
-          QualType DepThisType = DepMethod->getThisType(Context);
-          CXXThisExpr *DepThis = new (Context) CXXThisExpr(R.getNameLoc(),
-                                                           DepThisType, false);
-          TemplateArgumentListInfo TList;
-          if (ULE->hasExplicitTemplateArgs())
-            ULE->copyTemplateArgumentsInto(TList);
-          CXXDependentScopeMemberExpr *DepExpr =
-              CXXDependentScopeMemberExpr::Create(
-                  Context, DepThis, DepThisType, true, SourceLocation(),
-                  ULE->getQualifier(), ULE->getQualifierRange(), NULL, Name,
-                  R.getNameLoc(), &TList);
-          CallsUndergoingInstantiation.back()->setCallee(DepExpr);
+          if (DepMethod) {
+            Diag(R.getNameLoc(), diagnostic) << Name
+              << FixItHint::CreateInsertion(R.getNameLoc(), "this->");
+            QualType DepThisType = DepMethod->getThisType(Context);
+            CXXThisExpr *DepThis = new (Context) CXXThisExpr(
+                                       R.getNameLoc(), DepThisType, false);
+            TemplateArgumentListInfo TList;
+            if (ULE->hasExplicitTemplateArgs())
+              ULE->copyTemplateArgumentsInto(TList);
+            CXXDependentScopeMemberExpr *DepExpr =
+                CXXDependentScopeMemberExpr::Create(
+                    Context, DepThis, DepThisType, true, SourceLocation(),
+                    ULE->getQualifier(), ULE->getQualifierRange(), NULL,
+                    R.getLookupNameInfo(), &TList);
+            CallsUndergoingInstantiation.back()->setCallee(DepExpr);
+          } else {
+            // FIXME: we should be able to handle this case too. It is correct
+            // to add this-> here. This is a workaround for PR7947.
+            Diag(R.getNameLoc(), diagnostic) << Name;
+          }
         } else {
           Diag(R.getNameLoc(), diagnostic) << Name;
         }
@@ -935,6 +929,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
         // Tell the callee to try to recover.
         return false;
       }
+
+      R.clear();
     }
   }
 
@@ -1005,11 +1001,74 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
   return true;
 }
 
-Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
-                                               CXXScopeSpec &SS,
-                                               UnqualifiedId &Id,
-                                               bool HasTrailingLParen,
-                                               bool isAddressOfOperand) {
+static ObjCPropertyDecl *OkToSynthesizeProvisionalIvar(Sema &SemaRef,
+                                                       IdentifierInfo *II,
+                                                       SourceLocation NameLoc) {
+  ObjCMethodDecl *CurMeth = SemaRef.getCurMethodDecl();
+  ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
+  if (!IDecl)
+    return 0;
+  ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
+  if (!ClassImpDecl)
+    return 0;
+  ObjCPropertyDecl *property = SemaRef.LookupPropertyDecl(IDecl, II);
+  if (!property)
+    return 0;
+  if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II))
+    if (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+      return 0;
+  return property;
+}
+
+static ObjCIvarDecl *SynthesizeProvisionalIvar(Sema &SemaRef,
+                                               LookupResult &Lookup,
+                                               IdentifierInfo *II,
+                                               SourceLocation NameLoc) {
+  ObjCMethodDecl *CurMeth = SemaRef.getCurMethodDecl();
+  bool LookForIvars;
+  if (Lookup.empty())
+    LookForIvars = true;
+  else if (CurMeth->isClassMethod())
+    LookForIvars = false;
+  else
+    LookForIvars = (Lookup.isSingleResult() &&
+                    Lookup.getFoundDecl()->isDefinedOutsideFunctionOrMethod());
+  if (!LookForIvars)
+    return 0;
+  
+  ObjCInterfaceDecl *IDecl = CurMeth->getClassInterface();
+  if (!IDecl)
+    return 0;
+  ObjCImplementationDecl *ClassImpDecl = IDecl->getImplementation();
+  if (!ClassImpDecl)
+    return 0;
+  bool DynamicImplSeen = false;
+  ObjCPropertyDecl *property = SemaRef.LookupPropertyDecl(IDecl, II);
+  if (!property)
+    return 0;
+  if (ObjCPropertyImplDecl *PIDecl = ClassImpDecl->FindPropertyImplDecl(II))
+    DynamicImplSeen = 
+      (PIDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic);
+  if (!DynamicImplSeen) {
+    QualType PropType = SemaRef.Context.getCanonicalType(property->getType());
+    ObjCIvarDecl *Ivar = ObjCIvarDecl::Create(SemaRef.Context, ClassImpDecl, 
+                                              NameLoc,
+                                              II, PropType, /*Dinfo=*/0,
+                                              ObjCIvarDecl::Protected,
+                                              (Expr *)0, true);
+    ClassImpDecl->addDecl(Ivar);
+    IDecl->makeDeclVisibleInContext(Ivar, false);
+    property->setPropertyIvarDecl(Ivar);
+    return Ivar;
+  }
+  return 0;
+}
+
+ExprResult Sema::ActOnIdExpression(Scope *S,
+                                   CXXScopeSpec &SS,
+                                   UnqualifiedId &Id,
+                                   bool HasTrailingLParen,
+                                   bool isAddressOfOperand) {
   assert(!(isAddressOfOperand && HasTrailingLParen) &&
          "cannot be direct & operand and have a trailing lparen");
 
@@ -1019,13 +1078,13 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
   TemplateArgumentListInfo TemplateArgsBuffer;
 
   // Decompose the UnqualifiedId into the following data.
-  DeclarationName Name;
-  SourceLocation NameLoc;
+  DeclarationNameInfo NameInfo;
   const TemplateArgumentListInfo *TemplateArgs;
-  DecomposeUnqualifiedId(*this, Id, TemplateArgsBuffer,
-                         Name, NameLoc, TemplateArgs);
+  DecomposeUnqualifiedId(*this, Id, TemplateArgsBuffer, NameInfo, TemplateArgs);
 
+  DeclarationName Name = NameInfo.getName();
   IdentifierInfo *II = Name.getAsIdentifierInfo();
+  SourceLocation NameLoc = NameInfo.getLoc();
 
   // C++ [temp.dep.expr]p3:
   //   An id-expression is type-dependent if it contains:
@@ -1038,16 +1097,30 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
   //        names a dependent type.
   // Determine whether this is a member of an unknown specialization;
   // we need to handle these differently.
-  if ((Name.getNameKind() == DeclarationName::CXXConversionFunctionName &&
-       Name.getCXXNameType()->isDependentType()) ||
-      (SS.isSet() && IsDependentIdExpression(*this, SS))) {
-    return ActOnDependentIdExpression(SS, Name, NameLoc,
-                                      isAddressOfOperand,
-                                      TemplateArgs);
+  bool DependentID = false;
+  if (Name.getNameKind() == DeclarationName::CXXConversionFunctionName &&
+      Name.getCXXNameType()->isDependentType()) {
+    DependentID = true;
+  } else if (SS.isSet()) {
+    DeclContext *DC = computeDeclContext(SS, false);
+    if (DC) {
+      if (RequireCompleteDeclContext(SS, DC))
+        return ExprError();
+      // FIXME: We should be checking whether DC is the current instantiation.
+      if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(DC))
+        DependentID = !IsFullyFormedScope(*this, RD);
+    } else {
+      DependentID = true;
+    }
   }
 
+  if (DependentID) {
+    return ActOnDependentIdExpression(SS, NameInfo, isAddressOfOperand,
+                                      TemplateArgs);
+  }
+  bool IvarLookupFollowUp = false;
   // Perform the required lookup.
-  LookupResult R(*this, Name, NameLoc, LookupOrdinaryName);
+  LookupResult R(*this, NameInfo, LookupOrdinaryName);
   if (TemplateArgs) {
     // Lookup the template name again to correctly establish the context in
     // which it was found. This is really unfortunate as we already did the
@@ -1058,18 +1131,26 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
     LookupTemplateName(R, S, SS, QualType(), /*EnteringContext=*/false,
                        MemberOfUnknownSpecialization);
   } else {
-    bool IvarLookupFollowUp = (!SS.isSet() && II && getCurMethodDecl());
+    IvarLookupFollowUp = (!SS.isSet() && II && getCurMethodDecl());
     LookupParsedName(R, S, &SS, !IvarLookupFollowUp);
 
     // If this reference is in an Objective-C method, then we need to do
     // some special Objective-C lookup, too.
     if (IvarLookupFollowUp) {
-      OwningExprResult E(LookupInObjCMethod(R, S, II, true));
+      ExprResult E(LookupInObjCMethod(R, S, II, true));
       if (E.isInvalid())
         return ExprError();
 
       Expr *Ex = E.takeAs<Expr>();
       if (Ex) return Owned(Ex);
+      // Synthesize ivars lazily
+      if (getLangOptions().ObjCNonFragileABI2) {
+        if (SynthesizeProvisionalIvar(*this, R, II, NameLoc))
+          return ActOnIdExpression(S, SS, Id, HasTrailingLParen,
+                                   isAddressOfOperand);
+      }
+      // for further use, this must be set to false if in class method.
+      IvarLookupFollowUp = getCurMethodDecl()->isInstanceMethod();
     }
   }
 
@@ -1102,7 +1183,7 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
       // reference the ivar.
       if (ObjCIvarDecl *Ivar = R.getAsSingle<ObjCIvarDecl>()) {
         R.clear();
-        OwningExprResult E(LookupInObjCMethod(R, S, Ivar->getIdentifier()));
+        ExprResult E(LookupInObjCMethod(R, S, Ivar->getIdentifier()));
         assert(E.isInvalid() || E.get());
         return move(E);
       }
@@ -1113,23 +1194,15 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
   assert(!R.empty() || ADL);
 
   if (VarDecl *Var = R.getAsSingle<VarDecl>()) {
-    // Warn about constructs like:
-    //   if (void *X = foo()) { ... } else { X }.
-    // In the else block, the pointer is always false.
-    if (Var->isDeclaredInCondition() && Var->getType()->isScalarType()) {
-      Scope *CheckS = S;
-      while (CheckS && CheckS->getControlParent()) {
-        if ((CheckS->getFlags() & Scope::ElseScope) &&
-            CheckS->getControlParent()->isDeclScope(DeclPtrTy::make(Var))) {
-          ExprError(Diag(NameLoc, diag::warn_value_always_zero)
-            << Var->getDeclName()
-            << (Var->getType()->isPointerType() ? 2 :
-                Var->getType()->isBooleanType() ? 1 : 0));
-          break;
-        }
-
-        // Move to the parent of this scope.
-        CheckS = CheckS->getParent();
+    if (getLangOptions().ObjCNonFragileABI && IvarLookupFollowUp &&
+        !getLangOptions().ObjCNonFragileABI2 &&
+        Var->isFileVarDecl()) {
+      ObjCPropertyDecl *Property = 
+        OkToSynthesizeProvisionalIvar(*this, II, NameLoc);
+      if (Property) {
+        Diag(NameLoc, diag::warn_ivar_variable_conflict) << Var->getDeclName();
+        Diag(Property->getLocation(), diag::note_property_declare);
+        Diag(Var->getLocation(), diag::note_global_declared_at);
       }
     }
   } else if (FunctionDecl *Func = R.getAsSingle<FunctionDecl>()) {
@@ -1152,15 +1225,43 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
   }
 
   // Check whether this might be a C++ implicit instance member access.
-  // C++ [expr.prim.general]p6:
-  //   Within the definition of a non-static member function, an
-  //   identifier that names a non-static member is transformed to a
-  //   class member access expression.
-  // But note that &SomeClass::foo is grammatically distinct, even
-  // though we don't parse it that way.
+  // C++ [class.mfct.non-static]p3:
+  //   When an id-expression that is not part of a class member access
+  //   syntax and not used to form a pointer to member is used in the
+  //   body of a non-static member function of class X, if name lookup
+  //   resolves the name in the id-expression to a non-static non-type
+  //   member of some class C, the id-expression is transformed into a
+  //   class member access expression using (*this) as the
+  //   postfix-expression to the left of the . operator.
+  //
+  // But we don't actually need to do this for '&' operands if R
+  // resolved to a function or overloaded function set, because the
+  // expression is ill-formed if it actually works out to be a
+  // non-static member function:
+  //
+  // C++ [expr.ref]p4:
+  //   Otherwise, if E1.E2 refers to a non-static member function. . .
+  //   [t]he expression can be used only as the left-hand operand of a
+  //   member function call.
+  //
+  // There are other safeguards against such uses, but it's important
+  // to get this right here so that we don't end up making a
+  // spuriously dependent expression if we're inside a dependent
+  // instance method.
   if (!R.empty() && (*R.begin())->isCXXClassMember()) {
-    bool isAbstractMemberPointer = (isAddressOfOperand && !SS.isEmpty());
-    if (!isAbstractMemberPointer)
+    bool MightBeImplicitMember;
+    if (!isAddressOfOperand)
+      MightBeImplicitMember = true;
+    else if (!SS.isEmpty())
+      MightBeImplicitMember = false;
+    else if (R.isOverloadedResult())
+      MightBeImplicitMember = false;
+    else if (R.isUnresolvableResult())
+      MightBeImplicitMember = true;
+    else
+      MightBeImplicitMember = isa<FieldDecl>(R.getFoundDecl());
+
+    if (MightBeImplicitMember)
       return BuildPossibleImplicitMemberExpr(SS, R, TemplateArgs);
   }
 
@@ -1171,7 +1272,7 @@ Sema::OwningExprResult Sema::ActOnIdExpression(Scope *S,
 }
 
 /// Builds an expression which might be an implicit member expression.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
                                       LookupResult &R,
                                 const TemplateArgumentListInfo *TemplateArgs) {
@@ -1210,25 +1311,25 @@ Sema::BuildPossibleImplicitMemberExpr(const CXXScopeSpec &SS,
 /// declaration name, generally during template instantiation.
 /// There's a large number of things which don't need to be done along
 /// this path.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS,
-                                        DeclarationName Name,
-                                        SourceLocation NameLoc) {
+                                        const DeclarationNameInfo &NameInfo) {
   DeclContext *DC;
   if (!(DC = computeDeclContext(SS, false)) || DC->isDependentContext())
-    return BuildDependentDeclRefExpr(SS, Name, NameLoc, 0);
+    return BuildDependentDeclRefExpr(SS, NameInfo, 0);
 
   if (RequireCompleteDeclContext(SS, DC))
     return ExprError();
 
-  LookupResult R(*this, Name, NameLoc, LookupOrdinaryName);
+  LookupResult R(*this, NameInfo, LookupOrdinaryName);
   LookupQualifiedName(R, DC);
 
   if (R.isAmbiguous())
     return ExprError();
 
   if (R.empty()) {
-    Diag(NameLoc, diag::err_no_member) << Name << DC << SS.getRange();
+    Diag(NameInfo.getLoc(), diag::err_no_member)
+      << NameInfo.getName() << DC << SS.getRange();
     return ExprError();
   }
 
@@ -1243,7 +1344,7 @@ Sema::BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS,
 /// actually quite a lot of extra work involved.
 ///
 /// Returns a null sentinel to indicate trivial success.
-Sema::OwningExprResult
+ExprResult
 Sema::LookupInObjCMethod(LookupResult &Lookup, Scope *S,
                          IdentifierInfo *II, bool AllowBuiltinCreation) {
   SourceLocation Loc = Lookup.getNameLoc();
@@ -1298,7 +1399,7 @@ Sema::LookupInObjCMethod(LookupResult &Lookup, Scope *S,
       UnqualifiedId SelfName;
       SelfName.setIdentifier(&II, SourceLocation());
       CXXScopeSpec SelfScopeSpec;
-      OwningExprResult SelfExpr = ActOnIdExpression(S, SelfScopeSpec,
+      ExprResult SelfExpr = ActOnIdExpression(S, SelfScopeSpec,
                                                     SelfName, false, false);
       MarkDeclarationReferenced(Loc, IV);
       return Owned(new (Context)
@@ -1403,9 +1504,8 @@ Sema::PerformObjectMemberConversion(Expr *&From,
   SourceRange FromRange = From->getSourceRange();
   SourceLocation FromLoc = FromRange.getBegin();
 
-  bool isLvalue
-    = (From->isLvalue(Context) == Expr::LV_Valid) && !PointerConversions;
-  
+  ExprValueKind VK = CastCategory(From);
+
   // C++ [class.member.lookup]p8:
   //   [...] Ambiguities can often be resolved by qualifying a name with its
   //   class name.
@@ -1435,15 +1535,15 @@ Sema::PerformObjectMemberConversion(Expr *&From,
     // type of the object type, in which case we just ignore it.
     // Otherwise build the appropriate casts.
     if (IsDerivedFrom(FromRecordType, QRecordType)) {
-      CXXBaseSpecifierArray BasePath;
+      CXXCastPath BasePath;
       if (CheckDerivedToBaseConversion(FromRecordType, QRecordType,
                                        FromLoc, FromRange, &BasePath))
         return true;
 
       if (PointerConversions)
         QType = Context.getPointerType(QType);
-      ImpCastExprToType(From, QType, CastExpr::CK_UncheckedDerivedToBase,
-                        isLvalue, BasePath);
+      ImpCastExprToType(From, QType, CK_UncheckedDerivedToBase,
+                        VK, &BasePath);
 
       FromType = QType;
       FromRecordType = QRecordType;
@@ -1471,7 +1571,7 @@ Sema::PerformObjectMemberConversion(Expr *&From,
     // conversion is non-trivial.
     if (!Context.hasSameUnqualifiedType(FromRecordType, URecordType)) {
       assert(IsDerivedFrom(FromRecordType, URecordType));
-      CXXBaseSpecifierArray BasePath;
+      CXXCastPath BasePath;
       if (CheckDerivedToBaseConversion(FromRecordType, URecordType,
                                        FromLoc, FromRange, &BasePath))
         return true;
@@ -1479,8 +1579,8 @@ Sema::PerformObjectMemberConversion(Expr *&From,
       QualType UType = URecordType;
       if (PointerConversions)
         UType = Context.getPointerType(UType);
-      ImpCastExprToType(From, UType, CastExpr::CK_UncheckedDerivedToBase,
-                        isLvalue, BasePath);
+      ImpCastExprToType(From, UType, CK_UncheckedDerivedToBase,
+                        VK, &BasePath);
       FromType = UType;
       FromRecordType = URecordType;
     }
@@ -1490,14 +1590,14 @@ Sema::PerformObjectMemberConversion(Expr *&From,
     IgnoreAccess = true;
   }
 
-  CXXBaseSpecifierArray BasePath;
+  CXXCastPath BasePath;
   if (CheckDerivedToBaseConversion(FromRecordType, DestRecordType,
                                    FromLoc, FromRange, &BasePath,
                                    IgnoreAccess))
     return true;
 
-  ImpCastExprToType(From, DestType, CastExpr::CK_UncheckedDerivedToBase,
-                    isLvalue, BasePath);
+  ImpCastExprToType(From, DestType, CK_UncheckedDerivedToBase,
+                    VK, &BasePath);
   return false;
 }
 
@@ -1505,7 +1605,8 @@ Sema::PerformObjectMemberConversion(Expr *&From,
 static MemberExpr *BuildMemberExpr(ASTContext &C, Expr *Base, bool isArrow,
                                    const CXXScopeSpec &SS, ValueDecl *Member,
                                    DeclAccessPair FoundDecl,
-                                   SourceLocation Loc, QualType Ty,
+                                   const DeclarationNameInfo &MemberNameInfo,
+                                   QualType Ty,
                           const TemplateArgumentListInfo *TemplateArgs = 0) {
   NestedNameSpecifier *Qualifier = 0;
   SourceRange QualifierRange;
@@ -1515,14 +1616,15 @@ static MemberExpr *BuildMemberExpr(ASTContext &C, Expr *Base, bool isArrow,
   }
 
   return MemberExpr::Create(C, Base, isArrow, Qualifier, QualifierRange,
-                            Member, FoundDecl, Loc, TemplateArgs, Ty);
+                            Member, FoundDecl, MemberNameInfo,
+                            TemplateArgs, Ty);
 }
 
 /// Builds an implicit member access expression.  The current context
 /// is known to be an instance method, and the given unqualified lookup
 /// set is known to contain only instance members, at least one of which
 /// is from an appropriate type.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildImplicitMemberExpr(const CXXScopeSpec &SS,
                               LookupResult &R,
                               const TemplateArgumentListInfo *TemplateArgs,
@@ -1551,7 +1653,7 @@ Sema::BuildImplicitMemberExpr(const CXXScopeSpec &SS,
     This = new (Context) CXXThisExpr(Loc, ThisType, /*isImplicit=*/true);
   }
 
-  return BuildMemberReferenceExpr(ExprArg(*this, This), ThisType,
+  return BuildMemberReferenceExpr(This, ThisType,
                                   /*OpLoc*/ SourceLocation(),
                                   /*IsArrow*/ true,
                                   SS,
@@ -1638,14 +1740,15 @@ static bool CheckDeclInExpr(Sema &S, SourceLocation Loc, NamedDecl *D) {
   return false;
 }
 
-Sema::OwningExprResult
+ExprResult
 Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
                                LookupResult &R,
                                bool NeedsADL) {
   // If this is a single, fully-resolved result and we don't need ADL,
   // just build an ordinary singleton decl ref.
   if (!NeedsADL && R.isSingleResult() && !R.getAsSingle<FunctionTemplateDecl>())
-    return BuildDeclarationNameExpr(SS, R.getNameLoc(), R.getFoundDecl());
+    return BuildDeclarationNameExpr(SS, R.getLookupNameInfo(),
+                                    R.getFoundDecl());
 
   // We only need to check the declaration if there's exactly one
   // result, because in the overloaded case the results can only be
@@ -1664,8 +1767,7 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   UnresolvedLookupExpr *ULE
     = UnresolvedLookupExpr::Create(Context, Dependent, R.getNamingClass(),
                                    (NestedNameSpecifier*) SS.getScopeRep(),
-                                   SS.getRange(),
-                                   R.getLookupName(), R.getNameLoc(),
+                                   SS.getRange(), R.getLookupNameInfo(),
                                    NeedsADL, R.isOverloadedResult(),
                                    R.begin(), R.end());
 
@@ -1674,13 +1776,15 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
 
 
 /// \brief Complete semantic analysis for a reference to the given declaration.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
-                               SourceLocation Loc, NamedDecl *D) {
+                               const DeclarationNameInfo &NameInfo,
+                               NamedDecl *D) {
   assert(D && "Cannot refer to a NULL declaration");
   assert(!isa<FunctionTemplateDecl>(D) &&
          "Cannot refer unambiguously to a function template");
 
+  SourceLocation Loc = NameInfo.getLoc();
   if (CheckDeclInExpr(*this, Loc, D))
     return ExprError();
 
@@ -1755,13 +1859,13 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
                     DeclRefExpr(const_cast<ValueDecl*>(BDRE->getDecl()), T,
                                           SourceLocation());
       
-        OwningExprResult Res = PerformCopyInitialization(
+        ExprResult Res = PerformCopyInitialization(
                           InitializedEntity::InitializeBlock(VD->getLocation(), 
                                                          T, false),
                                                          SourceLocation(),
                                                          Owned(E));
         if (!Res.isInvalid()) {
-          Res = MaybeCreateCXXExprWithTemporaries(move(Res));
+          Res = MaybeCreateCXXExprWithTemporaries(Res.get());
           Expr *Init = Res.takeAs<Expr>();
           BDRE->setCopyConstructorExpr(Init);
         }
@@ -1772,10 +1876,11 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   // If this reference is not in a block or if the referenced variable is
   // within the block, create a normal DeclRefExpr.
 
-  return BuildDeclRefExpr(VD, VD->getType().getNonReferenceType(), Loc, &SS);
+  return BuildDeclRefExpr(VD, VD->getType().getNonReferenceType(),
+                          NameInfo, &SS);
 }
 
-Sema::OwningExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc,
+ExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc,
                                                  tok::TokenKind Kind) {
   PredefinedExpr::IdentType IT;
 
@@ -1790,6 +1895,8 @@ Sema::OwningExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc,
   // string.
 
   Decl *currentDecl = getCurFunctionOrMethodDecl();
+  if (!currentDecl && getCurBlock())
+    currentDecl = getCurBlock()->TheDecl;
   if (!currentDecl) {
     Diag(Loc, diag::ext_predef_outside_function);
     currentDecl = Context.getTranslationUnitDecl();
@@ -1808,7 +1915,7 @@ Sema::OwningExprResult Sema::ActOnPredefinedExpr(SourceLocation Loc,
   return Owned(new (Context) PredefinedExpr(Loc, ResTy, IT));
 }
 
-Sema::OwningExprResult Sema::ActOnCharacterConstant(const Token &Tok) {
+ExprResult Sema::ActOnCharacterConstant(const Token &Tok) {
   llvm::SmallString<16> CharBuffer;
   bool Invalid = false;
   llvm::StringRef ThisTok = PP.getSpelling(Tok, CharBuffer, &Invalid);
@@ -1835,13 +1942,13 @@ Sema::OwningExprResult Sema::ActOnCharacterConstant(const Token &Tok) {
                                               Ty, Tok.getLocation()));
 }
 
-Action::OwningExprResult Sema::ActOnNumericConstant(const Token &Tok) {
+ExprResult Sema::ActOnNumericConstant(const Token &Tok) {
   // Fast path for a single digit (which is quite common).  A single digit
   // cannot have a trigraph, escaped newline, radix prefix, or type suffix.
   if (Tok.getLength() == 1) {
     const char Val = PP.getSpellingOfSingleCharacterNumericConstant(Tok);
     unsigned IntSize = Context.Target.getIntWidth();
-    return Owned(new (Context) IntegerLiteral(llvm::APInt(IntSize, Val-'0'),
+    return Owned(IntegerLiteral::Create(Context, llvm::APInt(IntSize, Val-'0'),
                     Context.IntTy, Tok.getLocation()));
   }
 
@@ -1899,7 +2006,7 @@ Action::OwningExprResult Sema::ActOnNumericConstant(const Token &Tok) {
     }
 
     bool isExact = (result == APFloat::opOK);
-    Res = new (Context) FloatingLiteral(Val, isExact, Ty, Tok.getLocation());
+    Res = FloatingLiteral::Create(Context, Val, isExact, Ty, Tok.getLocation());
 
   } else if (!Literal.isIntegerLiteral()) {
     return ExprError();
@@ -1986,7 +2093,7 @@ Action::OwningExprResult Sema::ActOnNumericConstant(const Token &Tok) {
       if (ResultVal.getBitWidth() != Width)
         ResultVal.trunc(Width);
     }
-    Res = new (Context) IntegerLiteral(ResultVal, Ty, Tok.getLocation());
+    Res = IntegerLiteral::Create(Context, ResultVal, Ty, Tok.getLocation());
   }
 
   // If this is an imaginary literal, create the ImaginaryLiteral wrapper.
@@ -1997,9 +2104,8 @@ Action::OwningExprResult Sema::ActOnNumericConstant(const Token &Tok) {
   return Owned(Res);
 }
 
-Action::OwningExprResult Sema::ActOnParenExpr(SourceLocation L,
-                                              SourceLocation R, ExprArg Val) {
-  Expr *E = Val.takeAs<Expr>();
+ExprResult Sema::ActOnParenExpr(SourceLocation L,
+                                              SourceLocation R, Expr *E) {
   assert((E != 0) && "ActOnParenExpr() missing expr");
   return Owned(new (Context) ParenExpr(L, R, E));
 }
@@ -2083,7 +2189,7 @@ bool Sema::CheckAlignOfExpr(Expr *E, SourceLocation OpLoc,
 }
 
 /// \brief Build a sizeof or alignof expression given a type operand.
-Action::OwningExprResult
+ExprResult
 Sema::CreateSizeOfAlignOfExpr(TypeSourceInfo *TInfo,
                               SourceLocation OpLoc,
                               bool isSizeOf, SourceRange R) {
@@ -2104,7 +2210,7 @@ Sema::CreateSizeOfAlignOfExpr(TypeSourceInfo *TInfo,
 
 /// \brief Build a sizeof or alignof expression given an expression
 /// operand.
-Action::OwningExprResult
+ExprResult
 Sema::CreateSizeOfAlignOfExpr(Expr *E, SourceLocation OpLoc,
                               bool isSizeOf, SourceRange R) {
   // Verify that the operand is valid.
@@ -2132,7 +2238,7 @@ Sema::CreateSizeOfAlignOfExpr(Expr *E, SourceLocation OpLoc,
 /// ActOnSizeOfAlignOfExpr - Handle @c sizeof(type) and @c sizeof @c expr and
 /// the same for @c alignof and @c __alignof
 /// Note that the ArgRange is invalid if isType is false.
-Action::OwningExprResult
+ExprResult
 Sema::ActOnSizeOfAlignOfExpr(SourceLocation OpLoc, bool isSizeof, bool isType,
                              void *TyOrEx, const SourceRange &ArgRange) {
   // If error parsing type, ignore.
@@ -2140,17 +2246,14 @@ Sema::ActOnSizeOfAlignOfExpr(SourceLocation OpLoc, bool isSizeof, bool isType,
 
   if (isType) {
     TypeSourceInfo *TInfo;
-    (void) GetTypeFromParser(TyOrEx, &TInfo);
+    (void) GetTypeFromParser(ParsedType::getFromOpaquePtr(TyOrEx), &TInfo);
     return CreateSizeOfAlignOfExpr(TInfo, OpLoc, isSizeof, ArgRange);
   }
 
   Expr *ArgEx = (Expr *)TyOrEx;
-  Action::OwningExprResult Result
+  ExprResult Result
     = CreateSizeOfAlignOfExpr(ArgEx, OpLoc, isSizeof, ArgEx->getSourceRange());
 
-  if (Result.isInvalid())
-    DeleteExpr(ArgEx);
-
   return move(Result);
 }
 
@@ -2174,32 +2277,31 @@ QualType Sema::CheckRealImagOperand(Expr *&V, SourceLocation Loc, bool isReal) {
 
 
 
-Action::OwningExprResult
+ExprResult
 Sema::ActOnPostfixUnaryOp(Scope *S, SourceLocation OpLoc,
-                          tok::TokenKind Kind, ExprArg Input) {
-  UnaryOperator::Opcode Opc;
+                          tok::TokenKind Kind, Expr *Input) {
+  UnaryOperatorKind Opc;
   switch (Kind) {
   default: assert(0 && "Unknown unary op!");
-  case tok::plusplus:   Opc = UnaryOperator::PostInc; break;
-  case tok::minusminus: Opc = UnaryOperator::PostDec; break;
+  case tok::plusplus:   Opc = UO_PostInc; break;
+  case tok::minusminus: Opc = UO_PostDec; break;
   }
 
-  return BuildUnaryOp(S, OpLoc, Opc, move(Input));
+  return BuildUnaryOp(S, OpLoc, Opc, Input);
 }
 
-Action::OwningExprResult
-Sema::ActOnArraySubscriptExpr(Scope *S, ExprArg Base, SourceLocation LLoc,
-                              ExprArg Idx, SourceLocation RLoc) {
+ExprResult
+Sema::ActOnArraySubscriptExpr(Scope *S, Expr *Base, SourceLocation LLoc,
+                              Expr *Idx, SourceLocation RLoc) {
   // Since this might be a postfix expression, get rid of ParenListExprs.
-  Base = MaybeConvertParenListExprToParenExpr(S, move(Base));
+  ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Base);
+  if (Result.isInvalid()) return ExprError();
+  Base = Result.take();
 
-  Expr *LHSExp = static_cast<Expr*>(Base.get()),
-       *RHSExp = static_cast<Expr*>(Idx.get());
+  Expr *LHSExp = Base, *RHSExp = Idx;
 
   if (getLangOptions().CPlusPlus &&
       (LHSExp->isTypeDependent() || RHSExp->isTypeDependent())) {
-    Base.release();
-    Idx.release();
     return Owned(new (Context) ArraySubscriptExpr(LHSExp, RHSExp,
                                                   Context.DependentTy, RLoc));
   }
@@ -2209,18 +2311,18 @@ Sema::ActOnArraySubscriptExpr(Scope *S, ExprArg Base, SourceLocation LLoc,
        LHSExp->getType()->isEnumeralType() ||
        RHSExp->getType()->isRecordType() ||
        RHSExp->getType()->isEnumeralType())) {
-    return CreateOverloadedArraySubscriptExpr(LLoc, RLoc, move(Base),move(Idx));
+    return CreateOverloadedArraySubscriptExpr(LLoc, RLoc, Base, Idx);
   }
 
-  return CreateBuiltinArraySubscriptExpr(move(Base), LLoc, move(Idx), RLoc);
+  return CreateBuiltinArraySubscriptExpr(Base, LLoc, Idx, RLoc);
 }
 
 
-Action::OwningExprResult
-Sema::CreateBuiltinArraySubscriptExpr(ExprArg Base, SourceLocation LLoc,
-                                     ExprArg Idx, SourceLocation RLoc) {
-  Expr *LHSExp = static_cast<Expr*>(Base.get());
-  Expr *RHSExp = static_cast<Expr*>(Idx.get());
+ExprResult
+Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
+                                     Expr *Idx, SourceLocation RLoc) {
+  Expr *LHSExp = Base;
+  Expr *RHSExp = Idx;
 
   // Perform default conversions.
   if (!LHSExp->getType()->getAs<VectorType>())
@@ -2274,7 +2376,7 @@ Sema::CreateBuiltinArraySubscriptExpr(ExprArg Base, SourceLocation LLoc,
     Diag(LHSExp->getLocStart(), diag::ext_subscript_non_lvalue) <<
         LHSExp->getSourceRange();
     ImpCastExprToType(LHSExp, Context.getArrayDecayedType(LHSTy),
-                      CastExpr::CK_ArrayToPointerDecay);
+                      CK_ArrayToPointerDecay);
     LHSTy = LHSExp->getType();
 
     BaseExpr = LHSExp;
@@ -2285,7 +2387,7 @@ Sema::CreateBuiltinArraySubscriptExpr(ExprArg Base, SourceLocation LLoc,
     Diag(RHSExp->getLocStart(), diag::ext_subscript_non_lvalue) <<
         RHSExp->getSourceRange();
     ImpCastExprToType(RHSExp, Context.getArrayDecayedType(RHSTy),
-                      CastExpr::CK_ArrayToPointerDecay);
+                      CK_ArrayToPointerDecay);
     RHSTy = RHSExp->getType();
 
     BaseExpr = RHSExp;
@@ -2296,8 +2398,7 @@ Sema::CreateBuiltinArraySubscriptExpr(ExprArg Base, SourceLocation LLoc,
        << LHSExp->getSourceRange() << RHSExp->getSourceRange());
   }
   // C99 6.5.2.1p1
-  if (!(IndexExpr->getType()->isIntegerType() &&
-        IndexExpr->getType()->isScalarType()) && !IndexExpr->isTypeDependent())
+  if (!IndexExpr->getType()->isIntegerType() && !IndexExpr->isTypeDependent())
     return ExprError(Diag(LLoc, diag::err_typecheck_subscript_not_integer)
                      << IndexExpr->getSourceRange());
 
@@ -2329,8 +2430,6 @@ Sema::CreateBuiltinArraySubscriptExpr(ExprArg Base, SourceLocation LLoc,
     return ExprError();
   }
 
-  Base.release();
-  Idx.release();
   return Owned(new (Context) ArraySubscriptExpr(LHSExp, RHSExp,
                                                 ResultType, RLoc));
 }
@@ -2377,7 +2476,7 @@ CheckExtVectorComponent(QualType baseType, SourceLocation OpLoc,
     // We didn't get to the end of the string. This means the component names
     // didn't come from the same set *or* we encountered an illegal name.
     Diag(OpLoc, diag::err_ext_vector_component_name_illegal)
-      << std::string(compStr,compStr+1) << SourceRange(CompLoc);
+      << llvm::StringRef(compStr, 1) << SourceRange(CompLoc);
     return QualType();
   }
 
@@ -2470,15 +2569,13 @@ static Decl *FindGetterNameDecl(const ObjCObjectPointerType *QIdTy,
   return GDecl;
 }
 
-Sema::OwningExprResult
-Sema::ActOnDependentMemberExpr(ExprArg Base, QualType BaseType,
+ExprResult
+Sema::ActOnDependentMemberExpr(Expr *BaseExpr, QualType BaseType,
                                bool IsArrow, SourceLocation OpLoc,
                                const CXXScopeSpec &SS,
                                NamedDecl *FirstQualifierInScope,
-                               DeclarationName Name, SourceLocation NameLoc,
+                               const DeclarationNameInfo &NameInfo,
                                const TemplateArgumentListInfo *TemplateArgs) {
-  Expr *BaseExpr = Base.takeAs<Expr>();
-
   // Even in dependent contexts, try to diagnose base expressions with
   // obviously wrong types, e.g.:
   //
@@ -2493,24 +2590,24 @@ Sema::ActOnDependentMemberExpr(ExprArg Base, QualType BaseType,
     if (PT && (!getLangOptions().ObjC1 ||
                PT->getPointeeType()->isRecordType())) {
       assert(BaseExpr && "cannot happen with implicit member accesses");
-      Diag(NameLoc, diag::err_typecheck_member_reference_struct_union)
+      Diag(NameInfo.getLoc(), diag::err_typecheck_member_reference_struct_union)
         << BaseType << BaseExpr->getSourceRange();
       return ExprError();
     }
   }
 
-  assert(BaseType->isDependentType() || Name.isDependentName() ||
+  assert(BaseType->isDependentType() ||
+         NameInfo.getName().isDependentName() ||
          isDependentScopeSpecifier(SS));
 
   // Get the type being accessed in BaseType.  If this is an arrow, the BaseExpr
   // must have pointer type, and the accessed type is the pointee.
   return Owned(CXXDependentScopeMemberExpr::Create(Context, BaseExpr, BaseType,
                                                    IsArrow, OpLoc,
-                 static_cast<NestedNameSpecifier*>(SS.getScopeRep()),
+                                                   SS.getScopeRep(),
                                                    SS.getRange(),
                                                    FirstQualifierInScope,
-                                                   Name, NameLoc,
-                                                   TemplateArgs));
+                                                   NameInfo, TemplateArgs));
 }
 
 /// We know that the given qualified member reference points only to
@@ -2562,12 +2659,15 @@ bool Sema::CheckQualifiedMemberReference(Expr *BaseExpr,
       return false;
 
     // Note that we use the DC of the decl, not the underlying decl.
-    CXXRecordDecl *RecordD = cast<CXXRecordDecl>((*I)->getDeclContext());
-    while (RecordD->isAnonymousStructOrUnion())
-      RecordD = cast<CXXRecordDecl>(RecordD->getParent());
+    DeclContext *DC = (*I)->getDeclContext();
+    while (DC->isTransparentContext())
+      DC = DC->getParent();
 
+    if (!DC->isRecord())
+      continue;
+    
     llvm::SmallPtrSet<CXXRecordDecl*,4> MemberRecord;
-    MemberRecord.insert(RecordD->getCanonicalDecl());
+    MemberRecord.insert(cast<CXXRecordDecl>(DC)->getCanonicalDecl());
 
     if (!IsProvablyNotDerivedFrom(*this, BaseRecord, MemberRecord))
       return false;
@@ -2646,24 +2746,21 @@ LookupMemberExprInRecord(Sema &SemaRef, LookupResult &R,
   return false;
 }
 
-Sema::OwningExprResult
-Sema::BuildMemberReferenceExpr(ExprArg BaseArg, QualType BaseType,
+ExprResult
+Sema::BuildMemberReferenceExpr(Expr *Base, QualType BaseType,
                                SourceLocation OpLoc, bool IsArrow,
                                CXXScopeSpec &SS,
                                NamedDecl *FirstQualifierInScope,
-                               DeclarationName Name, SourceLocation NameLoc,
+                               const DeclarationNameInfo &NameInfo,
                                const TemplateArgumentListInfo *TemplateArgs) {
-  Expr *Base = BaseArg.takeAs<Expr>();
-
   if (BaseType->isDependentType() ||
       (SS.isSet() && isDependentScopeSpecifier(SS)))
-    return ActOnDependentMemberExpr(ExprArg(*this, Base), BaseType,
+    return ActOnDependentMemberExpr(Base, BaseType,
                                     IsArrow, OpLoc,
                                     SS, FirstQualifierInScope,
-                                    Name, NameLoc,
-                                    TemplateArgs);
+                                    NameInfo, TemplateArgs);
 
-  LookupResult R(*this, Name, NameLoc, LookupMemberName);
+  LookupResult R(*this, NameInfo, LookupMemberName);
 
   // Implicit member accesses.
   if (!Base) {
@@ -2676,9 +2773,9 @@ Sema::BuildMemberReferenceExpr(ExprArg BaseArg, QualType BaseType,
 
   // Explicit member accesses.
   } else {
-    OwningExprResult Result =
+    ExprResult Result =
       LookupMemberExpr(R, Base, IsArrow, OpLoc,
-                       SS, /*ObjCImpDecl*/ DeclPtrTy(), TemplateArgs != 0);
+                       SS, /*ObjCImpDecl*/ 0, TemplateArgs != 0);
 
     if (Result.isInvalid()) {
       Owned(Base);
@@ -2692,20 +2789,19 @@ Sema::BuildMemberReferenceExpr(ExprArg BaseArg, QualType BaseType,
     BaseType = Base->getType();
   }
 
-  return BuildMemberReferenceExpr(ExprArg(*this, Base), BaseType,
+  return BuildMemberReferenceExpr(Base, BaseType,
                                   OpLoc, IsArrow, SS, FirstQualifierInScope,
                                   R, TemplateArgs);
 }
 
-Sema::OwningExprResult
-Sema::BuildMemberReferenceExpr(ExprArg Base, QualType BaseExprType,
+ExprResult
+Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
                                SourceLocation OpLoc, bool IsArrow,
                                const CXXScopeSpec &SS,
                                NamedDecl *FirstQualifierInScope,
                                LookupResult &R,
                          const TemplateArgumentListInfo *TemplateArgs,
                                bool SuppressQualifierCheck) {
-  Expr *BaseExpr = Base.takeAs<Expr>();
   QualType BaseType = BaseExprType;
   if (IsArrow) {
     assert(BaseType->isPointerType());
@@ -2713,10 +2809,10 @@ Sema::BuildMemberReferenceExpr(ExprArg Base, QualType BaseExprType,
   }
   R.setBaseObjectType(BaseType);
 
-  NestedNameSpecifier *Qualifier =
-    static_cast<NestedNameSpecifier*>(SS.getScopeRep());
-  DeclarationName MemberName = R.getLookupName();
-  SourceLocation MemberLoc = R.getNameLoc();
+  NestedNameSpecifier *Qualifier = SS.getScopeRep();
+  const DeclarationNameInfo &MemberNameInfo = R.getLookupNameInfo();
+  DeclarationName MemberName = MemberNameInfo.getName();
+  SourceLocation MemberLoc = MemberNameInfo.getLoc();
 
   if (R.isAmbiguous())
     return ExprError();
@@ -2765,7 +2861,7 @@ Sema::BuildMemberReferenceExpr(ExprArg Base, QualType BaseExprType,
                                      BaseExpr, BaseExprType,
                                      IsArrow, OpLoc,
                                      Qualifier, SS.getRange(),
-                                     MemberName, MemberLoc,
+                                     MemberNameInfo,
                                      TemplateArgs, R.begin(), R.end());
 
     return Owned(MemExpr);
@@ -2787,7 +2883,7 @@ Sema::BuildMemberReferenceExpr(ExprArg Base, QualType BaseExprType,
   if (!BaseExpr) {
     // If this is not an instance member, convert to a non-member access.
     if (!MemberDecl->isCXXInstanceMember())
-      return BuildDeclarationNameExpr(SS, R.getNameLoc(), MemberDecl);
+      return BuildDeclarationNameExpr(SS, R.getLookupNameInfo(), MemberDecl);
 
     SourceLocation Loc = R.getNameLoc();
     if (SS.getRange().isValid())
@@ -2838,34 +2934,36 @@ Sema::BuildMemberReferenceExpr(ExprArg Base, QualType BaseExprType,
     if (PerformObjectMemberConversion(BaseExpr, Qualifier, FoundDecl, FD))
       return ExprError();
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
-                                 FD, FoundDecl, MemberLoc, MemberType));
+                                 FD, FoundDecl, MemberNameInfo,
+                                 MemberType));
   }
 
   if (VarDecl *Var = dyn_cast<VarDecl>(MemberDecl)) {
     MarkDeclarationReferenced(MemberLoc, Var);
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
-                                 Var, FoundDecl, MemberLoc,
+                                 Var, FoundDecl, MemberNameInfo,
                                  Var->getType().getNonReferenceType()));
   }
 
   if (FunctionDecl *MemberFn = dyn_cast<FunctionDecl>(MemberDecl)) {
     MarkDeclarationReferenced(MemberLoc, MemberDecl);
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
-                                 MemberFn, FoundDecl, MemberLoc,
+                                 MemberFn, FoundDecl, MemberNameInfo,
                                  MemberFn->getType()));
   }
 
   if (EnumConstantDecl *Enum = dyn_cast<EnumConstantDecl>(MemberDecl)) {
     MarkDeclarationReferenced(MemberLoc, MemberDecl);
     return Owned(BuildMemberExpr(Context, BaseExpr, IsArrow, SS,
-                                 Enum, FoundDecl, MemberLoc, Enum->getType()));
+                                 Enum, FoundDecl, MemberNameInfo,
+                                 Enum->getType()));
   }
 
   Owned(BaseExpr);
 
   // We found something that we didn't expect. Complain.
   if (isa<TypeDecl>(MemberDecl))
-    Diag(MemberLoc,diag::err_typecheck_member_reference_type)
+    Diag(MemberLoc, diag::err_typecheck_member_reference_type)
       << MemberName << BaseType << int(IsArrow);
   else
     Diag(MemberLoc, diag::err_typecheck_member_reference_unknown)
@@ -2887,11 +2985,11 @@ Sema::BuildMemberReferenceExpr(ExprArg Base, QualType BaseExprType,
 ///
 /// The ObjCImpDecl bit is a gross hack that will need to be properly
 /// fixed for ObjC++.
-Sema::OwningExprResult
+ExprResult
 Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
                        bool &IsArrow, SourceLocation OpLoc,
                        CXXScopeSpec &SS,
-                       DeclPtrTy ObjCImpDecl, bool HasTemplateArgs) {
+                       Decl *ObjCImpDecl, bool HasTemplateArgs) {
   assert(BaseExpr && "no base expression");
 
   // Perform default conversions.
@@ -2921,8 +3019,8 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
           << QualType(Fun, 0)
           << FixItHint::CreateInsertion(Loc, "()");
 
-        OwningExprResult NewBase
-          = ActOnCallExpr(0, ExprArg(*this, BaseExpr), Loc,
+        ExprResult NewBase
+          = ActOnCallExpr(0, BaseExpr, Loc,
                           MultiExprArg(*this, 0, 0), 0, Loc);
         BaseExpr = 0;
         if (NewBase.isInvalid())
@@ -2952,7 +3050,7 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
     // is a reference to 'isa'.
     if (BaseType != Context.ObjCIdRedefinitionType) {
       BaseType = Context.ObjCIdRedefinitionType;
-      ImpCastExprToType(BaseExpr, BaseType, CastExpr::CK_BitCast);
+      ImpCastExprToType(BaseExpr, BaseType, CK_BitCast);
     }
   }
 
@@ -2963,7 +3061,7 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
     // is a reference to 'sel_id'.
     if (BaseType != Context.ObjCSelRedefinitionType) {
       BaseType = Context.ObjCSelRedefinitionType;
-      ImpCastExprToType(BaseExpr, BaseType, CastExpr::CK_BitCast);
+      ImpCastExprToType(BaseExpr, BaseType, CK_BitCast);
     }
   }
 
@@ -3022,7 +3120,7 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
   if (BaseType->isObjCClassType() &&
       BaseType != Context.ObjCClassRedefinitionType) {
     BaseType = Context.ObjCClassRedefinitionType;
-    ImpCastExprToType(BaseExpr, BaseType, CastExpr::CK_BitCast);
+    ImpCastExprToType(BaseExpr, BaseType, CK_BitCast);
   }
 
   if (IsArrow) {
@@ -3129,12 +3227,11 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
             // down the context as argument to this routine. Ideally, this context
             // need be passed down in the AST node and somehow calculated from the
             // AST for a function decl.
-            Decl *ImplDecl = ObjCImpDecl.getAs<Decl>();
             if (ObjCImplementationDecl *IMPD =
-                dyn_cast<ObjCImplementationDecl>(ImplDecl))
+                dyn_cast<ObjCImplementationDecl>(ObjCImpDecl))
               ClassOfMethodDecl = IMPD->getClassInterface();
             else if (ObjCCategoryImplDecl* CatImplClass =
-                        dyn_cast<ObjCCategoryImplDecl>(ImplDecl))
+                        dyn_cast<ObjCCategoryImplDecl>(ObjCImpDecl))
               ClassOfMethodDecl = CatImplClass->getClassInterface();
           }
 
@@ -3235,12 +3332,12 @@ Sema::LookupMemberExpr(LookupResult &R, Expr *&BaseExpr,
 /// \param ObjCImpDecl the current ObjC @implementation decl;
 ///   this is an ugly hack around the fact that ObjC @implementations
 ///   aren't properly put in the context chain
-Sema::OwningExprResult Sema::ActOnMemberAccessExpr(Scope *S, ExprArg BaseArg,
+ExprResult Sema::ActOnMemberAccessExpr(Scope *S, Expr *Base,
                                                    SourceLocation OpLoc,
                                                    tok::TokenKind OpKind,
                                                    CXXScopeSpec &SS,
                                                    UnqualifiedId &Id,
-                                                   DeclPtrTy ObjCImpDecl,
+                                                   Decl *ObjCImpDecl,
                                                    bool HasTrailingLParen) {
   if (SS.isSet() && SS.isInvalid())
     return ExprError();
@@ -3248,12 +3345,12 @@ Sema::OwningExprResult Sema::ActOnMemberAccessExpr(Scope *S, ExprArg BaseArg,
   TemplateArgumentListInfo TemplateArgsBuffer;
 
   // Decompose the name into its component parts.
-  DeclarationName Name;
-  SourceLocation NameLoc;
+  DeclarationNameInfo NameInfo;
   const TemplateArgumentListInfo *TemplateArgs;
   DecomposeUnqualifiedId(*this, Id, TemplateArgsBuffer,
-                         Name, NameLoc, TemplateArgs);
+                         NameInfo, TemplateArgs);
 
+  DeclarationName Name = NameInfo.getName();
   bool IsArrow = (OpKind == tok::arrow);
 
   NamedDecl *FirstQualifierInScope
@@ -3261,19 +3358,18 @@ Sema::OwningExprResult Sema::ActOnMemberAccessExpr(Scope *S, ExprArg BaseArg,
                        static_cast<NestedNameSpecifier*>(SS.getScopeRep())));
 
   // This is a postfix expression, so get rid of ParenListExprs.
-  BaseArg = MaybeConvertParenListExprToParenExpr(S, move(BaseArg));
+  ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Base);
+  if (Result.isInvalid()) return ExprError();
+  Base = Result.take();
 
-  Expr *Base = BaseArg.takeAs<Expr>();
-  OwningExprResult Result(*this);
   if (Base->getType()->isDependentType() || Name.isDependentName() ||
       isDependentScopeSpecifier(SS)) {
-    Result = ActOnDependentMemberExpr(ExprArg(*this, Base), Base->getType(),
+    Result = ActOnDependentMemberExpr(Base, Base->getType(),
                                       IsArrow, OpLoc,
                                       SS, FirstQualifierInScope,
-                                      Name, NameLoc,
-                                      TemplateArgs);
+                                      NameInfo, TemplateArgs);
   } else {
-    LookupResult R(*this, Name, NameLoc, LookupMemberName);
+    LookupResult R(*this, NameInfo, LookupMemberName);
     Result = LookupMemberExpr(R, Base, IsArrow, OpLoc,
                               SS, ObjCImpDecl, TemplateArgs != 0);
 
@@ -3289,12 +3385,12 @@ Sema::OwningExprResult Sema::ActOnMemberAccessExpr(Scope *S, ExprArg BaseArg,
       // call now.
       if (!HasTrailingLParen &&
           Id.getKind() == UnqualifiedId::IK_DestructorName)
-        return DiagnoseDtorReference(NameLoc, move(Result));
+        return DiagnoseDtorReference(NameInfo.getLoc(), Result.get());
 
       return move(Result);
     }
 
-    Result = BuildMemberReferenceExpr(ExprArg(*this, Base), Base->getType(),
+    Result = BuildMemberReferenceExpr(Base, Base->getType(),
                                       OpLoc, IsArrow, SS, FirstQualifierInScope,
                                       R, TemplateArgs);
   }
@@ -3302,7 +3398,7 @@ Sema::OwningExprResult Sema::ActOnMemberAccessExpr(Scope *S, ExprArg BaseArg,
   return move(Result);
 }
 
-Sema::OwningExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
+ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
                                                     FunctionDecl *FD,
                                                     ParmVarDecl *Param) {
   if (Param->hasUnparsedDefaultArg()) {
@@ -3324,7 +3420,7 @@ Sema::OwningExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
       InstantiatingTemplate Inst(*this, CallLoc, Param, Innermost.first,
                                  Innermost.second);
 
-      OwningExprResult Result = SubstExpr(UninstExpr, ArgList);
+      ExprResult Result = SubstExpr(UninstExpr, ArgList);
       if (Result.isInvalid())
         return ExprError();
 
@@ -3338,7 +3434,7 @@ Sema::OwningExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
 
       InitializationSequence InitSeq(*this, Entity, Kind, &ResultE, 1);
       Result = InitSeq.Perform(*this, Entity, Kind,
-                               MultiExprArg(*this, (void**)&ResultE, 1));
+                               MultiExprArg(*this, &ResultE, 1));
       if (Result.isInvalid())
         return ExprError();
 
@@ -3457,7 +3553,7 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
       InitializedEntity Entity =
         Param? InitializedEntity::InitializeParameter(Param)
              : InitializedEntity::InitializeParameter(ProtoArgType);
-      OwningExprResult ArgE = PerformCopyInitialization(Entity,
+      ExprResult ArgE = PerformCopyInitialization(Entity,
                                                         SourceLocation(),
                                                         Owned(Arg));
       if (ArgE.isInvalid())
@@ -3467,7 +3563,7 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
     } else {
       ParmVarDecl *Param = FDecl->getParamDecl(i);
 
-      OwningExprResult ArgExpr =
+      ExprResult ArgExpr =
         BuildCXXDefaultArgExpr(CallLoc, FDecl, Param);
       if (ArgExpr.isInvalid())
         return true;
@@ -3492,18 +3588,18 @@ bool Sema::GatherArgumentsForCall(SourceLocation CallLoc,
 /// ActOnCallExpr - Handle a call to Fn with the specified array of arguments.
 /// This provides the location of the left/right parens and a list of comma
 /// locations.
-Action::OwningExprResult
-Sema::ActOnCallExpr(Scope *S, ExprArg fn, SourceLocation LParenLoc,
+ExprResult
+Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
                     MultiExprArg args,
                     SourceLocation *CommaLocs, SourceLocation RParenLoc) {
   unsigned NumArgs = args.size();
 
   // Since this might be a postfix expression, get rid of ParenListExprs.
-  fn = MaybeConvertParenListExprToParenExpr(S, move(fn));
+  ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Fn);
+  if (Result.isInvalid()) return ExprError();
+  Fn = Result.take();
 
-  Expr *Fn = fn.takeAs<Expr>();
-  Expr **Args = reinterpret_cast<Expr**>(args.release());
-  assert(Fn && "no function call expression");
+  Expr **Args = args.release();
 
   if (getLangOptions().CPlusPlus) {
     // If this is a pseudo-destructor expression, build the call immediately.
@@ -3515,9 +3611,6 @@ Sema::ActOnCallExpr(Scope *S, ExprArg fn, SourceLocation LParenLoc,
                                     SourceRange(Args[0]->getLocStart(),
                                                 Args[NumArgs-1]->getLocEnd()));
 
-        for (unsigned I = 0; I != NumArgs; ++I)
-          Args[I]->Destroy(Context);
-
         NumArgs = 0;
       }
 
@@ -3572,27 +3665,27 @@ Sema::ActOnCallExpr(Scope *S, ExprArg fn, SourceLocation LParenLoc,
 
     // Determine whether this is a call to a pointer-to-member function.
     if (BinaryOperator *BO = dyn_cast<BinaryOperator>(NakedFn)) {
-      if (BO->getOpcode() == BinaryOperator::PtrMemD ||
-          BO->getOpcode() == BinaryOperator::PtrMemI) {
+      if (BO->getOpcode() == BO_PtrMemD ||
+          BO->getOpcode() == BO_PtrMemI) {
         if (const FunctionProtoType *FPT
                                 = BO->getType()->getAs<FunctionProtoType>()) {
           QualType ResultTy = FPT->getCallResultType(Context);
 
-          ExprOwningPtr<CXXMemberCallExpr>
-            TheCall(this, new (Context) CXXMemberCallExpr(Context, BO, Args,
-                                                          NumArgs, ResultTy,
-                                                          RParenLoc));
+          CXXMemberCallExpr *TheCall
+            = new (Context) CXXMemberCallExpr(Context, BO, Args,
+                                              NumArgs, ResultTy,
+                                              RParenLoc);
 
           if (CheckCallReturnType(FPT->getResultType(),
                                   BO->getRHS()->getSourceRange().getBegin(),
-                                  TheCall.get(), 0))
+                                  TheCall, 0))
             return ExprError();
 
-          if (ConvertArgumentsForCall(&*TheCall, BO, 0, FPT, Args, NumArgs,
+          if (ConvertArgumentsForCall(TheCall, BO, 0, FPT, Args, NumArgs,
                                       RParenLoc))
             return ExprError();
 
-          return Owned(MaybeBindToTemporary(TheCall.release()).release());
+          return MaybeBindToTemporary(TheCall);
         }
         return ExprError(Diag(Fn->getLocStart(),
                               diag::err_typecheck_call_not_function)
@@ -3625,7 +3718,7 @@ Sema::ActOnCallExpr(Scope *S, ExprArg fn, SourceLocation LParenLoc,
 /// block-pointer type.
 ///
 /// \param NDecl the declaration being called, if available
-Sema::OwningExprResult
+ExprResult
 Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
                             SourceLocation LParenLoc,
                             Expr **Args, unsigned NumArgs,
@@ -3637,10 +3730,10 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
 
   // Make the call expr early, before semantic checks.  This guarantees cleanup
   // of arguments and function on error.
-  ExprOwningPtr<CallExpr> TheCall(this, new (Context) CallExpr(Context, Fn,
-                                                               Args, NumArgs,
-                                                               Context.BoolTy,
-                                                               RParenLoc));
+  CallExpr *TheCall = new (Context) CallExpr(Context, Fn,
+                                             Args, NumArgs,
+                                             Context.BoolTy,
+                                             RParenLoc);
 
   const FunctionType *FuncT;
   if (!Fn->getType()->isBlockPointerType()) {
@@ -3661,7 +3754,7 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
 
   // Check for a valid return type
   if (CheckCallReturnType(FuncT->getResultType(),
-                          Fn->getSourceRange().getBegin(), TheCall.get(),
+                          Fn->getSourceRange().getBegin(), TheCall,
                           FDecl))
     return ExprError();
 
@@ -3669,7 +3762,7 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
   TheCall->setType(FuncT->getCallResultType(Context));
 
   if (const FunctionProtoType *Proto = dyn_cast<FunctionProtoType>(FuncT)) {
-    if (ConvertArgumentsForCall(&*TheCall, Fn, FDecl, Proto, Args, NumArgs,
+    if (ConvertArgumentsForCall(TheCall, Fn, FDecl, Proto, Args, NumArgs,
                                 RParenLoc))
       return ExprError();
   } else {
@@ -3713,22 +3806,22 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
 
   // Do special checking on direct calls to functions.
   if (FDecl) {
-    if (CheckFunctionCall(FDecl, TheCall.get()))
+    if (CheckFunctionCall(FDecl, TheCall))
       return ExprError();
 
     if (unsigned BuiltinID = FDecl->getBuiltinID())
-      return CheckBuiltinFunctionCall(BuiltinID, TheCall.take());
+      return CheckBuiltinFunctionCall(BuiltinID, TheCall);
   } else if (NDecl) {
-    if (CheckBlockCall(NDecl, TheCall.get()))
+    if (CheckBlockCall(NDecl, TheCall))
       return ExprError();
   }
 
-  return MaybeBindToTemporary(TheCall.take());
+  return MaybeBindToTemporary(TheCall);
 }
 
-Action::OwningExprResult
-Sema::ActOnCompoundLiteral(SourceLocation LParenLoc, TypeTy *Ty,
-                           SourceLocation RParenLoc, ExprArg InitExpr) {
+ExprResult
+Sema::ActOnCompoundLiteral(SourceLocation LParenLoc, ParsedType Ty,
+                           SourceLocation RParenLoc, Expr *InitExpr) {
   assert((Ty != 0) && "ActOnCompoundLiteral(): missing type");
   // FIXME: put back this assert when initializers are worked out.
   //assert((InitExpr != 0) && "ActOnCompoundLiteral(): missing expression");
@@ -3738,14 +3831,13 @@ Sema::ActOnCompoundLiteral(SourceLocation LParenLoc, TypeTy *Ty,
   if (!TInfo)
     TInfo = Context.getTrivialTypeSourceInfo(literalType);
 
-  return BuildCompoundLiteralExpr(LParenLoc, TInfo, RParenLoc, move(InitExpr));
+  return BuildCompoundLiteralExpr(LParenLoc, TInfo, RParenLoc, InitExpr);
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
-                               SourceLocation RParenLoc, ExprArg InitExpr) {
+                               SourceLocation RParenLoc, Expr *literalExpr) {
   QualType literalType = TInfo->getType();
-  Expr *literalExpr = static_cast<Expr*>(InitExpr.get());
 
   if (literalType->isArrayType()) {
     if (literalType->isVariableArrayType())
@@ -3764,13 +3856,12 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
     = InitializationKind::CreateCast(SourceRange(LParenLoc, RParenLoc),
                                      /*IsCStyleCast=*/true);
   InitializationSequence InitSeq(*this, Entity, Kind, &literalExpr, 1);
-  OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                   MultiExprArg(*this, (void**)&literalExpr, 1),
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
+                                       MultiExprArg(*this, &literalExpr, 1),
                                             &literalType);
   if (Result.isInvalid())
     return ExprError();
-  InitExpr.release();
-  literalExpr = static_cast<Expr*>(Result.get());
+  literalExpr = Result.get();
 
   bool isFileScope = getCurFunctionOrMethodDecl() == 0;
   if (isFileScope) { // 6.5.2.5p3
@@ -3778,17 +3869,15 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
       return ExprError();
   }
 
-  Result.release();
-
   return Owned(new (Context) CompoundLiteralExpr(LParenLoc, TInfo, literalType,
                                                  literalExpr, isFileScope));
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg initlist,
                     SourceLocation RBraceLoc) {
   unsigned NumInit = initlist.size();
-  Expr **InitList = reinterpret_cast<Expr**>(initlist.release());
+  Expr **InitList = initlist.release();
 
   // Semantic analysis for initializers is done by ActOnDeclarator() and
   // CheckInitializer() - it requires knowledge of the object being intialized.
@@ -3799,45 +3888,45 @@ Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg initlist,
   return Owned(E);
 }
 
-static CastExpr::CastKind getScalarCastKind(ASTContext &Context,
+static CastKind getScalarCastKind(ASTContext &Context,
                                             QualType SrcTy, QualType DestTy) {
   if (Context.hasSameUnqualifiedType(SrcTy, DestTy))
-    return CastExpr::CK_NoOp;
+    return CK_NoOp;
 
   if (SrcTy->hasPointerRepresentation()) {
     if (DestTy->hasPointerRepresentation())
       return DestTy->isObjCObjectPointerType() ?
-                CastExpr::CK_AnyPointerToObjCPointerCast :
-                CastExpr::CK_BitCast;
+                CK_AnyPointerToObjCPointerCast :
+                CK_BitCast;
     if (DestTy->isIntegerType())
-      return CastExpr::CK_PointerToIntegral;
+      return CK_PointerToIntegral;
   }
 
   if (SrcTy->isIntegerType()) {
     if (DestTy->isIntegerType())
-      return CastExpr::CK_IntegralCast;
+      return CK_IntegralCast;
     if (DestTy->hasPointerRepresentation())
-      return CastExpr::CK_IntegralToPointer;
+      return CK_IntegralToPointer;
     if (DestTy->isRealFloatingType())
-      return CastExpr::CK_IntegralToFloating;
+      return CK_IntegralToFloating;
   }
 
   if (SrcTy->isRealFloatingType()) {
     if (DestTy->isRealFloatingType())
-      return CastExpr::CK_FloatingCast;
+      return CK_FloatingCast;
     if (DestTy->isIntegerType())
-      return CastExpr::CK_FloatingToIntegral;
+      return CK_FloatingToIntegral;
   }
 
   // FIXME: Assert here.
   // assert(false && "Unhandled cast combination!");
-  return CastExpr::CK_Unknown;
+  return CK_Unknown;
 }
 
 /// CheckCastTypes - Check type constraints for casting between types.
 bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
-                          CastExpr::CastKind& Kind,
-                          CXXBaseSpecifierArray &BasePath,
+                          CastKind& Kind,
+                          CXXCastPath &BasePath,
                           bool FunctionalStyle) {
   if (getLangOptions().CPlusPlus)
     return CXXCheckCStyleCast(TyR, castType, castExpr, Kind, BasePath,
@@ -3849,10 +3938,14 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
   // type needs to be scalar.
   if (castType->isVoidType()) {
     // Cast to void allows any expr type.
-    Kind = CastExpr::CK_ToVoid;
+    Kind = CK_ToVoid;
     return false;
   }
 
+  if (RequireCompleteType(TyR.getBegin(), castType,
+                          diag::err_typecheck_cast_to_incomplete))
+    return true;
+
   if (!castType->isScalarType() && !castType->isVectorType()) {
     if (Context.hasSameUnqualifiedType(castType, castExpr->getType()) &&
         (castType->isStructureType() || castType->isUnionType())) {
@@ -3860,7 +3953,7 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
       // FIXME: Check that the cast destination type is complete.
       Diag(TyR.getBegin(), diag::ext_typecheck_cast_nonscalar)
         << castType << castExpr->getSourceRange();
-      Kind = CastExpr::CK_NoOp;
+      Kind = CK_NoOp;
       return false;
     }
 
@@ -3880,7 +3973,7 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
       if (Field == FieldEnd)
         return Diag(TyR.getBegin(), diag::err_typecheck_cast_to_union_no_type)
           << castExpr->getType() << castExpr->getSourceRange();
-      Kind = CastExpr::CK_ToUnion;
+      Kind = CK_ToUnion;
       return false;
     }
 
@@ -3922,11 +4015,15 @@ bool Sema::CheckCastTypes(SourceRange TyR, QualType castType, Expr *&castExpr,
   }
 
   Kind = getScalarCastKind(Context, castExpr->getType(), castType);
+
+  if (Kind == CK_Unknown || Kind == CK_BitCast)
+    CheckCastAlign(castExpr, castType, TyR);
+
   return false;
 }
 
 bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
-                           CastExpr::CastKind &Kind) {
+                           CastKind &Kind) {
   assert(VectorTy->isVectorType() && "Not a vector type!");
 
   if (Ty->isVectorType() || Ty->isIntegerType()) {
@@ -3941,12 +4038,12 @@ bool Sema::CheckVectorCast(SourceRange R, QualType VectorTy, QualType Ty,
                 diag::err_invalid_conversion_between_vector_and_scalar)
       << VectorTy << Ty << R;
 
-  Kind = CastExpr::CK_BitCast;
+  Kind = CK_BitCast;
   return false;
 }
 
 bool Sema::CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *&CastExpr,
-                              CastExpr::CastKind &Kind) {
+                              CastKind &Kind) {
   assert(DestTy->isExtVectorType() && "Not an extended vector type!");
 
   QualType SrcTy = CastExpr->getType();
@@ -3957,7 +4054,7 @@ bool Sema::CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *&CastExpr,
     if (Context.getTypeSize(DestTy) != Context.getTypeSize(SrcTy))
       return Diag(R.getBegin(),diag::err_invalid_conversion_between_ext_vectors)
         << DestTy << SrcTy << R;
-    Kind = CastExpr::CK_BitCast;
+    Kind = CK_BitCast;
     return false;
   }
 
@@ -3973,14 +4070,14 @@ bool Sema::CheckExtVectorCast(SourceRange R, QualType DestTy, Expr *&CastExpr,
   ImpCastExprToType(CastExpr, DestElemTy,
                     getScalarCastKind(Context, SrcTy, DestElemTy));
 
-  Kind = CastExpr::CK_VectorSplat;
+  Kind = CK_VectorSplat;
   return false;
 }
 
-Action::OwningExprResult
-Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc, TypeTy *Ty,
-                    SourceLocation RParenLoc, ExprArg Op) {
-  assert((Ty != 0) && (Op.get() != 0) &&
+ExprResult
+Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc, ParsedType Ty,
+                    SourceLocation RParenLoc, Expr *castExpr) {
+  assert((Ty != 0) && (castExpr != 0) &&
          "ActOnCastExpr(): missing type or expr");
 
   TypeSourceInfo *castTInfo;
@@ -3989,55 +4086,52 @@ Sema::ActOnCastExpr(Scope *S, SourceLocation LParenLoc, TypeTy *Ty,
     castTInfo = Context.getTrivialTypeSourceInfo(castType);
 
   // If the Expr being casted is a ParenListExpr, handle it specially.
-  Expr *castExpr = (Expr *)Op.get();
   if (isa<ParenListExpr>(castExpr))
-    return ActOnCastOfParenListExpr(S, LParenLoc, RParenLoc, move(Op),
+    return ActOnCastOfParenListExpr(S, LParenLoc, RParenLoc, castExpr,
                                     castTInfo);
 
-  return BuildCStyleCastExpr(LParenLoc, castTInfo, RParenLoc, move(Op));
+  return BuildCStyleCastExpr(LParenLoc, castTInfo, RParenLoc, castExpr);
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::BuildCStyleCastExpr(SourceLocation LParenLoc, TypeSourceInfo *Ty,
-                          SourceLocation RParenLoc, ExprArg Op) {
-  Expr *castExpr = static_cast<Expr*>(Op.get());
-
-  CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-  CXXBaseSpecifierArray BasePath;
+                          SourceLocation RParenLoc, Expr *castExpr) {
+  CastKind Kind = CK_Unknown;
+  CXXCastPath BasePath;
   if (CheckCastTypes(SourceRange(LParenLoc, RParenLoc), Ty->getType(), castExpr,
                      Kind, BasePath))
     return ExprError();
 
-  Op.release();
-  return Owned(new (Context) CStyleCastExpr(
+  return Owned(CStyleCastExpr::Create(Context,
                                     Ty->getType().getNonLValueExprType(Context),
-                                            Kind, castExpr, BasePath, Ty,
-                                            LParenLoc, RParenLoc));
+                                      Kind, castExpr, &BasePath, Ty,
+                                      LParenLoc, RParenLoc));
 }
 
 /// This is not an AltiVec-style cast, so turn the ParenListExpr into a sequence
 /// of comma binary operators.
-Action::OwningExprResult
-Sema::MaybeConvertParenListExprToParenExpr(Scope *S, ExprArg EA) {
-  Expr *expr = EA.takeAs<Expr>();
+ExprResult
+Sema::MaybeConvertParenListExprToParenExpr(Scope *S, Expr *expr) {
   ParenListExpr *E = dyn_cast<ParenListExpr>(expr);
   if (!E)
     return Owned(expr);
 
-  OwningExprResult Result(*this, E->getExpr(0));
+  ExprResult Result(E->getExpr(0));
 
   for (unsigned i = 1, e = E->getNumExprs(); i != e && !Result.isInvalid(); ++i)
-    Result = ActOnBinOp(S, E->getExprLoc(), tok::comma, move(Result),
-                        Owned(E->getExpr(i)));
+    Result = ActOnBinOp(S, E->getExprLoc(), tok::comma, Result.get(),
+                        E->getExpr(i));
 
-  return ActOnParenExpr(E->getLParenLoc(), E->getRParenLoc(), move(Result));
+  if (Result.isInvalid()) return ExprError();
+
+  return ActOnParenExpr(E->getLParenLoc(), E->getRParenLoc(), Result.get());
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCastOfParenListExpr(Scope *S, SourceLocation LParenLoc,
-                               SourceLocation RParenLoc, ExprArg Op,
+                               SourceLocation RParenLoc, Expr *Op,
                                TypeSourceInfo *TInfo) {
-  ParenListExpr *PE = (ParenListExpr *)Op.get();
+  ParenListExpr *PE = cast<ParenListExpr>(Op);
   QualType Ty = TInfo->getType();
   bool isAltiVecLiteral = false;
 
@@ -4065,24 +4159,24 @@ Sema::ActOnCastOfParenListExpr(Scope *S, SourceLocation LParenLoc,
 
     // FIXME: This means that pretty-printing the final AST will produce curly
     // braces instead of the original commas.
-    Op.release();
     InitListExpr *E = new (Context) InitListExpr(Context, LParenLoc,
                                                  &initExprs[0],
                                                  initExprs.size(), RParenLoc);
     E->setType(Ty);
-    return BuildCompoundLiteralExpr(LParenLoc, TInfo, RParenLoc, Owned(E));
+    return BuildCompoundLiteralExpr(LParenLoc, TInfo, RParenLoc, E);
   } else {
     // This is not an AltiVec-style cast, so turn the ParenListExpr into a
     // sequence of BinOp comma operators.
-    Op = MaybeConvertParenListExprToParenExpr(S, move(Op));
-    return BuildCStyleCastExpr(LParenLoc, TInfo, RParenLoc, move(Op));
+    ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Op);
+    if (Result.isInvalid()) return ExprError();
+    return BuildCStyleCastExpr(LParenLoc, TInfo, RParenLoc, Result.take());
   }
 }
 
-Action::OwningExprResult Sema::ActOnParenOrParenListExpr(SourceLocation L,
+ExprResult Sema::ActOnParenOrParenListExpr(SourceLocation L,
                                                   SourceLocation R,
                                                   MultiExprArg Val,
-                                                  TypeTy *TypeOfCast) {
+                                                  ParsedType TypeOfCast) {
   unsigned nexprs = Val.size();
   Expr **exprs = reinterpret_cast<Expr**>(Val.release());
   assert((exprs != 0) && "ActOnParenOrParenListExpr() missing expr list");
@@ -4148,8 +4242,8 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     if (!RHSTy->isVoidType())
       Diag(LHS->getLocStart(), diag::ext_typecheck_cond_one_void)
         << LHS->getSourceRange();
-    ImpCastExprToType(LHS, Context.VoidTy, CastExpr::CK_ToVoid);
-    ImpCastExprToType(RHS, Context.VoidTy, CastExpr::CK_ToVoid);
+    ImpCastExprToType(LHS, Context.VoidTy, CK_ToVoid);
+    ImpCastExprToType(RHS, Context.VoidTy, CK_ToVoid);
     return Context.VoidTy;
   }
   // C99 6.5.15p6 - "if one operand is a null pointer constant, the result has
@@ -4157,12 +4251,12 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
   if ((LHSTy->isAnyPointerType() || LHSTy->isBlockPointerType()) &&
       RHS->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
     // promote the null to a pointer.
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_Unknown);
+    ImpCastExprToType(RHS, LHSTy, CK_Unknown);
     return LHSTy;
   }
   if ((RHSTy->isAnyPointerType() || RHSTy->isBlockPointerType()) &&
       LHS->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
-    ImpCastExprToType(LHS, RHSTy, CastExpr::CK_Unknown);
+    ImpCastExprToType(LHS, RHSTy, CK_Unknown);
     return RHSTy;
   }
 
@@ -4178,8 +4272,8 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     if (!LHSTy->isBlockPointerType() || !RHSTy->isBlockPointerType()) {
       if (LHSTy->isVoidPointerType() || RHSTy->isVoidPointerType()) {
         QualType destType = Context.getPointerType(Context.VoidTy);
-        ImpCastExprToType(LHS, destType, CastExpr::CK_BitCast);
-        ImpCastExprToType(RHS, destType, CastExpr::CK_BitCast);
+        ImpCastExprToType(LHS, destType, CK_BitCast);
+        ImpCastExprToType(RHS, destType, CK_BitCast);
         return destType;
       }
       Diag(QuestionLoc, diag::err_typecheck_cond_incompatible_operands)
@@ -4203,13 +4297,13 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
       // reason, but this is what gcc does, and we do have to pick
       // to get a consistent AST.
       QualType incompatTy = Context.getPointerType(Context.VoidTy);
-      ImpCastExprToType(LHS, incompatTy, CastExpr::CK_BitCast);
-      ImpCastExprToType(RHS, incompatTy, CastExpr::CK_BitCast);
+      ImpCastExprToType(LHS, incompatTy, CK_BitCast);
+      ImpCastExprToType(RHS, incompatTy, CK_BitCast);
       return incompatTy;
     }
     // The block pointer types are compatible.
-    ImpCastExprToType(LHS, LHSTy, CastExpr::CK_BitCast);
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, LHSTy, CK_BitCast);
+    ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
 
@@ -4226,9 +4320,9 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
         = Context.getQualifiedType(lhptee, rhptee.getQualifiers());
       QualType destType = Context.getPointerType(destPointee);
       // Add qualifiers if necessary.
-      ImpCastExprToType(LHS, destType, CastExpr::CK_NoOp);
+      ImpCastExprToType(LHS, destType, CK_NoOp);
       // Promote to void*.
-      ImpCastExprToType(RHS, destType, CastExpr::CK_BitCast);
+      ImpCastExprToType(RHS, destType, CK_BitCast);
       return destType;
     }
     if (rhptee->isVoidType() && lhptee->isIncompleteOrObjectType()) {
@@ -4236,9 +4330,9 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
         = Context.getQualifiedType(rhptee, lhptee.getQualifiers());
       QualType destType = Context.getPointerType(destPointee);
       // Add qualifiers if necessary.
-      ImpCastExprToType(RHS, destType, CastExpr::CK_NoOp);
+      ImpCastExprToType(RHS, destType, CK_NoOp);
       // Promote to void*.
-      ImpCastExprToType(LHS, destType, CastExpr::CK_BitCast);
+      ImpCastExprToType(LHS, destType, CK_BitCast);
       return destType;
     }
 
@@ -4254,8 +4348,8 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
       // reason, but this is what gcc does, and we do have to pick
       // to get a consistent AST.
       QualType incompatTy = Context.getPointerType(Context.VoidTy);
-      ImpCastExprToType(LHS, incompatTy, CastExpr::CK_BitCast);
-      ImpCastExprToType(RHS, incompatTy, CastExpr::CK_BitCast);
+      ImpCastExprToType(LHS, incompatTy, CK_BitCast);
+      ImpCastExprToType(RHS, incompatTy, CK_BitCast);
       return incompatTy;
     }
     // The pointer types are compatible.
@@ -4265,8 +4359,8 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     // type.
     // FIXME: Need to calculate the composite type.
     // FIXME: Need to add qualifiers
-    ImpCastExprToType(LHS, LHSTy, CastExpr::CK_BitCast);
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, LHSTy, CK_BitCast);
+    ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
 
@@ -4274,13 +4368,13 @@ QualType Sema::CheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
   if (RHSTy->isPointerType() && LHSTy->isIntegerType()) {
     Diag(QuestionLoc, diag::warn_typecheck_cond_pointer_integer_mismatch)
       << LHSTy << RHSTy << LHS->getSourceRange() << RHS->getSourceRange();
-    ImpCastExprToType(LHS, RHSTy, CastExpr::CK_IntegralToPointer);
+    ImpCastExprToType(LHS, RHSTy, CK_IntegralToPointer);
     return RHSTy;
   }
   if (LHSTy->isPointerType() && RHSTy->isIntegerType()) {
     Diag(QuestionLoc, diag::warn_typecheck_cond_pointer_integer_mismatch)
       << LHSTy << RHSTy << LHS->getSourceRange() << RHS->getSourceRange();
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_IntegralToPointer);
+    ImpCastExprToType(RHS, LHSTy, CK_IntegralToPointer);
     return LHSTy;
   }
 
@@ -4302,34 +4396,34 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
   // redefinition type if an attempt is made to access its fields.
   if (LHSTy->isObjCClassType() &&
       (RHSTy.getDesugaredType() == Context.ObjCClassRedefinitionType)) {
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (RHSTy->isObjCClassType() &&
       (LHSTy.getDesugaredType() == Context.ObjCClassRedefinitionType)) {
-    ImpCastExprToType(LHS, RHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, RHSTy, CK_BitCast);
     return RHSTy;
   }
   // And the same for struct objc_object* / id
   if (LHSTy->isObjCIdType() &&
       (RHSTy.getDesugaredType() == Context.ObjCIdRedefinitionType)) {
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (RHSTy->isObjCIdType() &&
       (LHSTy.getDesugaredType() == Context.ObjCIdRedefinitionType)) {
-    ImpCastExprToType(LHS, RHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, RHSTy, CK_BitCast);
     return RHSTy;
   }
   // And the same for struct objc_selector* / SEL
   if (Context.isObjCSelType(LHSTy) &&
       (RHSTy.getDesugaredType() == Context.ObjCSelRedefinitionType)) {
-    ImpCastExprToType(RHS, LHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(RHS, LHSTy, CK_BitCast);
     return LHSTy;
   }
   if (Context.isObjCSelType(RHSTy) &&
       (LHSTy.getDesugaredType() == Context.ObjCSelRedefinitionType)) {
-    ImpCastExprToType(LHS, RHSTy, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, RHSTy, CK_BitCast);
     return RHSTy;
   }
   // Check constraints for Objective-C object pointers types.
@@ -4378,13 +4472,13 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
       << LHSTy << RHSTy
       << LHS->getSourceRange() << RHS->getSourceRange();
       QualType incompatTy = Context.getObjCIdType();
-      ImpCastExprToType(LHS, incompatTy, CastExpr::CK_BitCast);
-      ImpCastExprToType(RHS, incompatTy, CastExpr::CK_BitCast);
+      ImpCastExprToType(LHS, incompatTy, CK_BitCast);
+      ImpCastExprToType(RHS, incompatTy, CK_BitCast);
       return incompatTy;
     }
     // The object pointer types are compatible.
-    ImpCastExprToType(LHS, compositeType, CastExpr::CK_BitCast);
-    ImpCastExprToType(RHS, compositeType, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, compositeType, CK_BitCast);
+    ImpCastExprToType(RHS, compositeType, CK_BitCast);
     return compositeType;
   }
   // Check Objective-C object pointer types and 'void *'
@@ -4395,9 +4489,9 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
     = Context.getQualifiedType(lhptee, rhptee.getQualifiers());
     QualType destType = Context.getPointerType(destPointee);
     // Add qualifiers if necessary.
-    ImpCastExprToType(LHS, destType, CastExpr::CK_NoOp);
+    ImpCastExprToType(LHS, destType, CK_NoOp);
     // Promote to void*.
-    ImpCastExprToType(RHS, destType, CastExpr::CK_BitCast);
+    ImpCastExprToType(RHS, destType, CK_BitCast);
     return destType;
   }
   if (LHSTy->isObjCObjectPointerType() && RHSTy->isVoidPointerType()) {
@@ -4407,9 +4501,9 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
     = Context.getQualifiedType(rhptee, lhptee.getQualifiers());
     QualType destType = Context.getPointerType(destPointee);
     // Add qualifiers if necessary.
-    ImpCastExprToType(RHS, destType, CastExpr::CK_NoOp);
+    ImpCastExprToType(RHS, destType, CK_NoOp);
     // Promote to void*.
-    ImpCastExprToType(LHS, destType, CastExpr::CK_BitCast);
+    ImpCastExprToType(LHS, destType, CK_BitCast);
     return destType;
   }
   return QualType();
@@ -4417,30 +4511,27 @@ QualType Sema::FindCompositeObjCPointerType(Expr *&LHS, Expr *&RHS,
 
 /// ActOnConditionalOp - Parse a ?: operation.  Note that 'LHS' may be null
 /// in the case of a the GNU conditional expr extension.
-Action::OwningExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
+ExprResult Sema::ActOnConditionalOp(SourceLocation QuestionLoc,
                                                   SourceLocation ColonLoc,
-                                                  ExprArg Cond, ExprArg LHS,
-                                                  ExprArg RHS) {
-  Expr *CondExpr = (Expr *) Cond.get();
-  Expr *LHSExpr = (Expr *) LHS.get(), *RHSExpr = (Expr *) RHS.get();
-
+                                                  Expr *CondExpr, Expr *LHSExpr,
+                                                  Expr *RHSExpr) {
   // If this is the gnu "x ?: y" extension, analyze the types as though the LHS
   // was the condition.
   bool isLHSNull = LHSExpr == 0;
-  if (isLHSNull)
-    LHSExpr = CondExpr;
+  Expr *SAVEExpr = 0;
+  if (isLHSNull) {
+    LHSExpr = SAVEExpr = CondExpr;
+  }
 
   QualType result = CheckConditionalOperands(CondExpr, LHSExpr,
                                              RHSExpr, QuestionLoc);
   if (result.isNull())
     return ExprError();
 
-  Cond.release();
-  LHS.release();
-  RHS.release();
   return Owned(new (Context) ConditionalOperator(CondExpr, QuestionLoc,
-                                                 isLHSNull ? 0 : LHSExpr,
-                                                 ColonLoc, RHSExpr, result));
+                                                 LHSExpr, ColonLoc, 
+                                                 RHSExpr, SAVEExpr,
+                                                 result));
 }
 
 // CheckPointerTypesForAssignment - This is a very tricky routine (despite
@@ -4506,12 +4597,12 @@ Sema::CheckPointerTypesForAssignment(QualType lhsType, QualType rhsType) {
     // "unsigned char" on systems where "char" is unsigned.
     if (lhptee->isCharType())
       lhptee = Context.UnsignedCharTy;
-    else if (lhptee->isSignedIntegerType())
+    else if (lhptee->hasSignedIntegerRepresentation())
       lhptee = Context.getCorrespondingUnsignedType(lhptee);
 
     if (rhptee->isCharType())
       rhptee = Context.UnsignedCharTy;
-    else if (rhptee->isSignedIntegerType())
+    else if (rhptee->hasSignedIntegerRepresentation())
       rhptee = Context.getCorrespondingUnsignedType(rhptee);
 
     if (lhptee == rhptee) {
@@ -4668,13 +4759,18 @@ Sema::CheckAssignmentConstraints(QualType lhsType, QualType rhsType) {
   }
 
   if (lhsType->isVectorType() || rhsType->isVectorType()) {
-    // If we are allowing lax vector conversions, and LHS and RHS are both
-    // vectors, the total size only needs to be the same. This is a bitcast;
-    // no bits are changed but the result type is different.
-    if (getLangOptions().LaxVectorConversions &&
-        lhsType->isVectorType() && rhsType->isVectorType()) {
-      if (Context.getTypeSize(lhsType) == Context.getTypeSize(rhsType))
+    if (lhsType->isVectorType() && rhsType->isVectorType()) {
+      // If we are allowing lax vector conversions, and LHS and RHS are both
+      // vectors, the total size only needs to be the same. This is a bitcast;
+      // no bits are changed but the result type is different.
+      if (getLangOptions().LaxVectorConversions &&
+         (Context.getTypeSize(lhsType) == Context.getTypeSize(rhsType)))
         return IncompatibleVectors;
+
+      // Allow assignments of an AltiVec vector type to an equivalent GCC
+      // vector type and vice versa
+      if (Context.areCompatibleVectorTypes(lhsType, rhsType))
+        return Compatible;
     }
     return Incompatible;
   }
@@ -4832,14 +4928,14 @@ Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType, Expr *&rExpr) {
       // 2) null pointer constant
       if (FromType->isPointerType())
         if (FromType->getAs<PointerType>()->getPointeeType()->isVoidType()) {
-          ImpCastExprToType(rExpr, it->getType(), CastExpr::CK_BitCast);
+          ImpCastExprToType(rExpr, it->getType(), CK_BitCast);
           InitField = *it;
           break;
         }
 
       if (rExpr->isNullPointerConstant(Context,
                                        Expr::NPC_ValueDependentIsNull)) {
-        ImpCastExprToType(rExpr, it->getType(), CastExpr::CK_IntegralToPointer);
+        ImpCastExprToType(rExpr, it->getType(), CK_IntegralToPointer);
         InitField = *it;
         break;
       }
@@ -4883,14 +4979,14 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
        lhsType->isBlockPointerType())
       && rExpr->isNullPointerConstant(Context,
                                       Expr::NPC_ValueDependentIsNull)) {
-    ImpCastExprToType(rExpr, lhsType, CastExpr::CK_Unknown);
+    ImpCastExprToType(rExpr, lhsType, CK_Unknown);
     return Compatible;
   }
 
   // This check seems unnatural, however it is necessary to ensure the proper
   // conversion of functions/arrays. If the conversion were done for all
   // DeclExpr's (created by ActOnIdExpression), it would mess up the unary
-  // expressions that surpress this implicit conversion (&, sizeof).
+  // expressions that suppress this implicit conversion (&, sizeof).
   //
   // Suppress this for references: C++ 8.5.3p5.
   if (!lhsType->isReferenceType())
@@ -4907,7 +5003,7 @@ Sema::CheckSingleAssignmentConstraints(QualType lhsType, Expr *&rExpr) {
   // does not have reference type.
   if (result != Incompatible && rExpr->getType() != lhsType)
     ImpCastExprToType(rExpr, lhsType.getNonLValueExprType(Context),
-                      CastExpr::CK_Unknown);
+                      CK_Unknown);
   return result;
 }
 
@@ -4933,22 +5029,35 @@ QualType Sema::CheckVectorOperands(SourceLocation Loc, Expr *&lex, Expr *&rex) {
   // Handle the case of a vector & extvector type of the same size and element
   // type.  It would be nice if we only had one vector type someday.
   if (getLangOptions().LaxVectorConversions) {
-    // FIXME: Should we warn here?
     if (const VectorType *LV = lhsType->getAs<VectorType>()) {
-      if (const VectorType *RV = rhsType->getAs<VectorType>())
+      if (const VectorType *RV = rhsType->getAs<VectorType>()) {
         if (LV->getElementType() == RV->getElementType() &&
             LV->getNumElements() == RV->getNumElements()) {
           if (lhsType->isExtVectorType()) {
-            ImpCastExprToType(rex, lhsType, CastExpr::CK_BitCast);
+            ImpCastExprToType(rex, lhsType, CK_BitCast);
             return lhsType;
           } 
 
-          ImpCastExprToType(lex, rhsType, CastExpr::CK_BitCast);
+          ImpCastExprToType(lex, rhsType, CK_BitCast);
           return rhsType;
+        } else if (Context.getTypeSize(lhsType) ==Context.getTypeSize(rhsType)){
+          // If we are allowing lax vector conversions, and LHS and RHS are both
+          // vectors, the total size only needs to be the same. This is a
+          // bitcast; no bits are changed but the result type is different.
+          ImpCastExprToType(rex, lhsType, CK_BitCast);
+          return lhsType;
         }
+      }
     }
   }
 
+  // Handle the case of equivalent AltiVec and GCC vector types
+  if (lhsType->isVectorType() && rhsType->isVectorType() &&
+      Context.areCompatibleVectorTypes(lhsType, rhsType)) {
+    ImpCastExprToType(lex, rhsType, CK_BitCast);
+    return rhsType;
+  }
+
   // Canonicalize the ExtVector to the LHS, remember if we swapped so we can
   // swap back (so that we don't reverse the inputs to a subtract, for instance.
   bool swapped = false;
@@ -4963,7 +5072,7 @@ QualType Sema::CheckVectorOperands(SourceLocation Loc, Expr *&lex, Expr *&rex) {
     QualType EltTy = LV->getElementType();
     if (EltTy->isIntegralType(Context) && rhsType->isIntegralType(Context)) {
       if (Context.getIntegerTypeOrder(EltTy, rhsType) >= 0) {
-        ImpCastExprToType(rex, lhsType, CastExpr::CK_IntegralCast);
+        ImpCastExprToType(rex, lhsType, CK_IntegralCast);
         if (swapped) std::swap(rex, lex);
         return lhsType;
       }
@@ -4971,7 +5080,7 @@ QualType Sema::CheckVectorOperands(SourceLocation Loc, Expr *&lex, Expr *&rex) {
     if (EltTy->isRealFloatingType() && rhsType->isScalarType() &&
         rhsType->isRealFloatingType()) {
       if (Context.getFloatingTypeOrder(EltTy, rhsType) >= 0) {
-        ImpCastExprToType(rex, lhsType, CastExpr::CK_FloatingCast);
+        ImpCastExprToType(rex, lhsType, CK_FloatingCast);
         if (swapped) std::swap(rex, lex);
         return lhsType;
       }
@@ -5008,7 +5117,8 @@ QualType Sema::CheckMultiplyDivideOperands(
 QualType Sema::CheckRemainderOperands(
   Expr *&lex, Expr *&rex, SourceLocation Loc, bool isCompAssign) {
   if (lex->getType()->isVectorType() || rex->getType()->isVectorType()) {
-    if (lex->getType()->isIntegerType() && rex->getType()->isIntegerType())
+    if (lex->getType()->hasIntegerRepresentation() && 
+        rex->getType()->hasIntegerRepresentation())
       return CheckVectorOperands(Loc, lex, rex);
     return InvalidOperands(Loc, lex, rex);
   }
@@ -5253,7 +5363,8 @@ QualType Sema::CheckSubtractionOperands(Expr *&lex, Expr *&rex,
 QualType Sema::CheckShiftOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
                                   bool isCompAssign) {
   // C99 6.5.7p2: Each of the operands shall have integer type.
-  if (!lex->getType()->isIntegerType() || !rex->getType()->isIntegerType())
+  if (!lex->getType()->hasIntegerRepresentation() || 
+      !rex->getType()->hasIntegerRepresentation())
     return InvalidOperands(Loc, lex, rex);
 
   // Vector shifts promote their scalar inputs to vector type.
@@ -5269,7 +5380,7 @@ QualType Sema::CheckShiftOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
       LHSTy = Context.getPromotedIntegerType(LHSTy);
   }
   if (!isCompAssign)
-    ImpCastExprToType(lex, LHSTy, CastExpr::CK_IntegralCast);
+    ImpCastExprToType(lex, LHSTy, CK_IntegralCast);
 
   UsualUnaryConversions(rex);
 
@@ -5305,7 +5416,7 @@ static bool IsWithinTemplateSpecialization(Decl *D) {
 // C99 6.5.8, C++ [expr.rel]
 QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
                                     unsigned OpaqueOpc, bool isRelational) {
-  BinaryOperator::Opcode Opc = (BinaryOperator::Opcode)OpaqueOpc;
+  BinaryOperatorKind Opc = (BinaryOperatorKind) OpaqueOpc;
 
   // Handle vector comparisons separately.
   if (lex->getType()->isVectorType() || rex->getType()->isVectorType())
@@ -5334,19 +5445,19 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
             !IsWithinTemplateSpecialization(DRL->getDecl())) {
           DiagRuntimeBehavior(Loc, PDiag(diag::warn_comparison_always)
                               << 0 // self-
-                              << (Opc == BinaryOperator::EQ
-                                  || Opc == BinaryOperator::LE
-                                  || Opc == BinaryOperator::GE));
+                              << (Opc == BO_EQ
+                                  || Opc == BO_LE
+                                  || Opc == BO_GE));
         } else if (lType->isArrayType() && rType->isArrayType() &&
                    !DRL->getDecl()->getType()->isReferenceType() &&
                    !DRR->getDecl()->getType()->isReferenceType()) {
             // what is it always going to eval to?
             char always_evals_to;
             switch(Opc) {
-            case BinaryOperator::EQ: // e.g. array1 == array2
+            case BO_EQ: // e.g. array1 == array2
               always_evals_to = 0; // false
               break;
-            case BinaryOperator::NE: // e.g. array1 != array2
+            case BO_NE: // e.g. array1 != array2
               always_evals_to = 1; // true
               break;
             default:
@@ -5386,12 +5497,12 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
     if (literalString) {
       std::string resultComparison;
       switch (Opc) {
-      case BinaryOperator::LT: resultComparison = ") < 0"; break;
-      case BinaryOperator::GT: resultComparison = ") > 0"; break;
-      case BinaryOperator::LE: resultComparison = ") <= 0"; break;
-      case BinaryOperator::GE: resultComparison = ") >= 0"; break;
-      case BinaryOperator::EQ: resultComparison = ") == 0"; break;
-      case BinaryOperator::NE: resultComparison = ") != 0"; break;
+      case BO_LT: resultComparison = ") < 0"; break;
+      case BO_GT: resultComparison = ") > 0"; break;
+      case BO_LE: resultComparison = ") <= 0"; break;
+      case BO_GE: resultComparison = ") >= 0"; break;
+      case BO_EQ: resultComparison = ") == 0"; break;
+      case BO_NE: resultComparison = ") != 0"; break;
       default: assert(false && "Invalid comparison operator");
       }
 
@@ -5461,7 +5572,7 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
           if (isSFINAEContext())
             return QualType();
           
-          ImpCastExprToType(rex, lType, CastExpr::CK_BitCast);
+          ImpCastExprToType(rex, lType, CK_BitCast);
           return ResultTy;
         }
       }
@@ -5487,8 +5598,8 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
           << lex->getSourceRange() << rex->getSourceRange();
       }
 
-      ImpCastExprToType(lex, T, CastExpr::CK_BitCast);
-      ImpCastExprToType(rex, T, CastExpr::CK_BitCast);
+      ImpCastExprToType(lex, T, CK_BitCast);
+      ImpCastExprToType(rex, T, CK_BitCast);
       return ResultTy;
     }
     // C99 6.5.9p2 and C99 6.5.8p2
@@ -5513,7 +5624,7 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
         << lType << rType << lex->getSourceRange() << rex->getSourceRange();
     }
     if (LCanPointeeTy != RCanPointeeTy)
-      ImpCastExprToType(rex, lType, CastExpr::CK_BitCast);
+      ImpCastExprToType(rex, lType, CK_BitCast);
     return ResultTy;
   }
 
@@ -5523,13 +5634,19 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
     if (RHSIsNull &&
         (lType->isPointerType() ||
          (!isRelational && lType->isMemberPointerType()))) {
-      ImpCastExprToType(rex, lType, CastExpr::CK_NullToMemberPointer);
+      ImpCastExprToType(rex, lType, 
+                        lType->isMemberPointerType()
+                          ? CK_NullToMemberPointer
+                          : CK_IntegralToPointer);
       return ResultTy;
     }
     if (LHSIsNull &&
         (rType->isPointerType() ||
          (!isRelational && rType->isMemberPointerType()))) {
-      ImpCastExprToType(lex, rType, CastExpr::CK_NullToMemberPointer);
+      ImpCastExprToType(lex, rType, 
+                        rType->isMemberPointerType()
+                          ? CK_NullToMemberPointer
+                          : CK_IntegralToPointer);
       return ResultTy;
     }
 
@@ -5560,8 +5677,8 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
           << lex->getSourceRange() << rex->getSourceRange();
       }
 
-      ImpCastExprToType(lex, T, CastExpr::CK_BitCast);
-      ImpCastExprToType(rex, T, CastExpr::CK_BitCast);
+      ImpCastExprToType(lex, T, CK_BitCast);
+      ImpCastExprToType(rex, T, CK_BitCast);
       return ResultTy;
     }
 
@@ -5580,7 +5697,7 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
       Diag(Loc, diag::err_typecheck_comparison_of_distinct_blocks)
         << lType << rType << lex->getSourceRange() << rex->getSourceRange();
     }
-    ImpCastExprToType(rex, lType, CastExpr::CK_BitCast);
+    ImpCastExprToType(rex, lType, CK_BitCast);
     return ResultTy;
   }
   // Allow block pointers to be compared with null pointer constants.
@@ -5595,7 +5712,7 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
         Diag(Loc, diag::err_typecheck_comparison_of_distinct_blocks)
           << lType << rType << lex->getSourceRange() << rex->getSourceRange();
     }
-    ImpCastExprToType(rex, lType, CastExpr::CK_BitCast);
+    ImpCastExprToType(rex, lType, CK_BitCast);
     return ResultTy;
   }
 
@@ -5613,14 +5730,14 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
         Diag(Loc, diag::ext_typecheck_comparison_of_distinct_pointers)
           << lType << rType << lex->getSourceRange() << rex->getSourceRange();
       }
-      ImpCastExprToType(rex, lType, CastExpr::CK_BitCast);
+      ImpCastExprToType(rex, lType, CK_BitCast);
       return ResultTy;
     }
     if (lType->isObjCObjectPointerType() && rType->isObjCObjectPointerType()) {
       if (!Context.areComparableObjCPointerTypes(lType, rType))
         Diag(Loc, diag::ext_typecheck_comparison_of_distinct_pointers)
           << lType << rType << lex->getSourceRange() << rex->getSourceRange();
-      ImpCastExprToType(rex, lType, CastExpr::CK_BitCast);
+      ImpCastExprToType(rex, lType, CK_BitCast);
       return ResultTy;
     }
   }
@@ -5648,21 +5765,21 @@ QualType Sema::CheckCompareOperands(Expr *&lex, Expr *&rex, SourceLocation Loc,
     }
     
     if (lType->isIntegerType())
-      ImpCastExprToType(lex, rType, CastExpr::CK_IntegralToPointer);
+      ImpCastExprToType(lex, rType, CK_IntegralToPointer);
     else
-      ImpCastExprToType(rex, lType, CastExpr::CK_IntegralToPointer);
+      ImpCastExprToType(rex, lType, CK_IntegralToPointer);
     return ResultTy;
   }
   
   // Handle block pointers.
   if (!isRelational && RHSIsNull
       && lType->isBlockPointerType() && rType->isIntegerType()) {
-    ImpCastExprToType(rex, lType, CastExpr::CK_IntegralToPointer);
+    ImpCastExprToType(rex, lType, CK_IntegralToPointer);
     return ResultTy;
   }
   if (!isRelational && LHSIsNull
       && lType->isIntegerType() && rType->isBlockPointerType()) {
-    ImpCastExprToType(lex, rType, CastExpr::CK_IntegralToPointer);
+    ImpCastExprToType(lex, rType, CK_IntegralToPointer);
     return ResultTy;
   }
   return InvalidOperands(Loc, lex, rex);
@@ -5707,7 +5824,7 @@ QualType Sema::CheckVectorCompareOperands(Expr *&lex, Expr *&rex,
   // Return the type for the comparison, which is the same as vector type for
   // integer vectors, or an integer type of identical size and number of
   // elements for floating point vectors.
-  if (lType->isIntegerType())
+  if (lType->hasIntegerRepresentation())
     return lType;
 
   const VectorType *VTy = lType->getAs<VectorType>();
@@ -5724,8 +5841,13 @@ QualType Sema::CheckVectorCompareOperands(Expr *&lex, Expr *&rex,
 
 inline QualType Sema::CheckBitwiseOperands(
   Expr *&lex, Expr *&rex, SourceLocation Loc, bool isCompAssign) {
-  if (lex->getType()->isVectorType() || rex->getType()->isVectorType())
-    return CheckVectorOperands(Loc, lex, rex);
+  if (lex->getType()->isVectorType() || rex->getType()->isVectorType()) {
+    if (lex->getType()->hasIntegerRepresentation() &&
+        rex->getType()->hasIntegerRepresentation())
+      return CheckVectorOperands(Loc, lex, rex);
+    
+    return InvalidOperands(Loc, lex, rex);
+  }
 
   QualType compType = UsualArithmeticConversions(lex, rex, isCompAssign);
 
@@ -5741,18 +5863,21 @@ inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
   // bitwise one.  We do this when the LHS is a non-bool integer and the RHS
   // is a constant.
   if (lex->getType()->isIntegerType() && !lex->getType()->isBooleanType() &&
-      rex->getType()->isIntegerType() && rex->isEvaluatable(Context) &&
-      // Don't warn if the RHS is a (constant folded) boolean expression like
-      // "sizeof(int) == 4".
-      !rex->isKnownToHaveBooleanValue() &&
+      rex->getType()->isIntegerType() && !rex->isValueDependent() &&
       // Don't warn in macros.
-      !Loc.isMacroID())
-    Diag(Loc, diag::warn_logical_instead_of_bitwise)
-     << rex->getSourceRange()
-      << (Opc == BinaryOperator::LAnd ? "&&" : "||")
-      << (Opc == BinaryOperator::LAnd ? "&" : "|");
-  
-  
+      !Loc.isMacroID()) {
+    // If the RHS can be constant folded, and if it constant folds to something
+    // that isn't 0 or 1 (which indicate a potential logical operation that
+    // happened to fold to true/false) then warn.
+    Expr::EvalResult Result;
+    if (rex->Evaluate(Result, Context) && !Result.HasSideEffects &&
+        Result.Val.getInt() != 0 && Result.Val.getInt() != 1) {
+      Diag(Loc, diag::warn_logical_instead_of_bitwise)
+       << rex->getSourceRange()
+        << (Opc == BO_LAnd ? "&&" : "||")
+        << (Opc == BO_LAnd ? "&" : "|");
+    }
+  }
   
   if (!Context.getLangOptions().CPlusPlus) {
     UsualUnaryConversions(lex);
@@ -5907,8 +6032,8 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
     if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(RHSCheck))
       RHSCheck = ICE->getSubExpr();
     if (UnaryOperator *UO = dyn_cast<UnaryOperator>(RHSCheck)) {
-      if ((UO->getOpcode() == UnaryOperator::Plus ||
-           UO->getOpcode() == UnaryOperator::Minus) &&
+      if ((UO->getOpcode() == UO_Plus ||
+           UO->getOpcode() == UO_Minus) &&
           Loc.isFileID() && UO->getOperatorLoc().isFileID() &&
           // Only if the two operators are exactly adjacent.
           Loc.getFileLocWithOffset(1) == UO->getOperatorLoc() &&
@@ -5917,7 +6042,7 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
           Loc.getFileLocWithOffset(2) != UO->getSubExpr()->getLocStart() &&
           UO->getSubExpr()->getLocStart().isFileID()) {
         Diag(Loc, diag::warn_not_compound_assign)
-          << (UO->getOpcode() == UnaryOperator::Plus ? "+" : "-")
+          << (UO->getOpcode() == UO_Plus ? "+" : "-")
           << SourceRange(UO->getOperatorLoc(), UO->getOperatorLoc());
       }
     }
@@ -5938,7 +6063,7 @@ QualType Sema::CheckAssignmentOperands(Expr *LHS, Expr *&RHS,
   // only handles the pattern "*null = whatever", which is a very syntactic
   // check.
   if (UnaryOperator *UO = dyn_cast<UnaryOperator>(LHS->IgnoreParenCasts()))
-    if (UO->getOpcode() == UnaryOperator::Deref &&
+    if (UO->getOpcode() == UO_Deref &&
         UO->getSubExpr()->IgnoreParenCasts()->
           isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNotNull) &&
         !UO->getType().isVolatileQualified()) {
@@ -6083,9 +6208,9 @@ static NamedDecl *getPrimaryDecl(Expr *E) {
     UnaryOperator *UO = cast<UnaryOperator>(E);
 
     switch(UO->getOpcode()) {
-    case UnaryOperator::Real:
-    case UnaryOperator::Imag:
-    case UnaryOperator::Extension:
+    case UO_Real:
+    case UO_Imag:
+    case UO_Extension:
       return getPrimaryDecl(UO->getSubExpr());
     default:
       return 0;
@@ -6109,17 +6234,19 @@ static NamedDecl *getPrimaryDecl(Expr *E) {
 /// operator (C99 6.3.2.1p[2-4]), and its result is never an lvalue.
 /// In C++, the operand might be an overloaded function name, in which case
 /// we allow the '&' but retain the overloaded-function type.
-QualType Sema::CheckAddressOfOperand(Expr *op, SourceLocation OpLoc) {
-  // Make sure to ignore parentheses in subsequent checks
-  op = op->IgnoreParens();
-
-  if (op->isTypeDependent())
+QualType Sema::CheckAddressOfOperand(Expr *OrigOp, SourceLocation OpLoc) {
+  if (OrigOp->isTypeDependent())
     return Context.DependentTy;
+  if (OrigOp->getType() == Context.OverloadTy)
+    return Context.OverloadTy;
+
+  // Make sure to ignore parentheses in subsequent checks
+  Expr *op = OrigOp->IgnoreParens();
 
   if (getLangOptions().C99) {
     // Implement C99-only parts of addressof rules.
     if (UnaryOperator* uOp = dyn_cast<UnaryOperator>(op)) {
-      if (uOp->getOpcode() == UnaryOperator::Deref)
+      if (uOp->getOpcode() == UO_Deref)
         // Per C99 6.5.3.2, the address of a deref always returns a valid result
         // (assuming the deref expression is valid).
         return uOp->getSubExpr()->getType();
@@ -6130,32 +6257,41 @@ QualType Sema::CheckAddressOfOperand(Expr *op, SourceLocation OpLoc) {
   NamedDecl *dcl = getPrimaryDecl(op);
   Expr::isLvalueResult lval = op->isLvalue(Context);
 
-  MemberExpr *ME = dyn_cast<MemberExpr>(op);
-  if (lval == Expr::LV_MemberFunction && ME &&
-      isa<CXXMethodDecl>(ME->getMemberDecl())) {
-    ValueDecl *dcl = cast<MemberExpr>(op)->getMemberDecl();
-    // &f where f is a member of the current object, or &o.f, or &p->f
-    // All these are not allowed, and we need to catch them before the dcl
-    // branch of the if, below.
-    Diag(OpLoc, diag::err_unqualified_pointer_member_function)
-        << dcl;
-    // FIXME: Improve this diagnostic and provide a fixit.
-
-    // Now recover by acting as if the function had been accessed qualified.
-    return Context.getMemberPointerType(op->getType(),
-                Context.getTypeDeclType(cast<RecordDecl>(dcl->getDeclContext()))
-                       .getTypePtr());
-  }
-  
   if (lval == Expr::LV_ClassTemporary) {
     Diag(OpLoc, isSFINAEContext()? diag::err_typecheck_addrof_class_temporary
                                  : diag::ext_typecheck_addrof_class_temporary)
       << op->getType() << op->getSourceRange();
     if (isSFINAEContext())
       return QualType();
-  } else if (isa<ObjCSelectorExpr>(op))
+  } else if (isa<ObjCSelectorExpr>(op)) {
     return Context.getPointerType(op->getType());
-  else if (lval != Expr::LV_Valid && lval != Expr::LV_IncompleteVoidType) {
+  } else if (lval == Expr::LV_MemberFunction) {
+    // If it's an instance method, make a member pointer.
+    // The expression must have exactly the form &A::foo.
+
+    // If the underlying expression isn't a decl ref, give up.
+    if (!isa<DeclRefExpr>(op)) {
+      Diag(OpLoc, diag::err_invalid_form_pointer_member_function)
+        << OrigOp->getSourceRange();
+      return QualType();
+    }
+    DeclRefExpr *DRE = cast<DeclRefExpr>(op);
+    CXXMethodDecl *MD = cast<CXXMethodDecl>(DRE->getDecl());
+
+    // The id-expression was parenthesized.
+    if (OrigOp != DRE) {
+      Diag(OpLoc, diag::err_parens_pointer_member_function)
+        << OrigOp->getSourceRange();
+
+    // The method was named without a qualifier.
+    } else if (!DRE->getQualifier()) {
+      Diag(OpLoc, diag::err_unqualified_pointer_member_function)
+        << op->getSourceRange();
+    }
+
+    return Context.getMemberPointerType(op->getType(),
+              Context.getTypeDeclType(MD->getParent()).getTypePtr());
+  } else if (lval != Expr::LV_Valid && lval != Expr::LV_IncompleteVoidType) {
     // C99 6.5.3.2p1
     // The operand must be either an l-value or a function designator
     if (!op->getType()->isFunctionType()) {
@@ -6183,13 +6319,14 @@ QualType Sema::CheckAddressOfOperand(Expr *op, SourceLocation OpLoc) {
     // FIXME: Can LHS ever be null here?
     if (!CheckAddressOfOperand(CO->getTrueExpr(), OpLoc).isNull())
       return CheckAddressOfOperand(CO->getFalseExpr(), OpLoc);
-  } else if (isa<UnresolvedLookupExpr>(op)) {
-    return Context.OverloadTy;
   } else if (dcl) { // C99 6.5.3.2p1
     // We have an lvalue with a decl. Make sure the decl is not declared
     // with the register storage-class specifier.
     if (const VarDecl *vd = dyn_cast<VarDecl>(dcl)) {
-      if (vd->getStorageClass() == VarDecl::Register) {
+      // in C++ it is not error to take address of a register
+      // variable (c++03 7.1.1P3)
+      if (vd->getStorageClass() == SC_Register &&
+          !getLangOptions().CPlusPlus) {
         Diag(OpLoc, diag::err_typecheck_address_of)
           << "register variable" << op->getSourceRange();
         return QualType();
@@ -6214,13 +6351,6 @@ QualType Sema::CheckAddressOfOperand(Expr *op, SourceLocation OpLoc) {
                 Context.getTypeDeclType(cast<RecordDecl>(Ctx)).getTypePtr());
         }
       }
-    } else if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(dcl)) {
-      // Okay: we can take the address of a function.
-      // As above.
-      if (isa<DeclRefExpr>(op) && cast<DeclRefExpr>(op)->getQualifier() &&
-          MD->isInstance())
-        return Context.getMemberPointerType(op->getType(),
-              Context.getTypeDeclType(MD->getParent()).getTypePtr());
     } else if (!isa<FunctionDecl>(dcl))
       assert(0 && "Unknown/unexpected decl type");
   }
@@ -6233,6 +6363,8 @@ QualType Sema::CheckAddressOfOperand(Expr *op, SourceLocation OpLoc) {
   }
 
   // If the operand has type "type", the result has type "pointer to type".
+  if (op->getType()->isObjCObjectType())
+    return Context.getObjCObjectPointerType(op->getType());
   return Context.getPointerType(op->getType());
 }
 
@@ -6264,63 +6396,63 @@ QualType Sema::CheckIndirectionOperand(Expr *Op, SourceLocation OpLoc) {
   return Result;
 }
 
-static inline BinaryOperator::Opcode ConvertTokenKindToBinaryOpcode(
+static inline BinaryOperatorKind ConvertTokenKindToBinaryOpcode(
   tok::TokenKind Kind) {
-  BinaryOperator::Opcode Opc;
+  BinaryOperatorKind Opc;
   switch (Kind) {
   default: assert(0 && "Unknown binop!");
-  case tok::periodstar:           Opc = BinaryOperator::PtrMemD; break;
-  case tok::arrowstar:            Opc = BinaryOperator::PtrMemI; break;
-  case tok::star:                 Opc = BinaryOperator::Mul; break;
-  case tok::slash:                Opc = BinaryOperator::Div; break;
-  case tok::percent:              Opc = BinaryOperator::Rem; break;
-  case tok::plus:                 Opc = BinaryOperator::Add; break;
-  case tok::minus:                Opc = BinaryOperator::Sub; break;
-  case tok::lessless:             Opc = BinaryOperator::Shl; break;
-  case tok::greatergreater:       Opc = BinaryOperator::Shr; break;
-  case tok::lessequal:            Opc = BinaryOperator::LE; break;
-  case tok::less:                 Opc = BinaryOperator::LT; break;
-  case tok::greaterequal:         Opc = BinaryOperator::GE; break;
-  case tok::greater:              Opc = BinaryOperator::GT; break;
-  case tok::exclaimequal:         Opc = BinaryOperator::NE; break;
-  case tok::equalequal:           Opc = BinaryOperator::EQ; break;
-  case tok::amp:                  Opc = BinaryOperator::And; break;
-  case tok::caret:                Opc = BinaryOperator::Xor; break;
-  case tok::pipe:                 Opc = BinaryOperator::Or; break;
-  case tok::ampamp:               Opc = BinaryOperator::LAnd; break;
-  case tok::pipepipe:             Opc = BinaryOperator::LOr; break;
-  case tok::equal:                Opc = BinaryOperator::Assign; break;
-  case tok::starequal:            Opc = BinaryOperator::MulAssign; break;
-  case tok::slashequal:           Opc = BinaryOperator::DivAssign; break;
-  case tok::percentequal:         Opc = BinaryOperator::RemAssign; break;
-  case tok::plusequal:            Opc = BinaryOperator::AddAssign; break;
-  case tok::minusequal:           Opc = BinaryOperator::SubAssign; break;
-  case tok::lesslessequal:        Opc = BinaryOperator::ShlAssign; break;
-  case tok::greatergreaterequal:  Opc = BinaryOperator::ShrAssign; break;
-  case tok::ampequal:             Opc = BinaryOperator::AndAssign; break;
-  case tok::caretequal:           Opc = BinaryOperator::XorAssign; break;
-  case tok::pipeequal:            Opc = BinaryOperator::OrAssign; break;
-  case tok::comma:                Opc = BinaryOperator::Comma; break;
+  case tok::periodstar:           Opc = BO_PtrMemD; break;
+  case tok::arrowstar:            Opc = BO_PtrMemI; break;
+  case tok::star:                 Opc = BO_Mul; break;
+  case tok::slash:                Opc = BO_Div; break;
+  case tok::percent:              Opc = BO_Rem; break;
+  case tok::plus:                 Opc = BO_Add; break;
+  case tok::minus:                Opc = BO_Sub; break;
+  case tok::lessless:             Opc = BO_Shl; break;
+  case tok::greatergreater:       Opc = BO_Shr; break;
+  case tok::lessequal:            Opc = BO_LE; break;
+  case tok::less:                 Opc = BO_LT; break;
+  case tok::greaterequal:         Opc = BO_GE; break;
+  case tok::greater:              Opc = BO_GT; break;
+  case tok::exclaimequal:         Opc = BO_NE; break;
+  case tok::equalequal:           Opc = BO_EQ; break;
+  case tok::amp:                  Opc = BO_And; break;
+  case tok::caret:                Opc = BO_Xor; break;
+  case tok::pipe:                 Opc = BO_Or; break;
+  case tok::ampamp:               Opc = BO_LAnd; break;
+  case tok::pipepipe:             Opc = BO_LOr; break;
+  case tok::equal:                Opc = BO_Assign; break;
+  case tok::starequal:            Opc = BO_MulAssign; break;
+  case tok::slashequal:           Opc = BO_DivAssign; break;
+  case tok::percentequal:         Opc = BO_RemAssign; break;
+  case tok::plusequal:            Opc = BO_AddAssign; break;
+  case tok::minusequal:           Opc = BO_SubAssign; break;
+  case tok::lesslessequal:        Opc = BO_ShlAssign; break;
+  case tok::greatergreaterequal:  Opc = BO_ShrAssign; break;
+  case tok::ampequal:             Opc = BO_AndAssign; break;
+  case tok::caretequal:           Opc = BO_XorAssign; break;
+  case tok::pipeequal:            Opc = BO_OrAssign; break;
+  case tok::comma:                Opc = BO_Comma; break;
   }
   return Opc;
 }
 
-static inline UnaryOperator::Opcode ConvertTokenKindToUnaryOpcode(
+static inline UnaryOperatorKind ConvertTokenKindToUnaryOpcode(
   tok::TokenKind Kind) {
-  UnaryOperator::Opcode Opc;
+  UnaryOperatorKind Opc;
   switch (Kind) {
   default: assert(0 && "Unknown unary op!");
-  case tok::plusplus:     Opc = UnaryOperator::PreInc; break;
-  case tok::minusminus:   Opc = UnaryOperator::PreDec; break;
-  case tok::amp:          Opc = UnaryOperator::AddrOf; break;
-  case tok::star:         Opc = UnaryOperator::Deref; break;
-  case tok::plus:         Opc = UnaryOperator::Plus; break;
-  case tok::minus:        Opc = UnaryOperator::Minus; break;
-  case tok::tilde:        Opc = UnaryOperator::Not; break;
-  case tok::exclaim:      Opc = UnaryOperator::LNot; break;
-  case tok::kw___real:    Opc = UnaryOperator::Real; break;
-  case tok::kw___imag:    Opc = UnaryOperator::Imag; break;
-  case tok::kw___extension__: Opc = UnaryOperator::Extension; break;
+  case tok::plusplus:     Opc = UO_PreInc; break;
+  case tok::minusminus:   Opc = UO_PreDec; break;
+  case tok::amp:          Opc = UO_AddrOf; break;
+  case tok::star:         Opc = UO_Deref; break;
+  case tok::plus:         Opc = UO_Plus; break;
+  case tok::minus:        Opc = UO_Minus; break;
+  case tok::tilde:        Opc = UO_Not; break;
+  case tok::exclaim:      Opc = UO_LNot; break;
+  case tok::kw___real:    Opc = UO_Real; break;
+  case tok::kw___imag:    Opc = UO_Imag; break;
+  case tok::kw___extension__: Opc = UO_Extension; break;
   }
   return Opc;
 }
@@ -6328,106 +6460,111 @@ static inline UnaryOperator::Opcode ConvertTokenKindToUnaryOpcode(
 /// CreateBuiltinBinOp - Creates a new built-in binary operation with
 /// operator @p Opc at location @c TokLoc. This routine only supports
 /// built-in operations; ActOnBinOp handles overloaded operators.
-Action::OwningExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
-                                                  unsigned Op,
-                                                  Expr *lhs, Expr *rhs) {
+ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
+                                    unsigned Op,
+                                    Expr *lhs, Expr *rhs) {
   QualType ResultTy;     // Result type of the binary operator.
-  BinaryOperator::Opcode Opc = (BinaryOperator::Opcode)Op;
+  BinaryOperatorKind Opc = (BinaryOperatorKind) Op;
   // The following two variables are used for compound assignment operators
   QualType CompLHSTy;    // Type of LHS after promotions for computation
   QualType CompResultTy; // Type of computation result
 
   switch (Opc) {
-  case BinaryOperator::Assign:
+  case BO_Assign:
     ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, QualType());
     break;
-  case BinaryOperator::PtrMemD:
-  case BinaryOperator::PtrMemI:
+  case BO_PtrMemD:
+  case BO_PtrMemI:
     ResultTy = CheckPointerToMemberOperands(lhs, rhs, OpLoc,
-                                            Opc == BinaryOperator::PtrMemI);
+                                            Opc == BO_PtrMemI);
     break;
-  case BinaryOperator::Mul:
-  case BinaryOperator::Div:
+  case BO_Mul:
+  case BO_Div:
     ResultTy = CheckMultiplyDivideOperands(lhs, rhs, OpLoc, false,
-                                           Opc == BinaryOperator::Div);
+                                           Opc == BO_Div);
     break;
-  case BinaryOperator::Rem:
+  case BO_Rem:
     ResultTy = CheckRemainderOperands(lhs, rhs, OpLoc);
     break;
-  case BinaryOperator::Add:
+  case BO_Add:
     ResultTy = CheckAdditionOperands(lhs, rhs, OpLoc);
     break;
-  case BinaryOperator::Sub:
+  case BO_Sub:
     ResultTy = CheckSubtractionOperands(lhs, rhs, OpLoc);
     break;
-  case BinaryOperator::Shl:
-  case BinaryOperator::Shr:
+  case BO_Shl:
+  case BO_Shr:
     ResultTy = CheckShiftOperands(lhs, rhs, OpLoc);
     break;
-  case BinaryOperator::LE:
-  case BinaryOperator::LT:
-  case BinaryOperator::GE:
-  case BinaryOperator::GT:
+  case BO_LE:
+  case BO_LT:
+  case BO_GE:
+  case BO_GT:
     ResultTy = CheckCompareOperands(lhs, rhs, OpLoc, Opc, true);
     break;
-  case BinaryOperator::EQ:
-  case BinaryOperator::NE:
+  case BO_EQ:
+  case BO_NE:
     ResultTy = CheckCompareOperands(lhs, rhs, OpLoc, Opc, false);
     break;
-  case BinaryOperator::And:
-  case BinaryOperator::Xor:
-  case BinaryOperator::Or:
+  case BO_And:
+  case BO_Xor:
+  case BO_Or:
     ResultTy = CheckBitwiseOperands(lhs, rhs, OpLoc);
     break;
-  case BinaryOperator::LAnd:
-  case BinaryOperator::LOr:
+  case BO_LAnd:
+  case BO_LOr:
     ResultTy = CheckLogicalOperands(lhs, rhs, OpLoc, Opc);
     break;
-  case BinaryOperator::MulAssign:
-  case BinaryOperator::DivAssign:
+  case BO_MulAssign:
+  case BO_DivAssign:
     CompResultTy = CheckMultiplyDivideOperands(lhs, rhs, OpLoc, true,
-                                              Opc == BinaryOperator::DivAssign);
+                                              Opc == BO_DivAssign);
     CompLHSTy = CompResultTy;
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
-  case BinaryOperator::RemAssign:
+  case BO_RemAssign:
     CompResultTy = CheckRemainderOperands(lhs, rhs, OpLoc, true);
     CompLHSTy = CompResultTy;
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
-  case BinaryOperator::AddAssign:
+  case BO_AddAssign:
     CompResultTy = CheckAdditionOperands(lhs, rhs, OpLoc, &CompLHSTy);
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
-  case BinaryOperator::SubAssign:
+  case BO_SubAssign:
     CompResultTy = CheckSubtractionOperands(lhs, rhs, OpLoc, &CompLHSTy);
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
-  case BinaryOperator::ShlAssign:
-  case BinaryOperator::ShrAssign:
+  case BO_ShlAssign:
+  case BO_ShrAssign:
     CompResultTy = CheckShiftOperands(lhs, rhs, OpLoc, true);
     CompLHSTy = CompResultTy;
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
-  case BinaryOperator::AndAssign:
-  case BinaryOperator::XorAssign:
-  case BinaryOperator::OrAssign:
+  case BO_AndAssign:
+  case BO_XorAssign:
+  case BO_OrAssign:
     CompResultTy = CheckBitwiseOperands(lhs, rhs, OpLoc, true);
     CompLHSTy = CompResultTy;
     if (!CompResultTy.isNull())
       ResultTy = CheckAssignmentOperands(lhs, rhs, OpLoc, CompResultTy);
     break;
-  case BinaryOperator::Comma:
+  case BO_Comma:
     ResultTy = CheckCommaOperands(lhs, rhs, OpLoc);
     break;
   }
   if (ResultTy.isNull())
     return ExprError();
+  if (ResultTy->isObjCObjectType() && LangOpts.ObjCNonFragileABI) {
+    if (Opc >= BO_Assign && Opc <= BO_OrAssign) 
+          Diag(OpLoc, diag::err_assignment_requires_nonfragile_object)
+                << ResultTy;
+  }
   if (CompResultTy.isNull())
     return Owned(new (Context) BinaryOperator(lhs, rhs, Opc, ResultTy, OpLoc));
   else
@@ -6479,7 +6616,7 @@ static void SuggestParentheses(Sema &Self, SourceLocation Loc,
 /// operators are mixed in a way that suggests that the programmer forgot that
 /// comparison operators have higher precedence. The most typical example of
 /// such code is "flags & 0x0020 != 0", which is equivalent to "flags & 1".
-static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperator::Opcode Opc,
+static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperatorKind Opc,
                                       SourceLocation OpLoc,Expr *lhs,Expr *rhs){
   typedef BinaryOperator BinOp;
   BinOp::Opcode lhsopc = static_cast<BinOp::Opcode>(-1),
@@ -6526,19 +6663,17 @@ static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperator::Opcode Opc,
 /// DiagnoseBinOpPrecedence - Emit warnings for expressions with tricky
 /// precedence. This currently diagnoses only "arg1 'bitwise' arg2 'eq' arg3".
 /// But it could also warn about arg1 && arg2 || arg3, as GCC 4.3+ does.
-static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperator::Opcode Opc,
+static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
                                     SourceLocation OpLoc, Expr *lhs, Expr *rhs){
   if (BinaryOperator::isBitwiseOp(Opc))
     DiagnoseBitwisePrecedence(Self, Opc, OpLoc, lhs, rhs);
 }
 
 // Binary Operators.  'Tok' is the token for the operator.
-Action::OwningExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
-                                          tok::TokenKind Kind,
-                                          ExprArg LHS, ExprArg RHS) {
-  BinaryOperator::Opcode Opc = ConvertTokenKindToBinaryOpcode(Kind);
-  Expr *lhs = LHS.takeAs<Expr>(), *rhs = RHS.takeAs<Expr>();
-
+ExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
+                            tok::TokenKind Kind,
+                            Expr *lhs, Expr *rhs) {
+  BinaryOperatorKind Opc = ConvertTokenKindToBinaryOpcode(Kind);
   assert((lhs != 0) && "ActOnBinOp(): missing left expression");
   assert((rhs != 0) && "ActOnBinOp(): missing right expression");
 
@@ -6548,9 +6683,9 @@ Action::OwningExprResult Sema::ActOnBinOp(Scope *S, SourceLocation TokLoc,
   return BuildBinOp(S, TokLoc, Opc, lhs, rhs);
 }
 
-Action::OwningExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
-                                          BinaryOperator::Opcode Opc,
-                                          Expr *lhs, Expr *rhs) {
+ExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
+                            BinaryOperatorKind Opc,
+                            Expr *lhs, Expr *rhs) {
   if (getLangOptions().CPlusPlus &&
       (lhs->getType()->isOverloadableType() ||
        rhs->getType()->isOverloadableType())) {
@@ -6573,38 +6708,32 @@ Action::OwningExprResult Sema::BuildBinOp(Scope *S, SourceLocation OpLoc,
   return CreateBuiltinBinOp(OpLoc, Opc, lhs, rhs);
 }
 
-Action::OwningExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
+ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
                                                     unsigned OpcIn,
-                                                    ExprArg InputArg) {
-  UnaryOperator::Opcode Opc = static_cast<UnaryOperator::Opcode>(OpcIn);
+                                                    Expr *Input) {
+  UnaryOperatorKind Opc = static_cast<UnaryOperatorKind>(OpcIn);
 
-  // FIXME: Input is modified below, but InputArg is not updated appropriately.
-  Expr *Input = (Expr *)InputArg.get();
   QualType resultType;
   switch (Opc) {
-  case UnaryOperator::OffsetOf:
-    assert(false && "Invalid unary operator");
-    break;
-      
-  case UnaryOperator::PreInc:
-  case UnaryOperator::PreDec:
-  case UnaryOperator::PostInc:
-  case UnaryOperator::PostDec:
+  case UO_PreInc:
+  case UO_PreDec:
+  case UO_PostInc:
+  case UO_PostDec:
     resultType = CheckIncrementDecrementOperand(Input, OpLoc,
-                                                Opc == UnaryOperator::PreInc ||
-                                                Opc == UnaryOperator::PostInc,
-                                                Opc == UnaryOperator::PreInc ||
-                                                Opc == UnaryOperator::PreDec);
+                                                Opc == UO_PreInc ||
+                                                Opc == UO_PostInc,
+                                                Opc == UO_PreInc ||
+                                                Opc == UO_PreDec);
     break;
-  case UnaryOperator::AddrOf:
+  case UO_AddrOf:
     resultType = CheckAddressOfOperand(Input, OpLoc);
     break;
-  case UnaryOperator::Deref:
+  case UO_Deref:
     DefaultFunctionArrayLvalueConversion(Input);
     resultType = CheckIndirectionOperand(Input, OpLoc);
     break;
-  case UnaryOperator::Plus:
-  case UnaryOperator::Minus:
+  case UO_Plus:
+  case UO_Minus:
     UsualUnaryConversions(Input);
     resultType = Input->getType();
     if (resultType->isDependentType())
@@ -6616,13 +6745,13 @@ Action::OwningExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
              resultType->isEnumeralType())
       break;
     else if (getLangOptions().CPlusPlus && // C++ [expr.unary.op]p6
-             Opc == UnaryOperator::Plus &&
+             Opc == UO_Plus &&
              resultType->isPointerType())
       break;
 
     return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
       << resultType << Input->getSourceRange());
-  case UnaryOperator::Not: // bitwise complement
+  case UO_Not: // bitwise complement
     UsualUnaryConversions(Input);
     resultType = Input->getType();
     if (resultType->isDependentType())
@@ -6632,11 +6761,11 @@ Action::OwningExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
       // C99 does not support '~' for complex conjugation.
       Diag(OpLoc, diag::ext_integer_complement_complex)
         << resultType << Input->getSourceRange();
-    else if (!resultType->isIntegerType())
+    else if (!resultType->hasIntegerRepresentation())
       return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
         << resultType << Input->getSourceRange());
     break;
-  case UnaryOperator::LNot: // logical negation
+  case UO_LNot: // logical negation
     // Unlike +/-/~, integer promotions aren't done here (C99 6.5.3.3p5).
     DefaultFunctionArrayLvalueConversion(Input);
     resultType = Input->getType();
@@ -6649,27 +6778,25 @@ Action::OwningExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
     // In C++, it's bool. C++ 5.3.1p8
     resultType = getLangOptions().CPlusPlus ? Context.BoolTy : Context.IntTy;
     break;
-  case UnaryOperator::Real:
-  case UnaryOperator::Imag:
-    resultType = CheckRealImagOperand(Input, OpLoc, Opc == UnaryOperator::Real);
+  case UO_Real:
+  case UO_Imag:
+    resultType = CheckRealImagOperand(Input, OpLoc, Opc == UO_Real);
     break;
-  case UnaryOperator::Extension:
+  case UO_Extension:
     resultType = Input->getType();
     break;
   }
   if (resultType.isNull())
     return ExprError();
 
-  InputArg.release();
   return Owned(new (Context) UnaryOperator(Input, Opc, resultType, OpLoc));
 }
 
-Action::OwningExprResult Sema::BuildUnaryOp(Scope *S, SourceLocation OpLoc,
-                                            UnaryOperator::Opcode Opc,
-                                            ExprArg input) {
-  Expr *Input = (Expr*)input.get();
+ExprResult Sema::BuildUnaryOp(Scope *S, SourceLocation OpLoc,
+                              UnaryOperatorKind Opc,
+                              Expr *Input) {
   if (getLangOptions().CPlusPlus && Input->getType()->isOverloadableType() &&
-      Opc != UnaryOperator::Extension) {
+      UnaryOperator::getOverloadedOperator(Opc) != OO_None) {
     // Find all of the overloaded operators visible from this
     // point. We perform both an operator-name lookup from the local
     // scope and an argument-dependent lookup based on the types of
@@ -6680,24 +6807,24 @@ Action::OwningExprResult Sema::BuildUnaryOp(Scope *S, SourceLocation OpLoc,
       LookupOverloadedOperatorName(OverOp, S, Input->getType(), QualType(),
                                    Functions);
 
-    return CreateOverloadedUnaryOp(OpLoc, Opc, Functions, move(input));
+    return CreateOverloadedUnaryOp(OpLoc, Opc, Functions, Input);
   }
 
-  return CreateBuiltinUnaryOp(OpLoc, Opc, move(input));
+  return CreateBuiltinUnaryOp(OpLoc, Opc, Input);
 }
 
 // Unary Operators.  'Tok' is the token for the operator.
-Action::OwningExprResult Sema::ActOnUnaryOp(Scope *S, SourceLocation OpLoc,
-                                            tok::TokenKind Op, ExprArg input) {
-  return BuildUnaryOp(S, OpLoc, ConvertTokenKindToUnaryOpcode(Op), move(input));
+ExprResult Sema::ActOnUnaryOp(Scope *S, SourceLocation OpLoc,
+                                            tok::TokenKind Op, Expr *Input) {
+  return BuildUnaryOp(S, OpLoc, ConvertTokenKindToUnaryOpcode(Op), Input);
 }
 
 /// ActOnAddrLabel - Parse the GNU address of label extension: "&&foo".
-Sema::OwningExprResult Sema::ActOnAddrLabel(SourceLocation OpLoc,
+ExprResult Sema::ActOnAddrLabel(SourceLocation OpLoc,
                                             SourceLocation LabLoc,
                                             IdentifierInfo *LabelII) {
   // Look up the record for this label identifier.
-  LabelStmt *&LabelDecl = getLabelMap()[LabelII];
+  LabelStmt *&LabelDecl = getCurFunction()->LabelMap[LabelII];
 
   // If we haven't seen this label yet, create a forward reference. It
   // will be validated and/or cleaned up in ActOnFinishFunctionBody.
@@ -6709,10 +6836,9 @@ Sema::OwningExprResult Sema::ActOnAddrLabel(SourceLocation OpLoc,
                                        Context.getPointerType(Context.VoidTy)));
 }
 
-Sema::OwningExprResult
-Sema::ActOnStmtExpr(SourceLocation LPLoc, StmtArg substmt,
+ExprResult
+Sema::ActOnStmtExpr(SourceLocation LPLoc, Stmt *SubStmt,
                     SourceLocation RPLoc) { // "({..})"
-  Stmt *SubStmt = static_cast<Stmt*>(substmt.get());
   assert(SubStmt && isa<CompoundStmt>(SubStmt) && "Invalid action invocation!");
   CompoundStmt *Compound = cast<CompoundStmt>(SubStmt);
 
@@ -6742,11 +6868,10 @@ Sema::ActOnStmtExpr(SourceLocation LPLoc, StmtArg substmt,
   // FIXME: Check that expression type is complete/non-abstract; statement
   // expressions are not lvalues.
 
-  substmt.release();
   return Owned(new (Context) StmtExpr(Compound, Ty, LPLoc, RPLoc));
 }
 
-Sema::OwningExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
+ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
                                                   TypeSourceInfo *TInfo,
                                                   OffsetOfComponent *CompPtr,
                                                   unsigned NumComponents,
@@ -6865,22 +6990,27 @@ Sema::OwningExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
       Diag(MemberDecl->getLocation(), diag::note_bitfield_decl);
       return ExprError();
     }
-      
+
+    RecordDecl *Parent = MemberDecl->getParent();
+    bool AnonStructUnion = Parent->isAnonymousStructOrUnion();
+    if (AnonStructUnion) {
+      do {
+        Parent = cast<RecordDecl>(Parent->getParent());
+      } while (Parent->isAnonymousStructOrUnion());
+    }
+
     // If the member was found in a base class, introduce OffsetOfNodes for
     // the base class indirections.
     CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
                        /*DetectVirtual=*/false);
-    if (IsDerivedFrom(CurrentType, 
-                      Context.getTypeDeclType(MemberDecl->getParent()), 
-                      Paths)) {
+    if (IsDerivedFrom(CurrentType, Context.getTypeDeclType(Parent), Paths)) {
       CXXBasePath &Path = Paths.front();
       for (CXXBasePath::iterator B = Path.begin(), BEnd = Path.end();
            B != BEnd; ++B)
         Comps.push_back(OffsetOfNode(B->Base));
     }
-    
-    if (cast<RecordDecl>(MemberDecl->getDeclContext())->
-                                                isAnonymousStructOrUnion()) {
+
+    if (AnonStructUnion) {
       llvm::SmallVector<FieldDecl*, 4> Path;
       BuildAnonymousStructUnionMemberPath(MemberDecl, Path);
       unsigned n = Path.size();
@@ -6897,10 +7027,10 @@ Sema::OwningExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
                                     Exprs.data(), Exprs.size(), RParenLoc));  
 }
 
-Sema::OwningExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
+ExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
                                                   SourceLocation BuiltinLoc,
                                                   SourceLocation TypeLoc,
-                                                  TypeTy *argty,
+                                                  ParsedType argty,
                                                   OffsetOfComponent *CompPtr,
                                                   unsigned NumComponents,
                                                   SourceLocation RPLoc) {
@@ -6910,151 +7040,32 @@ Sema::OwningExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
   if (ArgTy.isNull())
     return ExprError();
 
-  if (getLangOptions().CPlusPlus) {
-    if (!ArgTInfo)
-      ArgTInfo = Context.getTrivialTypeSourceInfo(ArgTy, TypeLoc);
-    
-    return BuildBuiltinOffsetOf(BuiltinLoc, ArgTInfo, CompPtr, NumComponents, 
-                                RPLoc);
-  }
-  
-  // FIXME: The code below is marked for death, once we have proper CodeGen
-  // support for non-constant OffsetOf expressions.
-  
-  bool Dependent = ArgTy->isDependentType();
-  
-  // We must have at least one component that refers to the type, and the first
-  // one is known to be a field designator.  Verify that the ArgTy represents
-  // a struct/union/class.
-  if (!Dependent && !ArgTy->isRecordType())
-    return ExprError(Diag(TypeLoc, diag::err_offsetof_record_type) << ArgTy);
-  
-  // FIXME: Type must be complete per C99 7.17p3 because a declaring a variable
-  // with an incomplete type would be illegal.
-  
-  // Otherwise, create a null pointer as the base, and iteratively process
-  // the offsetof designators.
-  QualType ArgTyPtr = Context.getPointerType(ArgTy);
-  Expr* Res = new (Context) ImplicitValueInitExpr(ArgTyPtr);
-  Res = new (Context) UnaryOperator(Res, UnaryOperator::Deref,
-                                    ArgTy, SourceLocation());
-  
-  // offsetof with non-identifier designators (e.g. "offsetof(x, a.b[c])") are a
-  // GCC extension, diagnose them.
-  // FIXME: This diagnostic isn't actually visible because the location is in
-  // a system header!
-  if (NumComponents != 1)
-    Diag(BuiltinLoc, diag::ext_offsetof_extended_field_designator)
-    << SourceRange(CompPtr[1].LocStart, CompPtr[NumComponents-1].LocEnd);
-  
-  if (!Dependent) {
-    bool DidWarnAboutNonPOD = false;
-    
-    if (RequireCompleteType(TypeLoc, Res->getType(),
-                            diag::err_offsetof_incomplete_type))
-      return ExprError();
-    
-    // FIXME: Dependent case loses a lot of information here. And probably
-    // leaks like a sieve.
-    for (unsigned i = 0; i != NumComponents; ++i) {
-      const OffsetOfComponent &OC = CompPtr[i];
-      if (OC.isBrackets) {
-        // Offset of an array sub-field.  TODO: Should we allow vector elements?
-        const ArrayType *AT = Context.getAsArrayType(Res->getType());
-        if (!AT) {
-          Res->Destroy(Context);
-          return ExprError(Diag(OC.LocEnd, diag::err_offsetof_array_type)
-                           << Res->getType());
-        }
-        
-        // FIXME: C++: Verify that operator[] isn't overloaded.
-        
-        // Promote the array so it looks more like a normal array subscript
-        // expression.
-        DefaultFunctionArrayLvalueConversion(Res);
-        
-        // C99 6.5.2.1p1
-        Expr *Idx = static_cast<Expr*>(OC.U.E);
-        // FIXME: Leaks Res
-        if (!Idx->isTypeDependent() && !Idx->getType()->isIntegerType())
-          return ExprError(Diag(Idx->getLocStart(),
-                                diag::err_typecheck_subscript_not_integer)
-                           << Idx->getSourceRange());
-        
-        Res = new (Context) ArraySubscriptExpr(Res, Idx, AT->getElementType(),
-                                               OC.LocEnd);
-        continue;
-      }
-      
-      const RecordType *RC = Res->getType()->getAs<RecordType>();
-      if (!RC) {
-        Res->Destroy(Context);
-        return ExprError(Diag(OC.LocEnd, diag::err_offsetof_record_type)
-                         << Res->getType());
-      }
-      
-      // Get the decl corresponding to this.
-      RecordDecl *RD = RC->getDecl();
-      if (CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(RD)) {
-        if (!CRD->isPOD() && !DidWarnAboutNonPOD &&
-            DiagRuntimeBehavior(BuiltinLoc,
-                                PDiag(diag::warn_offsetof_non_pod_type)
-                                << SourceRange(CompPtr[0].LocStart, OC.LocEnd)
-                                << Res->getType()))
-          DidWarnAboutNonPOD = true;
-      }
-      
-      LookupResult R(*this, OC.U.IdentInfo, OC.LocStart, LookupMemberName);
-      LookupQualifiedName(R, RD);
-      
-      FieldDecl *MemberDecl = R.getAsSingle<FieldDecl>();
-      // FIXME: Leaks Res
-      if (!MemberDecl)
-        return ExprError(Diag(BuiltinLoc, diag::err_no_member)
-                         << OC.U.IdentInfo << RD << SourceRange(OC.LocStart, OC.LocEnd));
-      
-      // C99 7.17p3:
-      //   (If the specified member is a bit-field, the behavior is undefined.)
-      //
-      // We diagnose this as an error.
-      if (MemberDecl->getBitWidth()) {
-        Diag(OC.LocEnd, diag::err_offsetof_bitfield)
-          << MemberDecl->getDeclName()
-          << SourceRange(BuiltinLoc, RPLoc);
-        Diag(MemberDecl->getLocation(), diag::note_bitfield_decl);
-        return ExprError();
-      }
-      
-      // FIXME: C++: Verify that MemberDecl isn't a static field.
-      // FIXME: Verify that MemberDecl isn't a bitfield.
-      if (cast<RecordDecl>(MemberDecl->getDeclContext())->isAnonymousStructOrUnion()) {
-        Res = BuildAnonymousStructUnionMemberReference(
-                                                       OC.LocEnd, MemberDecl, Res, OC.LocEnd).takeAs<Expr>();
-      } else {
-        PerformObjectMemberConversion(Res, /*Qualifier=*/0,
-                                      *R.begin(), MemberDecl);
-        // MemberDecl->getType() doesn't get the right qualifiers, but it
-        // doesn't matter here.
-        Res = new (Context) MemberExpr(Res, false, MemberDecl, OC.LocEnd,
-                                       MemberDecl->getType().getNonReferenceType());
-      }
-    }
-  }
-  
-  return Owned(new (Context) UnaryOperator(Res, UnaryOperator::OffsetOf,
-                                           Context.getSizeType(), BuiltinLoc));
+  if (!ArgTInfo)
+    ArgTInfo = Context.getTrivialTypeSourceInfo(ArgTy, TypeLoc);
+
+  return BuildBuiltinOffsetOf(BuiltinLoc, ArgTInfo, CompPtr, NumComponents, 
+                              RPLoc);
 }
 
 
-Sema::OwningExprResult Sema::ActOnTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                                                      TypeTy *arg1,TypeTy *arg2,
+ExprResult Sema::ActOnTypesCompatibleExpr(SourceLocation BuiltinLoc,
+                                                      ParsedType arg1,ParsedType arg2,
                                                       SourceLocation RPLoc) {
-  // FIXME: Preserve type source info.
-  QualType argT1 = GetTypeFromParser(arg1);
-  QualType argT2 = GetTypeFromParser(arg2);
+  TypeSourceInfo *argTInfo1;
+  QualType argT1 = GetTypeFromParser(arg1, &argTInfo1);
+  TypeSourceInfo *argTInfo2;
+  QualType argT2 = GetTypeFromParser(arg2, &argTInfo2);
 
   assert((!argT1.isNull() && !argT2.isNull()) && "Missing type argument(s)");
 
+  return BuildTypesCompatibleExpr(BuiltinLoc, argTInfo1, argTInfo2, RPLoc);
+}
+
+ExprResult
+Sema::BuildTypesCompatibleExpr(SourceLocation BuiltinLoc,
+                               TypeSourceInfo *argTInfo1,
+                               TypeSourceInfo *argTInfo2,
+                               SourceLocation RPLoc) {
   if (getLangOptions().CPlusPlus) {
     Diag(BuiltinLoc, diag::err_types_compatible_p_in_cplusplus)
       << SourceRange(BuiltinLoc, RPLoc);
@@ -7062,17 +7073,14 @@ Sema::OwningExprResult Sema::ActOnTypesCompatibleExpr(SourceLocation BuiltinLoc,
   }
 
   return Owned(new (Context) TypesCompatibleExpr(Context.IntTy, BuiltinLoc,
-                                                 argT1, argT2, RPLoc));
+                                                 argTInfo1, argTInfo2, RPLoc));
 }
 
-Sema::OwningExprResult Sema::ActOnChooseExpr(SourceLocation BuiltinLoc,
-                                             ExprArg cond,
-                                             ExprArg expr1, ExprArg expr2,
-                                             SourceLocation RPLoc) {
-  Expr *CondExpr = static_cast<Expr*>(cond.get());
-  Expr *LHSExpr = static_cast<Expr*>(expr1.get());
-  Expr *RHSExpr = static_cast<Expr*>(expr2.get());
 
+ExprResult Sema::ActOnChooseExpr(SourceLocation BuiltinLoc,
+                                             Expr *CondExpr,
+                                             Expr *LHSExpr, Expr *RHSExpr,
+                                             SourceLocation RPLoc) {
   assert((CondExpr && LHSExpr && RHSExpr) && "Missing type argument(s)");
 
   QualType resType;
@@ -7095,7 +7103,6 @@ Sema::OwningExprResult Sema::ActOnChooseExpr(SourceLocation BuiltinLoc,
                                              : RHSExpr->isValueDependent();
   }
 
-  cond.release(); expr1.release(); expr2.release();
   return Owned(new (Context) ChooseExpr(BuiltinLoc, CondExpr, LHSExpr, RHSExpr,
                                         resType, RPLoc,
                                         resType->isDependentType(),
@@ -7232,8 +7239,8 @@ void Sema::ActOnBlockError(SourceLocation CaretLoc, Scope *CurScope) {
 
 /// ActOnBlockStmtExpr - This is called when the body of a block statement
 /// literal was successfully completed.  ^(int x){...}
-Sema::OwningExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
-                                                StmtArg body, Scope *CurScope) {
+ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
+                                                Stmt *Body, Scope *CurScope) {
   // If blocks are disabled, emit an error.
   if (!LangOpts.Blocks)
     Diag(CaretLoc, diag::err_blocks_disable);
@@ -7295,10 +7302,10 @@ Sema::OwningExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
   BlockTy = Context.getBlockPointerType(BlockTy);
 
   // If needed, diagnose invalid gotos and switches in the block.
-  if (FunctionNeedsScopeChecking() && !hasAnyErrorsInThisFunction())
-    DiagnoseInvalidJumps(static_cast<CompoundStmt*>(body.get()));
+  if (getCurFunction()->NeedsScopeChecking() && !hasAnyErrorsInThisFunction())
+    DiagnoseInvalidJumps(cast<CompoundStmt>(Body));
 
-  BSI->TheDecl->setBody(body.takeAs<CompoundStmt>());
+  BSI->TheDecl->setBody(cast<CompoundStmt>(Body));
 
   bool Good = true;
   // Check goto/label use.
@@ -7320,22 +7327,29 @@ Sema::OwningExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
     return ExprError();
   }
 
+  BlockExpr *Result = new (Context) BlockExpr(BSI->TheDecl, BlockTy,
+                                              BSI->hasBlockDeclRefExprs);
+
   // Issue any analysis-based warnings.
   const sema::AnalysisBasedWarnings::Policy &WP =
     AnalysisWarnings.getDefaultPolicy();
-  AnalysisWarnings.IssueWarnings(WP, BSI->TheDecl, BlockTy);
+  AnalysisWarnings.IssueWarnings(WP, Result);
 
-  Expr *Result = new (Context) BlockExpr(BSI->TheDecl, BlockTy,
-                                         BSI->hasBlockDeclRefExprs);
   PopFunctionOrBlockScope();
   return Owned(Result);
 }
 
-Sema::OwningExprResult Sema::ActOnVAArg(SourceLocation BuiltinLoc,
-                                        ExprArg expr, TypeTy *type,
+ExprResult Sema::ActOnVAArg(SourceLocation BuiltinLoc,
+                                        Expr *expr, ParsedType type,
                                         SourceLocation RPLoc) {
-  QualType T = GetTypeFromParser(type);
-  Expr *E = static_cast<Expr*>(expr.get());
+  TypeSourceInfo *TInfo;
+  QualType T = GetTypeFromParser(type, &TInfo);
+  return BuildVAArgExpr(BuiltinLoc, expr, TInfo, RPLoc);
+}
+
+ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
+                                            Expr *E, TypeSourceInfo *TInfo,
+                                            SourceLocation RPLoc) {
   Expr *OrigExpr = E;
 
   InitBuiltinVaListType();
@@ -7367,13 +7381,11 @@ Sema::OwningExprResult Sema::ActOnVAArg(SourceLocation BuiltinLoc,
   // FIXME: Check that type is complete/non-abstract
   // FIXME: Warn if a non-POD type is passed in.
 
-  expr.release();
-  return Owned(new (Context) VAArgExpr(BuiltinLoc, E, 
-                                       T.getNonLValueExprType(Context),
-                                       RPLoc));
+  QualType T = TInfo->getType().getNonLValueExprType(Context);
+  return Owned(new (Context) VAArgExpr(BuiltinLoc, E, TInfo, RPLoc, T));
 }
 
-Sema::OwningExprResult Sema::ActOnGNUNullExpr(SourceLocation TokenLoc) {
+ExprResult Sema::ActOnGNUNullExpr(SourceLocation TokenLoc) {
   // The type of __null will be int or long, depending on the size of
   // pointers on the target.
   QualType Ty;
@@ -7647,8 +7659,10 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
   if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(D)) {
     unsigned TypeQuals;
     if (Constructor->isImplicit() && Constructor->isDefaultConstructor()) {
-        if (!Constructor->isUsed(false))
-          DefineImplicitDefaultConstructor(Loc, Constructor);
+      if (Constructor->getParent()->hasTrivialConstructor())
+        return;
+      if (!Constructor->isUsed(false))
+        DefineImplicitDefaultConstructor(Loc, Constructor);
     } else if (Constructor->isImplicit() &&
                Constructor->isCopyConstructor(TypeQuals)) {
       if (!Constructor->isUsed(false))
@@ -7696,13 +7710,20 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
           PendingLocalImplicitInstantiations.push_back(std::make_pair(Function,
                                                                       Loc));
         else
-          PendingImplicitInstantiations.push_back(std::make_pair(Function,
-                                                                 Loc));
+          PendingInstantiations.push_back(std::make_pair(Function, Loc));
+      }
+    } else // Walk redefinitions, as some of them may be instantiable.
+      for (FunctionDecl::redecl_iterator i(Function->redecls_begin()),
+           e(Function->redecls_end()); i != e; ++i) {
+        if (!i->isUsed(false) && i->isImplicitlyInstantiable())
+          MarkDeclarationReferenced(Loc, *i);
       }
-    }
 
     // FIXME: keep track of references to static functions
-    Function->setUsed(true);
+
+    // Recursive functions should be marked when used from another function.
+    if (CurContext != Function)
+      Function->setUsed(true);
 
     return;
   }
@@ -7716,7 +7737,7 @@ void Sema::MarkDeclarationReferenced(SourceLocation Loc, Decl *D) {
       if (MSInfo->getPointOfInstantiation().isInvalid() &&
           MSInfo->getTemplateSpecializationKind()== TSK_ImplicitInstantiation) {
         MSInfo->setPointOfInstantiation(Loc);
-        PendingImplicitInstantiations.push_back(std::make_pair(Var, Loc));
+        PendingInstantiations.push_back(std::make_pair(Var, Loc));
       }
     }
 
@@ -7836,7 +7857,7 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
 
   if (isa<BinaryOperator>(E)) {
     BinaryOperator *Op = cast<BinaryOperator>(E);
-    if (Op->getOpcode() != BinaryOperator::Assign)
+    if (Op->getOpcode() != BO_Assign)
       return;
 
     // Greylist some idioms by putting them into a warning subcategory.
@@ -7899,16 +7920,13 @@ bool Sema::CheckBooleanCondition(Expr *&E, SourceLocation Loc) {
   return false;
 }
 
-Sema::OwningExprResult Sema::ActOnBooleanCondition(Scope *S, SourceLocation Loc,
-                                                   ExprArg SubExpr) {
-  Expr *Sub = SubExpr.takeAs<Expr>();
+ExprResult Sema::ActOnBooleanCondition(Scope *S, SourceLocation Loc,
+                                       Expr *Sub) {
   if (!Sub)
     return ExprError();
   
-  if (CheckBooleanCondition(Sub, Loc)) {
-    Sub->Destroy(Context);
+  if (CheckBooleanCondition(Sub, Loc))
     return ExprError();
-  }
   
   return Owned(Sub);
 }
diff --git a/lib/Sema/SemaExprCXX.cpp b/lib/Sema/SemaExprCXX.cpp
index 090400fc4e23..5720d931b66b 100644
--- a/lib/Sema/SemaExprCXX.cpp
+++ b/lib/Sema/SemaExprCXX.cpp
@@ -11,28 +11,31 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/ParsedTemplate.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Template.h"
 #include "llvm/ADT/STLExtras.h"
 using namespace clang;
-
-Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
-                                        IdentifierInfo &II, 
-                                        SourceLocation NameLoc,
-                                        Scope *S, CXXScopeSpec &SS,
-                                        TypeTy *ObjectTypePtr,
-                                        bool EnteringContext) {
+using namespace sema;
+
+ParsedType Sema::getDestructorName(SourceLocation TildeLoc,
+                                   IdentifierInfo &II, 
+                                   SourceLocation NameLoc,
+                                   Scope *S, CXXScopeSpec &SS,
+                                   ParsedType ObjectTypePtr,
+                                   bool EnteringContext) {
   // Determine where to perform name lookup.
 
   // FIXME: This area of the standard is very messy, and the current
@@ -149,7 +152,7 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
 
     // FIXME: Should we be suppressing ambiguities here?
     if (Found.isAmbiguous())
-      return 0;
+      return ParsedType();
 
     if (TypeDecl *Type = Found.getAsSingle<TypeDecl>()) {
       QualType T = Context.getTypeDeclType(Type);
@@ -158,7 +161,7 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
           Context.hasSameUnqualifiedType(T, SearchType)) {
         // We found our type!
 
-        return T.getAsOpaquePtr();
+        return ParsedType::make(T);
       }
     }
 
@@ -191,7 +194,7 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
               = dyn_cast<ClassTemplateSpecializationDecl>(Record->getDecl())) {
           if (Spec->getSpecializedTemplate()->getCanonicalDecl() ==
                 Template->getCanonicalDecl())
-            return MemberOfType.getAsOpaquePtr();
+            return ParsedType::make(MemberOfType);
         }
 
         continue;
@@ -210,7 +213,7 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
         // specialized.
         if (TemplateDecl *SpecTemplate = SpecName.getAsTemplateDecl()) {
           if (SpecTemplate->getCanonicalDecl() == Template->getCanonicalDecl())
-            return MemberOfType.getAsOpaquePtr();
+            return ParsedType::make(MemberOfType);
 
           continue;
         }
@@ -221,7 +224,7 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
                                     = SpecName.getAsDependentTemplateName()) {
           if (DepTemplate->isIdentifier() &&
               DepTemplate->getIdentifier() == Template->getIdentifier())
-            return MemberOfType.getAsOpaquePtr();
+            return ParsedType::make(MemberOfType);
 
           continue;
         }
@@ -242,8 +245,10 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
       Range = SourceRange(NameLoc);
     }
 
-    return CheckTypenameType(ETK_None, NNS, II, SourceLocation(),
-                             Range, NameLoc).getAsOpaquePtr();
+    QualType T = CheckTypenameType(ETK_None, NNS, II,
+                                   SourceLocation(),
+                                   Range, NameLoc);
+    return ParsedType::make(T);
   }
 
   if (ObjectTypePtr)
@@ -252,11 +257,11 @@ Action::TypeTy *Sema::getDestructorName(SourceLocation TildeLoc,
   else
     Diag(NameLoc, diag::err_destructor_class_name);
 
-  return 0;
+  return ParsedType();
 }
 
 /// \brief Build a C++ typeid expression with a type operand.
-Sema::OwningExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
+ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
                                             SourceLocation TypeidLoc,
                                             TypeSourceInfo *Operand,
                                             SourceLocation RParenLoc) {
@@ -279,12 +284,11 @@ Sema::OwningExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
 }
 
 /// \brief Build a C++ typeid expression with an expression operand.
-Sema::OwningExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
+ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
                                             SourceLocation TypeidLoc,
-                                            ExprArg Operand,
+                                            Expr *E,
                                             SourceLocation RParenLoc) {
   bool isUnevaluatedOperand = true;
-  Expr *E = static_cast<Expr *>(Operand.get());
   if (E && !E->isTypeDependent()) {
     QualType T = E->getType();
     if (const RecordType *RecordT = T->getAs<RecordType>()) {
@@ -296,10 +300,10 @@ Sema::OwningExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
         return ExprError();
       
       // C++ [expr.typeid]p3:
-      //   When typeid is applied to an expression other than an lvalue of a
+      //   When typeid is applied to an expression other than an glvalue of a
       //   polymorphic class type [...] [the] expression is an unevaluated
       //   operand. [...]
-      if (RecordD->isPolymorphic() && E->isLvalue(Context) == Expr::LV_Valid) {
+      if (RecordD->isPolymorphic() && E->Classify(Context).isGLValue()) {
         isUnevaluatedOperand = false;
 
         // We require a vtable to query the type at run time.
@@ -316,9 +320,7 @@ Sema::OwningExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
     QualType UnqualT = Context.getUnqualifiedArrayType(T, Quals);
     if (!Context.hasSameType(T, UnqualT)) {
       T = UnqualT;
-      ImpCastExprToType(E, UnqualT, CastExpr::CK_NoOp, E->isLvalue(Context));
-      Operand.release();
-      Operand = Owned(E);
+      ImpCastExprToType(E, UnqualT, CK_NoOp, CastCategory(E));
     }
   }
   
@@ -329,12 +331,12 @@ Sema::OwningExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
     ExprEvalContexts.back().Context = Unevaluated;
   
   return Owned(new (Context) CXXTypeidExpr(TypeInfoType.withConst(),
-                                           Operand.takeAs<Expr>(),
+                                           E,
                                            SourceRange(TypeidLoc, RParenLoc)));  
 }
 
 /// ActOnCXXTypeidOfType - Parse typeid( type-id ) or typeid (expression);
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
                      bool isType, void *TyOrExpr, SourceLocation RParenLoc) {
   // Find the std::type_info type.
@@ -343,7 +345,7 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
 
   IdentifierInfo *TypeInfoII = &PP.getIdentifierTable().get("type_info");
   LookupResult R(*this, TypeInfoII, SourceLocation(), LookupTagName);
-  LookupQualifiedName(R, StdNamespace);
+  LookupQualifiedName(R, getStdNamespace());
   RecordDecl *TypeInfoRecordDecl = R.getAsSingle<RecordDecl>();
   if (!TypeInfoRecordDecl)
     return ExprError(Diag(OpLoc, diag::err_need_header_before_typeid));
@@ -353,7 +355,8 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
   if (isType) {
     // The operand is a type; handle it as such.
     TypeSourceInfo *TInfo = 0;
-    QualType T = GetTypeFromParser(TyOrExpr, &TInfo);
+    QualType T = GetTypeFromParser(ParsedType::getFromOpaquePtr(TyOrExpr),
+                                   &TInfo);
     if (T.isNull())
       return ExprError();
     
@@ -364,11 +367,11 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
   }
 
   // The operand is an expression.  
-  return BuildCXXTypeId(TypeInfoType, OpLoc, Owned((Expr*)TyOrExpr), RParenLoc);
+  return BuildCXXTypeId(TypeInfoType, OpLoc, (Expr*)TyOrExpr, RParenLoc);
 }
 
 /// ActOnCXXBoolLiteral - Parse {true,false} literals.
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCXXBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind) {
   assert((Kind == tok::kw_true || Kind == tok::kw_false) &&
          "Unknown C++ Boolean value!");
@@ -377,15 +380,14 @@ Sema::ActOnCXXBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind) {
 }
 
 /// ActOnCXXNullPtrLiteral - Parse 'nullptr'.
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCXXNullPtrLiteral(SourceLocation Loc) {
   return Owned(new (Context) CXXNullPtrLiteralExpr(Context.NullPtrTy, Loc));
 }
 
 /// ActOnCXXThrow - Parse throw expressions.
-Action::OwningExprResult
-Sema::ActOnCXXThrow(SourceLocation OpLoc, ExprArg E) {
-  Expr *Ex = E.takeAs<Expr>();
+ExprResult
+Sema::ActOnCXXThrow(SourceLocation OpLoc, Expr *Ex) {
   if (Ex && !Ex->isTypeDependent() && CheckCXXThrowOperand(OpLoc, Ex))
     return ExprError();
   return Owned(new (Context) CXXThrowExpr(Ex, Context.VoidTy, OpLoc));
@@ -400,8 +402,8 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
   //   the type from "array of T" or "function returning T" to "pointer to T" 
   //   or "pointer to function returning T", [...]
   if (E->getType().hasQualifiers())
-    ImpCastExprToType(E, E->getType().getUnqualifiedType(), CastExpr::CK_NoOp,
-                      E->isLvalue(Context) == Expr::LV_Valid);
+    ImpCastExprToType(E, E->getType().getUnqualifiedType(), CK_NoOp,
+                      CastCategory(E));
   
   DefaultFunctionArrayConversion(E);
 
@@ -432,7 +434,7 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
   InitializedEntity Entity =
     InitializedEntity::InitializeException(ThrowLoc, E->getType(),
                                            /*NRVO=*/false);
-  OwningExprResult Res = PerformCopyInitialization(Entity,
+  ExprResult Res = PerformCopyInitialization(Entity,
                                                    SourceLocation(),
                                                    Owned(E));
   if (Res.isInvalid())
@@ -464,7 +466,7 @@ bool Sema::CheckCXXThrowOperand(SourceLocation ThrowLoc, Expr *&E) {
   return false;
 }
 
-Action::OwningExprResult Sema::ActOnCXXThis(SourceLocation ThisLoc) {
+ExprResult Sema::ActOnCXXThis(SourceLocation ThisLoc) {
   /// C++ 9.3.2: In the body of a non-static member function, the keyword this
   /// is a non-lvalue expression whose value is the address of the object for
   /// which the function is called.
@@ -483,8 +485,8 @@ Action::OwningExprResult Sema::ActOnCXXThis(SourceLocation ThisLoc) {
 /// Can be interpreted either as function-style casting ("int(x)")
 /// or class type construction ("ClassType(x,y,z)")
 /// or creation of a value-initialized type ("int()").
-Action::OwningExprResult
-Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, TypeTy *TypeRep,
+ExprResult
+Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, ParsedType TypeRep,
                                 SourceLocation LParenLoc,
                                 MultiExprArg exprs,
                                 SourceLocation *CommaLocs,
@@ -532,19 +534,19 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, TypeTy *TypeRep,
   // corresponding cast expression.
   //
   if (NumExprs == 1) {
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-    CXXBaseSpecifierArray BasePath;
+    CastKind Kind = CK_Unknown;
+    CXXCastPath BasePath;
     if (CheckCastTypes(TypeRange, Ty, Exprs[0], Kind, BasePath,
                        /*FunctionalStyle=*/true))
       return ExprError();
 
     exprs.release();
 
-    return Owned(new (Context) CXXFunctionalCastExpr(
+    return Owned(CXXFunctionalCastExpr::Create(Context,
                                               Ty.getNonLValueExprType(Context),
-                                                     TInfo, TyBeginLoc, Kind,
-                                                     Exprs[0], BasePath,
-                                                     RParenLoc));
+                                               TInfo, TyBeginLoc, Kind,
+                                               Exprs[0], &BasePath,
+                                               RParenLoc));
   }
 
   if (Ty->isRecordType()) {
@@ -555,7 +557,7 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, TypeTy *TypeRep,
                  : InitializationKind::CreateValue(TypeRange.getBegin(), 
                                                    LParenLoc, RParenLoc);
     InitializationSequence InitSeq(*this, Entity, Kind, Exprs, NumExprs);
-    OwningExprResult Result = InitSeq.Perform(*this, Entity, Kind,
+    ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
                                               move(exprs));
 
     // FIXME: Improve AST representation?
@@ -587,7 +589,7 @@ Sema::ActOnCXXTypeConstructExpr(SourceRange TypeRange, TypeTy *TypeRep,
 /// or
 /// @code ::new Foo(23, "hello") @endcode
 /// For the interpretation of this heap of arguments, consult the base version.
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
                   SourceLocation PlacementLParen, MultiExprArg PlacementArgs,
                   SourceLocation PlacementRParen, SourceRange TypeIdParens, 
@@ -643,13 +645,13 @@ Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
                      AllocType,
                      D.getSourceRange().getBegin(),
                      R,
-                     Owned(ArraySize),
+                     ArraySize,
                      ConstructorLParen,
                      move(ConstructorArgs),
                      ConstructorRParen);
 }
 
-Sema::OwningExprResult
+ExprResult
 Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                   SourceLocation PlacementLParen,
                   MultiExprArg PlacementArgs,
@@ -658,7 +660,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                   QualType AllocType,
                   SourceLocation TypeLoc,
                   SourceRange TypeRange,
-                  ExprArg ArraySizeE,
+                  Expr *ArraySize,
                   SourceLocation ConstructorLParen,
                   MultiExprArg ConstructorArgs,
                   SourceLocation ConstructorRParen) {
@@ -667,12 +669,12 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
 
   // Per C++0x [expr.new]p5, the type being constructed may be a
   // typedef of an array type.
-  if (!ArraySizeE.get()) {
+  if (!ArraySize) {
     if (const ConstantArrayType *Array
                               = Context.getAsConstantArrayType(AllocType)) {
-      ArraySizeE = Owned(new (Context) IntegerLiteral(Array->getSize(),
-                                                      Context.getSizeType(),
-                                                      TypeRange.getEnd()));
+      ArraySize = IntegerLiteral::Create(Context, Array->getSize(),
+                                         Context.getSizeType(),
+                                         TypeRange.getEnd());
       AllocType = Array->getElementType();
     }
   }
@@ -681,13 +683,12 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
 
   // C++ 5.3.4p6: "The expression in a direct-new-declarator shall have integral
   //   or enumeration type with a non-negative value."
-  Expr *ArraySize = (Expr *)ArraySizeE.get();
   if (ArraySize && !ArraySize->isTypeDependent()) {
     
     QualType SizeType = ArraySize->getType();
     
-    OwningExprResult ConvertedSize
-      = ConvertToIntegralOrEnumerationType(StartLoc, move(ArraySizeE), 
+    ExprResult ConvertedSize
+      = ConvertToIntegralOrEnumerationType(StartLoc, ArraySize,
                                        PDiag(diag::err_array_size_not_integral),
                                      PDiag(diag::err_array_size_incomplete_type)
                                        << ArraySize->getSourceRange(),
@@ -700,8 +701,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     if (ConvertedSize.isInvalid())
       return ExprError();
     
-    ArraySize = ConvertedSize.takeAs<Expr>();
-    ArraySizeE = Owned(ArraySize);
+    ArraySize = ConvertedSize.take();
     SizeType = ArraySize->getType();
     if (!SizeType->isIntegralOrEnumerationType())
       return ExprError();
@@ -716,8 +716,20 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                         llvm::APInt::getNullValue(Value.getBitWidth()), 
                                  Value.isUnsigned()))
           return ExprError(Diag(ArraySize->getSourceRange().getBegin(),
-                           diag::err_typecheck_negative_array_size)
+                                diag::err_typecheck_negative_array_size)
             << ArraySize->getSourceRange());
+        
+        if (!AllocType->isDependentType()) {
+          unsigned ActiveSizeBits
+            = ConstantArrayType::getNumAddressingBits(Context, AllocType, Value);
+          if (ActiveSizeBits > ConstantArrayType::getMaxSizeBits(Context)) {
+            Diag(ArraySize->getSourceRange().getBegin(), 
+                 diag::err_array_too_large)
+              << Value.toString(10)
+              << ArraySize->getSourceRange();
+            return ExprError();
+          }
+        }
       } else if (TypeIdParens.isValid()) {
         // Can't have dynamic array size when the type-id is in parentheses.
         Diag(ArraySize->getLocStart(), diag::ext_new_paren_array_nonconst)
@@ -730,7 +742,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     }
     
     ImpCastExprToType(ArraySize, Context.getSizeType(),
-                      CastExpr::CK_IntegralCast);
+                      CK_IntegralCast);
   }
 
   FunctionDecl *OperatorNew = 0;
@@ -768,7 +780,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
   CXXConstructorDecl *Constructor = 0;
   Expr **ConsArgs = (Expr**)ConstructorArgs.get();
   unsigned NumConsArgs = ConstructorArgs.size();
-  ASTOwningVector<&ActionBase::DeleteExpr> ConvertedConstructorArgs(*this);
+  ASTOwningVector<Expr*> ConvertedConstructorArgs(*this);
 
   // Array 'new' can't have any initializers.
   if (NumConsArgs && (ResultType->isArrayType() || ArraySize)) {
@@ -798,7 +810,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     InitializedEntity Entity
       = InitializedEntity::InitializeNew(StartLoc, AllocType);
     InitializationSequence InitSeq(*this, Entity, Kind, ConsArgs, NumConsArgs);
-    OwningExprResult FullInit = InitSeq.Perform(*this, Entity, Kind, 
+    ExprResult FullInit = InitSeq.Perform(*this, Entity, Kind, 
                                                 move(ConstructorArgs));
     if (FullInit.isInvalid())
       return ExprError();
@@ -839,7 +851,6 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
   
   PlacementArgs.release();
   ConstructorArgs.release();
-  ArraySizeE.release();
   
   // FIXME: The TypeSourceInfo should also be included in CXXNewExpr.
   return Owned(new (Context) CXXNewExpr(Context, UseGlobal, OperatorNew,
@@ -911,7 +922,7 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
   // We don't care about the actual value of this argument.
   // FIXME: Should the Sema create the expression and embed it in the syntax
   // tree? Or should the consumer just recalculate the value?
-  IntegerLiteral Size(llvm::APInt::getNullValue(
+  IntegerLiteral Size(Context, llvm::APInt::getNullValue(
                       Context.Target.getPointerWidth(0)),
                       Context.getSizeType(),
                       SourceLocation());
@@ -929,9 +940,11 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
   DeclarationName DeleteName = Context.DeclarationNames.getCXXOperatorName(
                                         IsArray ? OO_Array_Delete : OO_Delete);
 
-  if (AllocType->isRecordType() && !UseGlobal) {
+  QualType AllocElemType = Context.getBaseElementType(AllocType);
+
+  if (AllocElemType->isRecordType() && !UseGlobal) {
     CXXRecordDecl *Record
-      = cast<CXXRecordDecl>(AllocType->getAs<RecordType>()->getDecl());
+      = cast<CXXRecordDecl>(AllocElemType->getAs<RecordType>()->getDecl());
     if (FindAllocationOverload(StartLoc, Range, NewName, &AllocArgs[0],
                           AllocArgs.size(), Record, /*AllowMissing=*/true,
                           OperatorNew))
@@ -969,9 +982,9 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
   //   the allocated type is not a class type or array thereof, the
   //   deallocation function’s name is looked up in the global scope.
   LookupResult FoundDelete(*this, DeleteName, StartLoc, LookupOrdinaryName);
-  if (AllocType->isRecordType() && !UseGlobal) {
+  if (AllocElemType->isRecordType() && !UseGlobal) {
     CXXRecordDecl *RD
-      = cast<CXXRecordDecl>(AllocType->getAs<RecordType>()->getDecl());
+      = cast<CXXRecordDecl>(AllocElemType->getAs<RecordType>()->getDecl());
     LookupQualifiedName(FoundDelete, RD);
   }
   if (FoundDelete.isAmbiguous())
@@ -1115,7 +1128,7 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
 
   // Do the resolution.
   OverloadCandidateSet::iterator Best;
-  switch(BestViableFunction(Candidates, StartLoc, Best)) {
+  switch (Candidates.BestViableFunction(*this, StartLoc, Best)) {
   case OR_Success: {
     // Got one!
     FunctionDecl *FnDecl = Best->Function;
@@ -1125,7 +1138,7 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
     // Watch out for variadic allocator function.
     unsigned NumArgsInFnDecl = FnDecl->getNumParams();
     for (unsigned i = 0; (i < NumArgs && i < NumArgsInFnDecl); ++i) {
-      OwningExprResult Result
+      ExprResult Result
         = PerformCopyInitialization(InitializedEntity::InitializeParameter(
                                                        FnDecl->getParamDecl(i)),
                                     SourceLocation(),
@@ -1143,20 +1156,20 @@ bool Sema::FindAllocationOverload(SourceLocation StartLoc, SourceRange Range,
   case OR_No_Viable_Function:
     Diag(StartLoc, diag::err_ovl_no_viable_function_in_call)
       << Name << Range;
-    PrintOverloadCandidates(Candidates, OCD_AllCandidates, Args, NumArgs);
+    Candidates.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
     return true;
 
   case OR_Ambiguous:
     Diag(StartLoc, diag::err_ovl_ambiguous_call)
       << Name << Range;
-    PrintOverloadCandidates(Candidates, OCD_ViableCandidates, Args, NumArgs);
+    Candidates.NoteCandidates(*this, OCD_ViableCandidates, Args, NumArgs);
     return true;
 
   case OR_Deleted:
     Diag(StartLoc, diag::err_ovl_deleted_call)
       << Best->Function->isDeleted()
       << Name << Range;
-    PrintOverloadCandidates(Candidates, OCD_AllCandidates, Args, NumArgs);
+    Candidates.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
     return true;
   }
   assert(false && "Unreachable, bad result from BestViableFunction");
@@ -1199,11 +1212,11 @@ void Sema::DeclareGlobalNewDelete() {
     // The "std::bad_alloc" class has not yet been declared, so build it
     // implicitly.
     StdBadAlloc = CXXRecordDecl::Create(Context, TTK_Class, 
-                                        getStdNamespace(), 
+                                        getOrCreateStdNamespace(), 
                                         SourceLocation(), 
                                       &PP.getIdentifierTable().get("bad_alloc"), 
                                         SourceLocation(), 0);
-    StdBadAlloc->setImplicit(true);
+    getStdBadAlloc()->setImplicit(true);
   }
   
   GlobalNewDeleteDeclared = true;
@@ -1245,8 +1258,11 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
           Context.getCanonicalType(
             Func->getParamDecl(0)->getType().getUnqualifiedType());
         // FIXME: Do we need to check for default arguments here?
-        if (Func->getNumParams() == 1 && InitialParamType == Argument)
+        if (Func->getNumParams() == 1 && InitialParamType == Argument) {
+          if(AddMallocAttr && !Func->hasAttr<MallocAttr>())
+            Func->addAttr(::new (Context) MallocAttr(SourceLocation(), Context));
           return;
+        }
       }
     }
   }
@@ -1257,7 +1273,7 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
        Name.getCXXOverloadedOperator() == OO_Array_New);
   if (HasBadAllocExceptionSpec) {
     assert(StdBadAlloc && "Must have std::bad_alloc declared");
-    BadAllocType = Context.getTypeDeclType(StdBadAlloc);
+    BadAllocType = Context.getTypeDeclType(getStdBadAlloc());
   }
   
   QualType FnType = Context.getFunctionType(Return, &Argument, 1, false, 0,
@@ -1267,23 +1283,23 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
                                             FunctionType::ExtInfo());
   FunctionDecl *Alloc =
     FunctionDecl::Create(Context, GlobalCtx, SourceLocation(), Name,
-                         FnType, /*TInfo=*/0, FunctionDecl::None,
-                         FunctionDecl::None, false, true);
+                         FnType, /*TInfo=*/0, SC_None,
+                         SC_None, false, true);
   Alloc->setImplicit();
   
   if (AddMallocAttr)
-    Alloc->addAttr(::new (Context) MallocAttr());
+    Alloc->addAttr(::new (Context) MallocAttr(SourceLocation(), Context));
   
   ParmVarDecl *Param = ParmVarDecl::Create(Context, Alloc, SourceLocation(),
                                            0, Argument, /*TInfo=*/0,
-                                           VarDecl::None,
-                                           VarDecl::None, 0);
+                                           SC_None,
+                                           SC_None, 0);
   Alloc->setParams(&Param, 1);
 
   // FIXME: Also add this declaration to the IdentifierResolver, but
   // make sure it is at the end of the chain to coincide with the
   // global scope.
-  ((DeclContext *)TUScope->getEntity())->addDecl(Alloc);
+  Context.getTranslationUnitDecl()->addDecl(Alloc);
 }
 
 bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
@@ -1298,15 +1314,37 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
 
   Found.suppressDiagnostics();
 
+  llvm::SmallVector<DeclAccessPair,4> Matches;
   for (LookupResult::iterator F = Found.begin(), FEnd = Found.end();
        F != FEnd; ++F) {
-    if (CXXMethodDecl *Delete = dyn_cast<CXXMethodDecl>(*F))
-      if (Delete->isUsualDeallocationFunction()) {
-        Operator = Delete;
-        CheckAllocationAccess(StartLoc, SourceRange(), Found.getNamingClass(),
-                              F.getPair());
-        return false;
-      }
+    NamedDecl *ND = (*F)->getUnderlyingDecl();
+
+    // Ignore template operator delete members from the check for a usual
+    // deallocation function.
+    if (isa<FunctionTemplateDecl>(ND))
+      continue;
+
+    if (cast<CXXMethodDecl>(ND)->isUsualDeallocationFunction())
+      Matches.push_back(F.getPair());
+  }
+
+  // There's exactly one suitable operator;  pick it.
+  if (Matches.size() == 1) {
+    Operator = cast<CXXMethodDecl>(Matches[0]->getUnderlyingDecl());
+    CheckAllocationAccess(StartLoc, SourceRange(), Found.getNamingClass(),
+                          Matches[0]);
+    return false;
+
+  // We found multiple suitable operators;  complain about the ambiguity.
+  } else if (!Matches.empty()) {
+    Diag(StartLoc, diag::err_ambiguous_suitable_delete_member_function_found)
+      << Name << RD;
+
+    for (llvm::SmallVectorImpl<DeclAccessPair>::iterator
+           F = Matches.begin(), FEnd = Matches.end(); F != FEnd; ++F)
+      Diag((*F)->getUnderlyingDecl()->getLocation(),
+           diag::note_member_declared_here) << Name;
+    return true;
   }
 
   // We did find operator delete/operator delete[] declarations, but
@@ -1316,10 +1354,9 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
       << Name << RD;
         
     for (LookupResult::iterator F = Found.begin(), FEnd = Found.end();
-         F != FEnd; ++F) {
-      Diag((*F)->getLocation(), diag::note_member_declared_here)
-        << Name;
-    }
+         F != FEnd; ++F)
+      Diag((*F)->getUnderlyingDecl()->getLocation(),
+           diag::note_member_declared_here) << Name;
 
     return true;
   }
@@ -1344,9 +1381,9 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
 /// @code ::delete ptr; @endcode
 /// or
 /// @code delete [] ptr; @endcode
-Action::OwningExprResult
+ExprResult
 Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
-                     bool ArrayForm, ExprArg Operand) {
+                     bool ArrayForm, Expr *Ex) {
   // C++ [expr.delete]p1:
   //   The operand shall have a pointer type, or a class type having a single
   //   conversion function to a pointer type. The result has type void.
@@ -1355,11 +1392,14 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
 
   FunctionDecl *OperatorDelete = 0;
 
-  Expr *Ex = (Expr *)Operand.get();
   if (!Ex->isTypeDependent()) {
     QualType Type = Ex->getType();
 
     if (const RecordType *Record = Type->getAs<RecordType>()) {
+      if (RequireCompleteType(StartLoc, Type, 
+                              PDiag(diag::err_delete_incomplete_class_type)))
+        return ExprError();
+      
       llvm::SmallVector<CXXConversionDecl*, 4> ObjectPtrConversions;
 
       CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
@@ -1378,18 +1418,16 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
         
         QualType ConvType = Conv->getConversionType().getNonReferenceType();
         if (const PointerType *ConvPtrType = ConvType->getAs<PointerType>())
-          if (ConvPtrType->getPointeeType()->isObjectType())
+          if (ConvPtrType->getPointeeType()->isIncompleteOrObjectType())
             ObjectPtrConversions.push_back(Conv);
       }
       if (ObjectPtrConversions.size() == 1) {
         // We have a single conversion to a pointer-to-object type. Perform
         // that conversion.
         // TODO: don't redo the conversion calculation.
-        Operand.release();
         if (!PerformImplicitConversion(Ex,
                             ObjectPtrConversions.front()->getConversionType(),
                                       AA_Converting)) {
-          Operand = Owned(Ex);
           Type = Ex->getType();
         }
       }
@@ -1428,16 +1466,13 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     //   (5.2.11) of the pointer expression before it is used as the operand 
     //   of the delete-expression. ]
     ImpCastExprToType(Ex, Context.getPointerType(Context.VoidTy), 
-                      CastExpr::CK_NoOp);
-    
-    // Update the operand.
-    Operand.take();
-    Operand = ExprArg(*this, Ex);
+                      CK_NoOp);
     
     DeclarationName DeleteName = Context.DeclarationNames.getCXXOperatorName(
                                       ArrayForm ? OO_Array_Delete : OO_Delete);
 
-    if (const RecordType *RT = Pointee->getAs<RecordType>()) {
+    QualType PointeeElem = Context.getBaseElementType(Pointee);
+    if (const RecordType *RT = PointeeElem->getAs<RecordType>()) {
       CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
 
       if (!UseGlobal && 
@@ -1465,14 +1500,13 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
     // FIXME: Check access and ambiguity of operator delete and destructor.
   }
 
-  Operand.release();
   return Owned(new (Context) CXXDeleteExpr(Context.VoidTy, UseGlobal, ArrayForm,
                                            OperatorDelete, Ex, StartLoc));
 }
 
 /// \brief Check the use of the given variable as a C++ condition in an if,
 /// while, do-while, or switch statement.
-Action::OwningExprResult Sema::CheckConditionVariable(VarDecl *ConditionVar,
+ExprResult Sema::CheckConditionVariable(VarDecl *ConditionVar,
                                                       SourceLocation StmtLoc,
                                                       bool ConvertToBoolean) {
   QualType T = ConditionVar->getType();
@@ -1491,10 +1525,8 @@ Action::OwningExprResult Sema::CheckConditionVariable(VarDecl *ConditionVar,
   Expr *Condition = DeclRefExpr::Create(Context, 0, SourceRange(), ConditionVar,
                                         ConditionVar->getLocation(), 
                                  ConditionVar->getType().getNonReferenceType());
-  if (ConvertToBoolean && CheckBooleanCondition(Condition, StmtLoc)) {
-    Condition->Destroy(Context);
+  if (ConvertToBoolean && CheckBooleanCondition(Condition, StmtLoc))
     return ExprError();
-  }
   
   return Owned(Condition);
 }
@@ -1543,34 +1575,33 @@ Sema::IsStringLiteralToNonConstPointerConversion(Expr *From, QualType ToType) {
   return false;
 }
 
-static Sema::OwningExprResult BuildCXXCastArgument(Sema &S, 
-                                                   SourceLocation CastLoc,
-                                                   QualType Ty,
-                                                   CastExpr::CastKind Kind,
-                                                   CXXMethodDecl *Method,
-                                                   Sema::ExprArg Arg) {
-  Expr *From = Arg.takeAs<Expr>();
-  
+static ExprResult BuildCXXCastArgument(Sema &S, 
+                                       SourceLocation CastLoc,
+                                       QualType Ty,
+                                       CastKind Kind,
+                                       CXXMethodDecl *Method,
+                                       Expr *From) {
   switch (Kind) {
   default: assert(0 && "Unhandled cast kind!");
-  case CastExpr::CK_ConstructorConversion: {
-    ASTOwningVector<&ActionBase::DeleteExpr> ConstructorArgs(S);
+  case CK_ConstructorConversion: {
+    ASTOwningVector<Expr*> ConstructorArgs(S);
     
     if (S.CompleteConstructorCall(cast<CXXConstructorDecl>(Method),
-                                  Sema::MultiExprArg(S, (void **)&From, 1),
+                                  MultiExprArg(&From, 1),
                                   CastLoc, ConstructorArgs))
-      return S.ExprError();
+      return ExprError();
     
-    Sema::OwningExprResult Result = 
+    ExprResult Result = 
     S.BuildCXXConstructExpr(CastLoc, Ty, cast<CXXConstructorDecl>(Method), 
-                            move_arg(ConstructorArgs));
+                            move_arg(ConstructorArgs),
+                            /*ZeroInit*/ false, CXXConstructExpr::CK_Complete);
     if (Result.isInvalid())
-      return S.ExprError();
+      return ExprError();
     
     return S.MaybeBindToTemporary(Result.takeAs<Expr>());
   }
     
-  case CastExpr::CK_UserDefinedConversion: {
+  case CK_UserDefinedConversion: {
     assert(!From->getType()->isPointerType() && "Arg can't have pointer type!");
     
     // Create an implicit call expr that calls it.
@@ -1601,10 +1632,10 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   case ImplicitConversionSequence::UserDefinedConversion: {
     
       FunctionDecl *FD = ICS.UserDefined.ConversionFunction;
-      CastExpr::CastKind CastKind = CastExpr::CK_Unknown;
+      CastKind CastKind = CK_Unknown;
       QualType BeforeToType;
       if (const CXXConversionDecl *Conv = dyn_cast<CXXConversionDecl>(FD)) {
-        CastKind = CastExpr::CK_UserDefinedConversion;
+        CastKind = CK_UserDefinedConversion;
         
         // If the user-defined conversion is specified by a conversion function,
         // the initial standard conversion sequence converts the source type to
@@ -1612,7 +1643,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
         BeforeToType = Context.getTagDeclType(Conv->getParent());
       } else if (const CXXConstructorDecl *Ctor = 
                   dyn_cast<CXXConstructorDecl>(FD)) {
-        CastKind = CastExpr::CK_ConstructorConversion;
+        CastKind = CK_ConstructorConversion;
         // Do no conversion if dealing with ... for the first conversion.
         if (!ICS.UserDefined.EllipsisConversion) {
           // If the user-defined conversion is specified by a constructor, the 
@@ -1631,12 +1662,12 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
           return true;
       }
     
-      OwningExprResult CastArg 
+      ExprResult CastArg 
         = BuildCXXCastArgument(*this,
                                From->getLocStart(),
                                ToType.getNonReferenceType(),
                                CastKind, cast<CXXMethodDecl>(FD), 
-                               Owned(From));
+                               From);
 
       if (CastArg.isInvalid())
         return true;
@@ -1648,7 +1679,7 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   }
 
   case ImplicitConversionSequence::AmbiguousConversion:
-    DiagnoseAmbiguousConversion(ICS, From->getExprLoc(),
+    ICS.DiagnoseAmbiguousConversion(*this, From->getExprLoc(),
                           PDiag(diag::err_typecheck_ambiguous_condition)
                             << From->getSourceRange());
      return true;
@@ -1685,25 +1716,29 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     // FIXME: When can ToType be a reference type?
     assert(!ToType->isReferenceType());
     if (SCS.Second == ICK_Derived_To_Base) {
-      ASTOwningVector<&ActionBase::DeleteExpr> ConstructorArgs(*this);
+      ASTOwningVector<Expr*> ConstructorArgs(*this);
       if (CompleteConstructorCall(cast<CXXConstructorDecl>(SCS.CopyConstructor),
-                                  MultiExprArg(*this, (void **)&From, 1),
+                                  MultiExprArg(*this, &From, 1),
                                   /*FIXME:ConstructLoc*/SourceLocation(), 
                                   ConstructorArgs))
         return true;
-      OwningExprResult FromResult =
+      ExprResult FromResult =
         BuildCXXConstructExpr(/*FIXME:ConstructLoc*/SourceLocation(),
                               ToType, SCS.CopyConstructor,
-                              move_arg(ConstructorArgs));
+                              move_arg(ConstructorArgs),
+                              /*ZeroInit*/ false,
+                              CXXConstructExpr::CK_Complete);
       if (FromResult.isInvalid())
         return true;
       From = FromResult.takeAs<Expr>();
       return false;
     }
-    OwningExprResult FromResult =
+    ExprResult FromResult =
       BuildCXXConstructExpr(/*FIXME:ConstructLoc*/SourceLocation(),
                             ToType, SCS.CopyConstructor,
-                            MultiExprArg(*this, (void**)&From, 1));
+                            MultiExprArg(*this, &From, 1),
+                            /*ZeroInit*/ false,
+                            CXXConstructExpr::CK_Complete);
 
     if (FromResult.isInvalid())
       return true;
@@ -1736,12 +1771,12 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
 
   case ICK_Array_To_Pointer:
     FromType = Context.getArrayDecayedType(FromType);
-    ImpCastExprToType(From, FromType, CastExpr::CK_ArrayToPointerDecay);
+    ImpCastExprToType(From, FromType, CK_ArrayToPointerDecay);
     break;
 
   case ICK_Function_To_Pointer:
     FromType = Context.getPointerType(FromType);
-    ImpCastExprToType(From, FromType, CastExpr::CK_FunctionToPointerDecay);
+    ImpCastExprToType(From, FromType, CK_FunctionToPointerDecay);
     break;
 
   default:
@@ -1766,33 +1801,33 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
       return true;      
       
     ImpCastExprToType(From, Context.getNoReturnType(From->getType(), false),
-                      CastExpr::CK_NoOp);
+                      CK_NoOp);
     break;
       
   case ICK_Integral_Promotion:
   case ICK_Integral_Conversion:
-    ImpCastExprToType(From, ToType, CastExpr::CK_IntegralCast);
+    ImpCastExprToType(From, ToType, CK_IntegralCast);
     break;
 
   case ICK_Floating_Promotion:
   case ICK_Floating_Conversion:
-    ImpCastExprToType(From, ToType, CastExpr::CK_FloatingCast);
+    ImpCastExprToType(From, ToType, CK_FloatingCast);
     break;
 
   case ICK_Complex_Promotion:
   case ICK_Complex_Conversion:
-    ImpCastExprToType(From, ToType, CastExpr::CK_Unknown);
+    ImpCastExprToType(From, ToType, CK_Unknown);
     break;
 
   case ICK_Floating_Integral:
     if (ToType->isRealFloatingType())
-      ImpCastExprToType(From, ToType, CastExpr::CK_IntegralToFloating);
+      ImpCastExprToType(From, ToType, CK_IntegralToFloating);
     else
-      ImpCastExprToType(From, ToType, CastExpr::CK_FloatingToIntegral);
+      ImpCastExprToType(From, ToType, CK_FloatingToIntegral);
     break;
 
   case ICK_Compatible_Conversion:
-    ImpCastExprToType(From, ToType, CastExpr::CK_NoOp);
+    ImpCastExprToType(From, ToType, CK_NoOp);
     break;
 
   case ICK_Pointer_Conversion: {
@@ -1805,36 +1840,36 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     }
 
     
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-    CXXBaseSpecifierArray BasePath;
+    CastKind Kind = CK_Unknown;
+    CXXCastPath BasePath;
     if (CheckPointerConversion(From, ToType, Kind, BasePath, IgnoreBaseAccess))
       return true;
-    ImpCastExprToType(From, ToType, Kind, /*isLvalue=*/false, BasePath);
+    ImpCastExprToType(From, ToType, Kind, VK_RValue, &BasePath);
     break;
   }
   
   case ICK_Pointer_Member: {
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
-    CXXBaseSpecifierArray BasePath;
+    CastKind Kind = CK_Unknown;
+    CXXCastPath BasePath;
     if (CheckMemberPointerConversion(From, ToType, Kind, BasePath,
                                      IgnoreBaseAccess))
       return true;
     if (CheckExceptionSpecCompatibility(From, ToType))
       return true;
-    ImpCastExprToType(From, ToType, Kind, /*isLvalue=*/false, BasePath);
+    ImpCastExprToType(From, ToType, Kind, VK_RValue, &BasePath);
     break;
   }
   case ICK_Boolean_Conversion: {
-    CastExpr::CastKind Kind = CastExpr::CK_Unknown;
+    CastKind Kind = CK_Unknown;
     if (FromType->isMemberPointerType())
-      Kind = CastExpr::CK_MemberPointerToBoolean;
+      Kind = CK_MemberPointerToBoolean;
     
     ImpCastExprToType(From, Context.BoolTy, Kind);
     break;
   }
 
   case ICK_Derived_To_Base: {
-    CXXBaseSpecifierArray BasePath;
+    CXXCastPath BasePath;
     if (CheckDerivedToBaseConversion(From->getType(), 
                                      ToType.getNonReferenceType(),
                                      From->getLocStart(),
@@ -1843,24 +1878,22 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                                      IgnoreBaseAccess))
       return true;
 
-    ImpCastExprToType(From, ToType.getNonReferenceType(), 
-                      CastExpr::CK_DerivedToBase, 
-                      /*isLvalue=*/(From->getType()->isRecordType() &&
-                                    From->isLvalue(Context) == Expr::LV_Valid),
-                      BasePath);
+    ImpCastExprToType(From, ToType.getNonReferenceType(),
+                      CK_DerivedToBase, CastCategory(From),
+                      &BasePath);
     break;
   }
 
   case ICK_Vector_Conversion:
-    ImpCastExprToType(From, ToType, CastExpr::CK_BitCast);
+    ImpCastExprToType(From, ToType, CK_BitCast);
     break;
 
   case ICK_Vector_Splat:
-    ImpCastExprToType(From, ToType, CastExpr::CK_VectorSplat);
+    ImpCastExprToType(From, ToType, CK_VectorSplat);
     break;
       
   case ICK_Complex_Real:
-    ImpCastExprToType(From, ToType, CastExpr::CK_Unknown);
+    ImpCastExprToType(From, ToType, CK_Unknown);
     break;
       
   case ICK_Lvalue_To_Rvalue:
@@ -1877,18 +1910,21 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
     // Nothing to do.
     break;
 
-  case ICK_Qualification:
-    // FIXME: Not sure about lvalue vs rvalue here in the presence of rvalue
-    // references.
+  case ICK_Qualification: {
+    // The qualification keeps the category of the inner expression, unless the
+    // target type isn't a reference.
+    ExprValueKind VK = ToType->isReferenceType() ?
+                                  CastCategory(From) : VK_RValue;
     ImpCastExprToType(From, ToType.getNonLValueExprType(Context),
-                      CastExpr::CK_NoOp, ToType->isLValueReferenceType());
+                      CK_NoOp, VK);
 
     if (SCS.DeprecatedStringLiteralToCharPtr)
       Diag(From->getLocStart(), diag::warn_deprecated_string_literal_conversion)
         << ToType.getNonReferenceType();
 
     break;
-      
+    }
+
   default:
     assert(false && "Improper third standard conversion");
     break;
@@ -1897,10 +1933,10 @@ Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
   return false;
 }
 
-Sema::OwningExprResult Sema::ActOnUnaryTypeTrait(UnaryTypeTrait OTT,
+ExprResult Sema::ActOnUnaryTypeTrait(UnaryTypeTrait OTT,
                                                  SourceLocation KWLoc,
                                                  SourceLocation LParen,
-                                                 TypeTy *Ty,
+                                                 ParsedType Ty,
                                                  SourceLocation RParen) {
   QualType T = GetTypeFromParser(Ty);
 
@@ -1974,12 +2010,12 @@ QualType Sema::CheckPointerToMemberOperands(
     }
     // Cast LHS to type of use.
     QualType UseType = isIndirect ? Context.getPointerType(Class) : Class;
-    bool isLValue = !isIndirect && lex->isLvalue(Context) == Expr::LV_Valid;
-    
-    CXXBaseSpecifierArray BasePath;
+    ExprValueKind VK =
+        isIndirect ? VK_RValue : CastCategory(lex);
+
+    CXXCastPath BasePath;
     BuildBasePathArray(Paths, BasePath);
-    ImpCastExprToType(lex, UseType, CastExpr::CK_DerivedToBase, isLValue,
-                      BasePath);
+    ImpCastExprToType(lex, UseType, CK_DerivedToBase, VK, &BasePath);
   }
 
   if (isa<CXXScalarValueInitExpr>(rex->IgnoreParens())) {
@@ -2108,7 +2144,7 @@ static bool FindConditionalOverload(Sema &Self, Expr *&LHS, Expr *&RHS,
   Self.AddBuiltinOperatorCandidates(OO_Conditional, Loc, Args, 2, CandidateSet);
 
   OverloadCandidateSet::iterator Best;
-  switch (Self.BestViableFunction(CandidateSet, Loc, Best)) {
+  switch (CandidateSet.BestViableFunction(Self, Loc, Best)) {
     case OR_Success:
       // We found a match. Perform the conversions on the arguments and move on.
       if (Self.PerformImplicitConversion(LHS, Best->BuiltinTypes.ParamTypes[0],
@@ -2146,8 +2182,7 @@ static bool ConvertForConditional(Sema &Self, Expr *&E, QualType T) {
   InitializationKind Kind = InitializationKind::CreateCopy(E->getLocStart(),
                                                            SourceLocation());
   InitializationSequence InitSeq(Self, Entity, Kind, &E, 1);
-  Sema::OwningExprResult Result = InitSeq.Perform(Self, Entity, Kind, 
-                                    Sema::MultiExprArg(Self, (void **)&E, 1));
+  ExprResult Result = InitSeq.Perform(Self, Entity, Kind, MultiExprArg(&E, 1));
   if (Result.isInvalid())
     return true;
   
@@ -2286,13 +2321,13 @@ QualType Sema::CXXCheckConditionalOperands(Expr *&Cond, Expr *&LHS, Expr *&RHS,
     if (LTy->isRecordType()) {
       // The operands have class type. Make a temporary copy.
       InitializedEntity Entity = InitializedEntity::InitializeTemporary(LTy);
-      OwningExprResult LHSCopy = PerformCopyInitialization(Entity, 
+      ExprResult LHSCopy = PerformCopyInitialization(Entity, 
                                                            SourceLocation(), 
                                                            Owned(LHS));
       if (LHSCopy.isInvalid())
         return QualType();
         
-      OwningExprResult RHSCopy = PerformCopyInitialization(Entity, 
+      ExprResult RHSCopy = PerformCopyInitialization(Entity, 
                                                            SourceLocation(), 
                                                            Owned(RHS));
       if (RHSCopy.isInvalid())
@@ -2385,16 +2420,16 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
   //   the type of the other operand.
   if (E1->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
     if (T2->isMemberPointerType())
-      ImpCastExprToType(E1, T2, CastExpr::CK_NullToMemberPointer);
+      ImpCastExprToType(E1, T2, CK_NullToMemberPointer);
     else
-      ImpCastExprToType(E1, T2, CastExpr::CK_IntegralToPointer);
+      ImpCastExprToType(E1, T2, CK_IntegralToPointer);
     return T2;
   }
   if (E2->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull)) {
     if (T1->isMemberPointerType())
-      ImpCastExprToType(E2, T1, CastExpr::CK_NullToMemberPointer);
+      ImpCastExprToType(E2, T1, CK_NullToMemberPointer);
     else
-      ImpCastExprToType(E2, T1, CastExpr::CK_IntegralToPointer);
+      ImpCastExprToType(E2, T1, CK_IntegralToPointer);
     return T1;
   }
 
@@ -2528,15 +2563,15 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
     }
 
     // Convert E1 to Composite1
-    OwningExprResult E1Result
-      = E1ToC1.Perform(*this, Entity1, Kind, MultiExprArg(*this,(void**)&E1,1));
+    ExprResult E1Result
+      = E1ToC1.Perform(*this, Entity1, Kind, MultiExprArg(*this,&E1,1));
     if (E1Result.isInvalid())
       return QualType();
     E1 = E1Result.takeAs<Expr>();
 
     // Convert E2 to Composite1
-    OwningExprResult E2Result
-      = E2ToC1.Perform(*this, Entity1, Kind, MultiExprArg(*this,(void**)&E2,1));
+    ExprResult E2Result
+      = E2ToC1.Perform(*this, Entity1, Kind, MultiExprArg(*this,&E2,1));
     if (E2Result.isInvalid())
       return QualType();
     E2 = E2Result.takeAs<Expr>();
@@ -2553,15 +2588,15 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
     return QualType();
   
   // Convert E1 to Composite2
-  OwningExprResult E1Result
-    = E1ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, (void**)&E1, 1));
+  ExprResult E1Result
+    = E1ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, &E1, 1));
   if (E1Result.isInvalid())
     return QualType();
   E1 = E1Result.takeAs<Expr>();
   
   // Convert E2 to Composite2
-  OwningExprResult E2Result
-    = E2ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, (void**)&E2, 1));
+  ExprResult E2Result
+    = E2ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, &E2, 1));
   if (E2Result.isInvalid())
     return QualType();
   E2 = E2Result.takeAs<Expr>();
@@ -2569,7 +2604,7 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
   return Composite2;
 }
 
-Sema::OwningExprResult Sema::MaybeBindToTemporary(Expr *E) {
+ExprResult Sema::MaybeBindToTemporary(Expr *E) {
   if (!Context.getLangOptions().CPlusPlus)
     return Owned(E);
 
@@ -2579,34 +2614,22 @@ Sema::OwningExprResult Sema::MaybeBindToTemporary(Expr *E) {
   if (!RT)
     return Owned(E);
 
-  // If this is the result of a call expression, our source might
-  // actually be a reference, in which case we shouldn't bind.
+  // If this is the result of a call or an Objective-C message send expression,
+  // our source might actually be a reference, in which case we shouldn't bind.
   if (CallExpr *CE = dyn_cast<CallExpr>(E)) {
-    QualType Ty = CE->getCallee()->getType();
-    if (const PointerType *PT = Ty->getAs<PointerType>())
-      Ty = PT->getPointeeType();
-    else if (const BlockPointerType *BPT = Ty->getAs<BlockPointerType>())
-      Ty = BPT->getPointeeType();
-
-    const FunctionType *FTy = Ty->getAs<FunctionType>();
-    if (FTy->getResultType()->isReferenceType())
+    if (CE->getCallReturnType()->isReferenceType())
       return Owned(E);
+  } else if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E)) {
+    if (const ObjCMethodDecl *MD = ME->getMethodDecl()) {
+      if (MD->getResultType()->isReferenceType())
+        return Owned(E);    
+    }
   }
-  else if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(E)) {
-    QualType Ty = ME->getType();
-    if (const PointerType *PT = Ty->getAs<PointerType>())
-      Ty = PT->getPointeeType();
-    else if (const BlockPointerType *BPT = Ty->getAs<BlockPointerType>())
-      Ty = BPT->getPointeeType();
-    if (Ty->isReferenceType())
-      return Owned(E);    
-  }
-
 
   // That should be enough to guarantee that this type is complete.
   // If it has a trivial destructor, we can avoid the extra copy.
   CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
-  if (RD->hasTrivialDestructor())
+  if (RD->isInvalidDecl() || RD->hasTrivialDestructor())
     return Owned(E);
 
   CXXTemporary *Temp = CXXTemporary::Create(Context, LookupDestructor(RD));
@@ -2641,8 +2664,8 @@ Expr *Sema::MaybeCreateCXXExprWithTemporaries(Expr *SubExpr) {
   return E;
 }
 
-Sema::OwningExprResult 
-Sema::MaybeCreateCXXExprWithTemporaries(OwningExprResult SubExpr) {
+ExprResult 
+Sema::MaybeCreateCXXExprWithTemporaries(ExprResult SubExpr) {
   if (SubExpr.isInvalid())
     return ExprError();
   
@@ -2665,17 +2688,16 @@ FullExpr Sema::CreateFullExpr(Expr *SubExpr) {
   return E;
 }
 
-Sema::OwningExprResult
-Sema::ActOnStartCXXMemberReference(Scope *S, ExprArg Base, SourceLocation OpLoc,
-                                   tok::TokenKind OpKind, TypeTy *&ObjectType,
+ExprResult
+Sema::ActOnStartCXXMemberReference(Scope *S, Expr *Base, SourceLocation OpLoc,
+                                   tok::TokenKind OpKind, ParsedType &ObjectType,
                                    bool &MayBePseudoDestructor) {
   // Since this might be a postfix expression, get rid of ParenListExprs.
-  Base = MaybeConvertParenListExprToParenExpr(S, move(Base));
+  ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Base);
+  if (Result.isInvalid()) return ExprError();
+  Base = Result.get();
 
-  Expr *BaseExpr = (Expr*)Base.get();
-  assert(BaseExpr && "no record expansion");
-
-  QualType BaseType = BaseExpr->getType();
+  QualType BaseType = Base->getType();
   MayBePseudoDestructor = false;
   if (BaseType->isDependentType()) {
     // If we have a pointer to a dependent type and are using the -> operator,
@@ -2685,9 +2707,9 @@ Sema::ActOnStartCXXMemberReference(Scope *S, ExprArg Base, SourceLocation OpLoc,
       if (const PointerType *Ptr = BaseType->getAs<PointerType>())
         BaseType = Ptr->getPointeeType();
     
-    ObjectType = BaseType.getAsOpaquePtr();
+    ObjectType = ParsedType::make(BaseType);
     MayBePseudoDestructor = true;
-    return move(Base);
+    return Owned(Base);
   }
 
   // C++ [over.match.oper]p8:
@@ -2700,13 +2722,13 @@ Sema::ActOnStartCXXMemberReference(Scope *S, ExprArg Base, SourceLocation OpLoc,
     CTypes.insert(Context.getCanonicalType(BaseType));
     
     while (BaseType->isRecordType()) {
-      Base = BuildOverloadedArrowExpr(S, move(Base), OpLoc);
-      BaseExpr = (Expr*)Base.get();
-      if (BaseExpr == NULL)
+      Result = BuildOverloadedArrowExpr(S, Base, OpLoc);
+      if (Result.isInvalid())
         return ExprError();
-      if (CXXOperatorCallExpr *OpCall = dyn_cast<CXXOperatorCallExpr>(BaseExpr))
+      Base = Result.get();
+      if (CXXOperatorCallExpr *OpCall = dyn_cast<CXXOperatorCallExpr>(Base))
         Locations.push_back(OpCall->getDirectCallee()->getLocation());
-      BaseType = BaseExpr->getType();
+      BaseType = Base->getType();
       CanQualType CBaseType = Context.getCanonicalType(BaseType);
       if (!CTypes.insert(CBaseType)) {
         Diag(OpLoc, diag::err_operator_arrow_circular);
@@ -2731,9 +2753,9 @@ Sema::ActOnStartCXXMemberReference(Scope *S, ExprArg Base, SourceLocation OpLoc,
     //
     // This also indicates that we should be parsing a
     // pseudo-destructor-name.
-    ObjectType = 0;
+    ObjectType = ParsedType();
     MayBePseudoDestructor = true;
-    return move(Base);
+    return Owned(Base);
   }
 
   // The object type must be complete (or dependent).
@@ -2747,27 +2769,26 @@ Sema::ActOnStartCXXMemberReference(Scope *S, ExprArg Base, SourceLocation OpLoc,
   //   unqualified-id, and the type of the object expression is of a class
   //   type C (or of pointer to a class type C), the unqualified-id is looked
   //   up in the scope of class C. [...]
-  ObjectType = BaseType.getAsOpaquePtr();
+  ObjectType = ParsedType::make(BaseType);
   return move(Base);
 }
 
-Sema::OwningExprResult Sema::DiagnoseDtorReference(SourceLocation NameLoc,
-                                                   ExprArg MemExpr) {
-  Expr *E = (Expr *) MemExpr.get();
+ExprResult Sema::DiagnoseDtorReference(SourceLocation NameLoc,
+                                                   Expr *MemExpr) {
   SourceLocation ExpectedLParenLoc = PP.getLocForEndOfToken(NameLoc);
-  Diag(E->getLocStart(), diag::err_dtor_expr_without_call)
-    << isa<CXXPseudoDestructorExpr>(E)
+  Diag(MemExpr->getLocStart(), diag::err_dtor_expr_without_call)
+    << isa<CXXPseudoDestructorExpr>(MemExpr)
     << FixItHint::CreateInsertion(ExpectedLParenLoc, "()");
   
   return ActOnCallExpr(/*Scope*/ 0,
-                       move(MemExpr),
+                       MemExpr,
                        /*LPLoc*/ ExpectedLParenLoc,
-                       Sema::MultiExprArg(*this, 0, 0),
+                       MultiExprArg(),
                        /*CommaLocs*/ 0,
                        /*RPLoc*/ ExpectedLParenLoc);
 }
 
-Sema::OwningExprResult Sema::BuildPseudoDestructorExpr(ExprArg Base,
+ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
                                                        SourceLocation OpLoc,
                                                        tok::TokenKind OpKind,
                                                        const CXXScopeSpec &SS,
@@ -2782,12 +2803,11 @@ Sema::OwningExprResult Sema::BuildPseudoDestructorExpr(ExprArg Base,
   //   The left-hand side of the dot operator shall be of scalar type. The 
   //   left-hand side of the arrow operator shall be of pointer to scalar type.
   //   This scalar type is the object type. 
-  Expr *BaseE = (Expr *)Base.get();
-  QualType ObjectType = BaseE->getType();
+  QualType ObjectType = Base->getType();
   if (OpKind == tok::arrow) {
     if (const PointerType *Ptr = ObjectType->getAs<PointerType>()) {
       ObjectType = Ptr->getPointeeType();
-    } else if (!BaseE->isTypeDependent()) {
+    } else if (!Base->isTypeDependent()) {
       // The user wrote "p->" when she probably meant "p."; fix it.
       Diag(OpLoc, diag::err_typecheck_member_reference_suggestion)
         << ObjectType << true
@@ -2801,7 +2821,7 @@ Sema::OwningExprResult Sema::BuildPseudoDestructorExpr(ExprArg Base,
   
   if (!ObjectType->isDependentType() && !ObjectType->isScalarType()) {
     Diag(OpLoc, diag::err_pseudo_dtor_base_not_scalar)
-      << ObjectType << BaseE->getSourceRange();
+      << ObjectType << Base->getSourceRange();
     return ExprError();
   }
 
@@ -2815,7 +2835,7 @@ Sema::OwningExprResult Sema::BuildPseudoDestructorExpr(ExprArg Base,
     if (!DestructedType->isDependentType() && !ObjectType->isDependentType() &&
         !Context.hasSameUnqualifiedType(DestructedType, ObjectType)) {
       Diag(DestructedTypeStart, diag::err_pseudo_dtor_type_mismatch)
-        << ObjectType << DestructedType << BaseE->getSourceRange()
+        << ObjectType << DestructedType << Base->getSourceRange()
         << DestructedTypeInfo->getTypeLoc().getLocalSourceRange();
       
       // Recover by setting the destructed type to the object type.
@@ -2840,7 +2860,7 @@ Sema::OwningExprResult Sema::BuildPseudoDestructorExpr(ExprArg Base,
       
       Diag(ScopeTypeInfo->getTypeLoc().getLocalSourceRange().getBegin(),
            diag::err_pseudo_dtor_type_mismatch)
-        << ObjectType << ScopeType << BaseE->getSourceRange()
+        << ObjectType << ScopeType << Base->getSourceRange()
         << ScopeTypeInfo->getTypeLoc().getLocalSourceRange();
   
       ScopeType = QualType();
@@ -2848,25 +2868,22 @@ Sema::OwningExprResult Sema::BuildPseudoDestructorExpr(ExprArg Base,
     }
   }
   
-  OwningExprResult Result
-    = Owned(new (Context) CXXPseudoDestructorExpr(Context, 
-                                                  Base.takeAs<Expr>(),
-                                                  OpKind == tok::arrow,
-                                                  OpLoc,
-                                       (NestedNameSpecifier *) SS.getScopeRep(),
-                                                  SS.getRange(),
-                                                  ScopeTypeInfo,
-                                                  CCLoc,
-                                                  TildeLoc,
-                                                  Destructed));
+  Expr *Result
+    = new (Context) CXXPseudoDestructorExpr(Context, Base,
+                                            OpKind == tok::arrow, OpLoc,
+                                            SS.getScopeRep(), SS.getRange(),
+                                            ScopeTypeInfo,
+                                            CCLoc,
+                                            TildeLoc,
+                                            Destructed);
             
   if (HasTrailingLParen)
-    return move(Result);
+    return Owned(Result);
   
-  return DiagnoseDtorReference(Destructed.getLocation(), move(Result));
+  return DiagnoseDtorReference(Destructed.getLocation(), Result);
 }
 
-Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
+ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
                                                        SourceLocation OpLoc,
                                                        tok::TokenKind OpKind,
                                                        CXXScopeSpec &SS,
@@ -2882,13 +2899,11 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
           SecondTypeName.getKind() == UnqualifiedId::IK_Identifier) &&
          "Invalid second type name in pseudo-destructor");
 
-  Expr *BaseE = (Expr *)Base.get();
-  
   // C++ [expr.pseudo]p2:
   //   The left-hand side of the dot operator shall be of scalar type. The 
   //   left-hand side of the arrow operator shall be of pointer to scalar type.
   //   This scalar type is the object type. 
-  QualType ObjectType = BaseE->getType();
+  QualType ObjectType = Base->getType();
   if (OpKind == tok::arrow) {
     if (const PointerType *Ptr = ObjectType->getAs<PointerType>()) {
       ObjectType = Ptr->getPointeeType();
@@ -2906,12 +2921,12 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
 
   // Compute the object type that we should use for name lookup purposes. Only
   // record types and dependent types matter.
-  void *ObjectTypePtrForLookup = 0;
+  ParsedType ObjectTypePtrForLookup;
   if (!SS.isSet()) {
-    ObjectTypePtrForLookup = const_cast<RecordType*>(
-                                               ObjectType->getAs<RecordType>());
-    if (!ObjectTypePtrForLookup && ObjectType->isDependentType())
-      ObjectTypePtrForLookup = Context.DependentTy.getAsOpaquePtr();
+    if (const Type *T = ObjectType->getAs<RecordType>())
+      ObjectTypePtrForLookup = ParsedType::make(QualType(T, 0));
+    else if (ObjectType->isDependentType())
+      ObjectTypePtrForLookup = ParsedType::make(Context.DependentTy);
   }
   
   // Convert the name of the type being destructed (following the ~) into a 
@@ -2920,9 +2935,9 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
   TypeSourceInfo *DestructedTypeInfo = 0;
   PseudoDestructorTypeStorage Destructed;
   if (SecondTypeName.getKind() == UnqualifiedId::IK_Identifier) {
-    TypeTy *T = getTypeName(*SecondTypeName.Identifier, 
-                            SecondTypeName.StartLocation,
-                            S, &SS, true, ObjectTypePtrForLookup);
+    ParsedType T = getTypeName(*SecondTypeName.Identifier, 
+                               SecondTypeName.StartLocation,
+                               S, &SS, true, ObjectTypePtrForLookup);
     if (!T && 
         ((SS.isSet() && !computeDeclContext(SS, false)) ||
          (!SS.isSet() && ObjectType->isDependentType()))) {
@@ -2949,7 +2964,7 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
     ASTTemplateArgsPtr TemplateArgsPtr(*this,
                                        TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
-    TypeResult T = ActOnTemplateIdType(TemplateTy::make(TemplateId->Template),
+    TypeResult T = ActOnTemplateIdType(TemplateId->Template,
                                        TemplateId->TemplateNameLoc,
                                        TemplateId->LAngleLoc,
                                        TemplateArgsPtr,
@@ -2976,9 +2991,9 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
   if (FirstTypeName.getKind() == UnqualifiedId::IK_TemplateId || 
       FirstTypeName.Identifier) {
     if (FirstTypeName.getKind() == UnqualifiedId::IK_Identifier) {
-      TypeTy *T = getTypeName(*FirstTypeName.Identifier, 
-                              FirstTypeName.StartLocation,
-                              S, &SS, false, ObjectTypePtrForLookup);
+      ParsedType T = getTypeName(*FirstTypeName.Identifier, 
+                                 FirstTypeName.StartLocation,
+                                 S, &SS, false, ObjectTypePtrForLookup);
       if (!T) {
         Diag(FirstTypeName.StartLocation, 
              diag::err_pseudo_dtor_destructor_non_type)
@@ -2997,7 +3012,7 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
       ASTTemplateArgsPtr TemplateArgsPtr(*this,
                                          TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
-      TypeResult T = ActOnTemplateIdType(TemplateTy::make(TemplateId->Template),
+      TypeResult T = ActOnTemplateIdType(TemplateId->Template,
                                          TemplateId->TemplateNameLoc,
                                          TemplateId->LAngleLoc,
                                          TemplateArgsPtr,
@@ -3015,7 +3030,7 @@ Sema::OwningExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, ExprArg Base,
                                                   FirstTypeName.StartLocation);
 
     
-  return BuildPseudoDestructorExpr(move(Base), OpLoc, OpKind, SS,
+  return BuildPseudoDestructorExpr(Base, OpLoc, OpKind, SS,
                                    ScopeTypeInfo, CCLoc, TildeLoc,
                                    Destructed, HasTrailingLParen);
 }
@@ -3028,7 +3043,7 @@ CXXMemberCallExpr *Sema::BuildCXXMemberCallExpr(Expr *Exp,
     assert(0 && "Calling BuildCXXMemberCallExpr with invalid call?");
 
   MemberExpr *ME = 
-      new (Context) MemberExpr(Exp, /*IsArrow=*/false, Method, 
+      new (Context) MemberExpr(Exp, /*IsArrow=*/false, Method,
                                SourceLocation(), Method->getType());
   QualType ResultType = Method->getCallResultType();
   MarkDeclarationReferenced(Exp->getLocStart(), Method);
@@ -3038,12 +3053,7 @@ CXXMemberCallExpr *Sema::BuildCXXMemberCallExpr(Expr *Exp,
   return CE;
 }
 
-Sema::OwningExprResult Sema::ActOnFinishFullExpr(ExprArg Arg) {
-  Expr *FullExpr = Arg.takeAs<Expr>();
-  if (FullExpr)
-    FullExpr = MaybeCreateCXXExprWithTemporaries(FullExpr);
-  else
-    return ExprError();
-  
-  return Owned(FullExpr);
+ExprResult Sema::ActOnFinishFullExpr(Expr *FullExpr) {
+  if (!FullExpr) return ExprError();
+  return MaybeCreateCXXExprWithTemporaries(FullExpr);
 }
diff --git a/lib/Sema/SemaExprObjC.cpp b/lib/Sema/SemaExprObjC.cpp
index 9f43471e0ade..b56159c453b5 100644
--- a/lib/Sema/SemaExprObjC.cpp
+++ b/lib/Sema/SemaExprObjC.cpp
@@ -11,9 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "Lookup.h"
-#include "SemaInit.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/Initialization.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprObjC.h"
@@ -23,9 +24,9 @@
 
 using namespace clang;
 
-Sema::ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
-                                              ExprTy **strings,
-                                              unsigned NumStrings) {
+ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
+                                        Expr **strings,
+                                        unsigned NumStrings) {
   StringLiteral **Strings = reinterpret_cast<StringLiteral**>(strings);
 
   // Most ObjC strings are formed out of a single piece.  However, we *can*
@@ -50,14 +51,11 @@ Sema::ExprResult Sema::ParseObjCStringLiteral(SourceLocation *AtLocs,
         return true;
       }
 
-      // Get the string data.
-      StrBuf.append(S->getStrData(), S->getStrData()+S->getByteLength());
+      // Append the string.
+      StrBuf += S->getString();
 
       // Get the locations of the string tokens.
       StrLocs.append(S->tokloc_begin(), S->tokloc_end());
-
-      // Free the temporary string.
-      S->Destroy(Context);
     }
 
     // Create the aggregate string with the appropriate content and location
@@ -135,11 +133,11 @@ Expr *Sema::BuildObjCEncodeExpression(SourceLocation AtLoc,
   return new (Context) ObjCEncodeExpr(StrTy, EncodedTypeInfo, AtLoc, RParenLoc);
 }
 
-Sema::ExprResult Sema::ParseObjCEncodeExpression(SourceLocation AtLoc,
-                                                 SourceLocation EncodeLoc,
-                                                 SourceLocation LParenLoc,
-                                                 TypeTy *ty,
-                                                 SourceLocation RParenLoc) {
+ExprResult Sema::ParseObjCEncodeExpression(SourceLocation AtLoc,
+                                           SourceLocation EncodeLoc,
+                                           SourceLocation LParenLoc,
+                                           ParsedType ty,
+                                           SourceLocation RParenLoc) {
   // FIXME: Preserve type source info ?
   TypeSourceInfo *TInfo;
   QualType EncodedType = GetTypeFromParser(ty, &TInfo);
@@ -150,28 +148,33 @@ Sema::ExprResult Sema::ParseObjCEncodeExpression(SourceLocation AtLoc,
   return BuildObjCEncodeExpression(AtLoc, TInfo, RParenLoc);
 }
 
-Sema::ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
-                                                   SourceLocation AtLoc,
-                                                   SourceLocation SelLoc,
-                                                   SourceLocation LParenLoc,
-                                                   SourceLocation RParenLoc) {
+ExprResult Sema::ParseObjCSelectorExpression(Selector Sel,
+                                             SourceLocation AtLoc,
+                                             SourceLocation SelLoc,
+                                             SourceLocation LParenLoc,
+                                             SourceLocation RParenLoc) {
   ObjCMethodDecl *Method = LookupInstanceMethodInGlobalPool(Sel,
-                             SourceRange(LParenLoc, RParenLoc), false);
+                             SourceRange(LParenLoc, RParenLoc), false, false);
   if (!Method)
     Method = LookupFactoryMethodInGlobalPool(Sel,
                                           SourceRange(LParenLoc, RParenLoc));
   if (!Method)
     Diag(SelLoc, diag::warn_undeclared_selector) << Sel;
 
+  llvm::DenseMap<Selector, SourceLocation>::iterator Pos
+    = ReferencedSelectors.find(Sel);
+  if (Pos == ReferencedSelectors.end())
+    ReferencedSelectors.insert(std::make_pair(Sel, SelLoc));
+
   QualType Ty = Context.getObjCSelType();
   return new (Context) ObjCSelectorExpr(Ty, Sel, AtLoc, RParenLoc);
 }
 
-Sema::ExprResult Sema::ParseObjCProtocolExpression(IdentifierInfo *ProtocolId,
-                                                   SourceLocation AtLoc,
-                                                   SourceLocation ProtoLoc,
-                                                   SourceLocation LParenLoc,
-                                                   SourceLocation RParenLoc) {
+ExprResult Sema::ParseObjCProtocolExpression(IdentifierInfo *ProtocolId,
+                                             SourceLocation AtLoc,
+                                             SourceLocation ProtoLoc,
+                                             SourceLocation LParenLoc,
+                                             SourceLocation RParenLoc) {
   ObjCProtocolDecl* PDecl = LookupProtocol(ProtocolId, ProtoLoc);
   if (!PDecl) {
     Diag(ProtoLoc, diag::err_undeclared_protocol) << ProtocolId;
@@ -239,7 +242,7 @@ bool Sema::CheckMessageArgumentTypes(Expr **Args, unsigned NumArgs,
       return true;
 
     InitializedEntity Entity = InitializedEntity::InitializeParameter(Param);
-    OwningExprResult ArgE = PerformCopyInitialization(Entity,
+    ExprResult ArgE = PerformCopyInitialization(Entity,
                                                       SourceLocation(),
                                                       Owned(argExpr->Retain()));
     if (ArgE.isInvalid())
@@ -329,7 +332,7 @@ ObjCMethodDecl *Sema::LookupPrivateInstanceMethod(Selector Sel,
 
 /// HandleExprPropertyRefExpr - Handle foo.bar where foo is a pointer to an
 /// objective C interface.  This is a property reference expression.
-Action::OwningExprResult Sema::
+ExprResult Sema::
 HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
                           Expr *BaseExpr, DeclarationName MemberName,
                           SourceLocation MemberLoc) {
@@ -431,7 +434,7 @@ HandleExprPropertyRefExpr(const ObjCObjectPointerType *OPT,
 
 
 
-Action::OwningExprResult Sema::
+ExprResult Sema::
 ActOnClassPropertyRefExpr(IdentifierInfo &receiverName,
                           IdentifierInfo &propertyName,
                           SourceLocation receiverNameLoc,
@@ -529,8 +532,8 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
                                                SourceLocation NameLoc,
                                                bool IsSuper,
                                                bool HasTrailingDot,
-                                               TypeTy *&ReceiverType) {
-  ReceiverType = 0;
+                                               ParsedType &ReceiverType) {
+  ReceiverType = ParsedType();
 
   // If the identifier is "super" and there is no trailing dot, we're
   // messaging super.
@@ -577,7 +580,7 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
     //  We have a class message, and T is the type we're
     //  messaging. Build source-location information for it.
     TypeSourceInfo *TSInfo = Context.getTrivialTypeSourceInfo(T, NameLoc);
-    ReceiverType = CreateLocInfoType(T, TSInfo).getAsOpaquePtr();
+    ReceiverType = CreateParsedType(T, TSInfo);
     return ObjCClassMessage;
   }
   }
@@ -604,7 +607,7 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
 
         QualType T = Context.getObjCInterfaceType(Class);
         TypeSourceInfo *TSInfo = Context.getTrivialTypeSourceInfo(T, NameLoc);
-        ReceiverType = CreateLocInfoType(T, TSInfo).getAsOpaquePtr();
+        ReceiverType = CreateParsedType(T, TSInfo);
         return ObjCClassMessage;
       }
     } else if (Result.empty() && Corrected.getAsIdentifierInfo() &&
@@ -622,7 +625,7 @@ Sema::ObjCMessageKind Sema::getObjCMessageKind(Scope *S,
   return ObjCInstanceMessage;
 }
 
-Sema::OwningExprResult Sema::ActOnSuperMessage(Scope *S, 
+ExprResult Sema::ActOnSuperMessage(Scope *S, 
                                                SourceLocation SuperLoc,
                                                Selector Sel,
                                                SourceLocation LBracLoc,
@@ -657,7 +660,7 @@ Sema::OwningExprResult Sema::ActOnSuperMessage(Scope *S,
     // message to the superclass instance.
     QualType SuperTy = Context.getObjCInterfaceType(Super);
     SuperTy = Context.getObjCObjectPointerType(SuperTy);
-    return BuildInstanceMessage(ExprArg(*this), SuperTy, SuperLoc,
+    return BuildInstanceMessage(0, SuperTy, SuperLoc,
                                 Sel, /*Method=*/0, LBracLoc, RBracLoc, 
                                 move(Args));
   }
@@ -698,7 +701,7 @@ Sema::OwningExprResult Sema::ActOnSuperMessage(Scope *S,
 /// \param RBrac The location of the closing square bracket ']'.
 ///
 /// \param Args The message arguments.
-Sema::OwningExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
+ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
                                                QualType ReceiverType,
                                                SourceLocation SuperLoc,
                                                Selector Sel,
@@ -757,11 +760,8 @@ Sema::OwningExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
   unsigned NumArgs = ArgsIn.size();
   Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
   if (CheckMessageArgumentTypes(Args, NumArgs, Sel, Method, true,
-                                LBracLoc, RBracLoc, ReturnType)) {
-    for (unsigned I = 0; I != NumArgs; ++I)
-      Args[I]->Destroy(Context);
+                                LBracLoc, RBracLoc, ReturnType))
     return ExprError();
-  }
 
   // Construct the appropriate ObjCMessageExpr.
   Expr *Result;
@@ -780,8 +780,8 @@ Sema::OwningExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
 // ActOnClassMessage - used for both unary and keyword messages.
 // ArgExprs is optional - if it is present, the number of expressions
 // is obtained from Sel.getNumArgs().
-Sema::OwningExprResult Sema::ActOnClassMessage(Scope *S, 
-                                               TypeTy *Receiver,
+ExprResult Sema::ActOnClassMessage(Scope *S, 
+                                               ParsedType Receiver,
                                                Selector Sel,
                                                SourceLocation LBracLoc,
                                                SourceLocation SelectorLoc,
@@ -829,7 +829,7 @@ Sema::OwningExprResult Sema::ActOnClassMessage(Scope *S,
 /// \param RBrac The location of the closing square bracket ']'.
 ///
 /// \param Args The message arguments.
-Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
+ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                                                   QualType ReceiverType,
                                                   SourceLocation SuperLoc,
                                                   Selector Sel,
@@ -839,7 +839,6 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
                                                   MultiExprArg ArgsIn) {
   // If we have a receiver expression, perform appropriate promotions
   // and determine receiver type.
-  Expr *Receiver = ReceiverE.takeAs<Expr>();
   if (Receiver) {
     if (Receiver->isTypeDependent()) {
       // If the receiver is type-dependent, we can't type-check anything
@@ -864,13 +863,16 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
 
   if (!Method) {
     // Handle messages to id.
-    if (ReceiverType->isObjCIdType() || ReceiverType->isBlockPointerType() ||
+    bool receiverIsId = ReceiverType->isObjCIdType();
+    if (receiverIsId || ReceiverType->isBlockPointerType() ||
         (Receiver && Context.isObjCNSObjectType(Receiver->getType()))) {
       Method = LookupInstanceMethodInGlobalPool(Sel, 
-                                              SourceRange(LBracLoc, RBracLoc));
+                                                SourceRange(LBracLoc, RBracLoc),
+                                                receiverIsId);
       if (!Method)
         Method = LookupFactoryMethodInGlobalPool(Sel, 
-                                               SourceRange(LBracLoc, RBracLoc));
+                                                 SourceRange(LBracLoc, RBracLoc),
+                                                 receiverIsId);
     } else if (ReceiverType->isObjCClassType() ||
                ReceiverType->isObjCQualifiedClassType()) {
       // Handle messages to Class.
@@ -892,12 +894,14 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
         // If not messaging 'self', look for any factory method named 'Sel'.
         if (!Receiver || !isSelfExpr(Receiver)) {
           Method = LookupFactoryMethodInGlobalPool(Sel, 
-                                               SourceRange(LBracLoc, RBracLoc));
+                                               SourceRange(LBracLoc, RBracLoc),
+                                                   true);
           if (!Method) {
             // If no class (factory) method was found, check if an _instance_
             // method of the same name exists in the root class only.
             Method = LookupInstanceMethodInGlobalPool(Sel,
-                                               SourceRange(LBracLoc, RBracLoc));
+                                               SourceRange(LBracLoc, RBracLoc),
+                                                      true);
             if (Method)
                 if (const ObjCInterfaceDecl *ID =
                   dyn_cast<ObjCInterfaceDecl>(Method->getDeclContext())) {
@@ -931,7 +935,7 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
         ClassDecl = OCIType->getInterfaceDecl();
         // FIXME: consider using LookupInstanceMethodInGlobalPool, since it will be
         // faster than the following method (which can do *many* linear searches).
-        // The idea is to add class info to InstanceMethodPool.
+        // The idea is to add class info to MethodPool.
         Method = ClassDecl->lookupInstanceMethod(Sel);
 
         if (!Method) {
@@ -952,7 +956,7 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
             // compatibility. FIXME: should we deviate??
             if (OCIType->qual_empty()) {
               Method = LookupInstanceMethodInGlobalPool(Sel,
-                                                 SourceRange(LBracLoc, RBracLoc));
+                                                 SourceRange(LBracLoc, RBracLoc)); 
               if (Method && !OCIType->getInterfaceDecl()->isForwardDecl())
                 Diag(Loc, diag::warn_maynot_respond)
                   << OCIType->getInterfaceDecl()->getIdentifier() << Sel;
@@ -962,19 +966,18 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
         if (Method && DiagnoseUseOfDecl(Method, Loc))
           return ExprError();
       } else if (!Context.getObjCIdType().isNull() &&
-                 (ReceiverType->isPointerType() ||
-                  (ReceiverType->isIntegerType() &&
-                   ReceiverType->isScalarType()))) {
+                 (ReceiverType->isPointerType() || 
+                  ReceiverType->isIntegerType())) {
         // Implicitly convert integers and pointers to 'id' but emit a warning.
         Diag(Loc, diag::warn_bad_receiver_type)
           << ReceiverType 
           << Receiver->getSourceRange();
         if (ReceiverType->isPointerType())
           ImpCastExprToType(Receiver, Context.getObjCIdType(), 
-                            CastExpr::CK_BitCast);
+                            CK_BitCast);
         else
           ImpCastExprToType(Receiver, Context.getObjCIdType(),
-                            CastExpr::CK_IntegralToPointer);
+                            CK_IntegralToPointer);
         ReceiverType = Receiver->getType();
       } 
       else if (getLangOptions().CPlusPlus &&
@@ -983,7 +986,7 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
           Receiver = ICE->getSubExpr();
           ReceiverType = Receiver->getType();
         }
-        return BuildInstanceMessage(Owned(Receiver),
+        return BuildInstanceMessage(Receiver,
                                     ReceiverType,
                                     SuperLoc,
                                     Sel,
@@ -1030,18 +1033,17 @@ Sema::OwningExprResult Sema::BuildInstanceMessage(ExprArg ReceiverE,
 // ActOnInstanceMessage - used for both unary and keyword messages.
 // ArgExprs is optional - if it is present, the number of expressions
 // is obtained from Sel.getNumArgs().
-Sema::OwningExprResult Sema::ActOnInstanceMessage(Scope *S,
-                                                  ExprArg ReceiverE, 
-                                                  Selector Sel,
-                                                  SourceLocation LBracLoc,
-                                                  SourceLocation SelectorLoc,
-                                                  SourceLocation RBracLoc,
-                                                  MultiExprArg Args) {
-  Expr *Receiver = static_cast<Expr *>(ReceiverE.get());
+ExprResult Sema::ActOnInstanceMessage(Scope *S,
+                                      Expr *Receiver, 
+                                      Selector Sel,
+                                      SourceLocation LBracLoc,
+                                      SourceLocation SelectorLoc,
+                                      SourceLocation RBracLoc,
+                                      MultiExprArg Args) {
   if (!Receiver)
     return ExprError();
 
-  return BuildInstanceMessage(move(ReceiverE), Receiver->getType(),
+  return BuildInstanceMessage(Receiver, Receiver->getType(),
                               /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0, 
                               LBracLoc, RBracLoc, move(Args));
 }
diff --git a/lib/Sema/SemaInit.cpp b/lib/Sema/SemaInit.cpp
index 7ad177557ef4..a28fd7fe12bd 100644
--- a/lib/Sema/SemaInit.cpp
+++ b/lib/Sema/SemaInit.cpp
@@ -15,12 +15,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "SemaInit.h"
-#include "Lookup.h"
-#include "Sema.h"
+#include "clang/Sema/Designator.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/SemaInternal.h"
 #include "clang/Lex/Preprocessor.h"
-#include "clang/Parse/Designator.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/TypeLoc.h"
@@ -263,9 +264,8 @@ void InitListChecker::FillInValueInitForField(unsigned Init, FieldDecl *Field,
       return;
     }
     
-    Sema::OwningExprResult MemberInit
-      = InitSeq.Perform(SemaRef, MemberEntity, Kind, 
-                        Sema::MultiExprArg(SemaRef, 0, 0));
+    ExprResult MemberInit
+      = InitSeq.Perform(SemaRef, MemberEntity, Kind, MultiExprArg());
     if (MemberInit.isInvalid()) {
       hadError = true;
       return;
@@ -373,9 +373,8 @@ InitListChecker::FillInValueInitializations(const InitializedEntity &Entity,
         return;
       }
 
-      Sema::OwningExprResult ElementInit
-        = InitSeq.Perform(SemaRef, ElementEntity, Kind, 
-                          Sema::MultiExprArg(SemaRef, 0, 0));
+      ExprResult ElementInit
+        = InitSeq.Perform(SemaRef, ElementEntity, Kind, MultiExprArg());
       if (ElementInit.isInvalid()) {
         hadError = true;
         return;
@@ -678,9 +677,8 @@ void InitListChecker::CheckSubElementType(const InitializedEntity &Entity,
       InitializationSequence Seq(SemaRef, Entity, Kind, &expr, 1);
       
       if (Seq) {
-        Sema::OwningExprResult Result = 
-          Seq.Perform(SemaRef, Entity, Kind,
-                      Sema::MultiExprArg(SemaRef, (void **)&expr, 1));
+        ExprResult Result = 
+          Seq.Perform(SemaRef, Entity, Kind, MultiExprArg(&expr, 1));
         if (Result.isInvalid())
           hadError = true;
         
@@ -740,13 +738,13 @@ void InitListChecker::CheckScalarType(const InitializedEntity &Entity,
                                       unsigned &StructuredIndex) {
   if (Index < IList->getNumInits()) {
     Expr *expr = IList->getInit(Index);
-    if (isa<InitListExpr>(expr)) {
-      SemaRef.Diag(IList->getLocStart(),
-                    diag::err_many_braces_around_scalar_init)
-        << IList->getSourceRange();
-      hadError = true;
-      ++Index;
-      ++StructuredIndex;
+    if (InitListExpr *SubIList = dyn_cast<InitListExpr>(expr)) {
+      SemaRef.Diag(SubIList->getLocStart(),
+                    diag::warn_many_braces_around_scalar_init)
+        << SubIList->getSourceRange();
+
+      CheckScalarType(Entity, SubIList, DeclType, Index, StructuredList,
+                      StructuredIndex);
       return;
     } else if (isa<DesignatedInitExpr>(expr)) {
       SemaRef.Diag(expr->getSourceRange().getBegin(),
@@ -758,7 +756,7 @@ void InitListChecker::CheckScalarType(const InitializedEntity &Entity,
       return;
     }
 
-    Sema::OwningExprResult Result =
+    ExprResult Result =
       SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
                                         SemaRef.Owned(expr));
 
@@ -805,7 +803,7 @@ void InitListChecker::CheckReferenceType(const InitializedEntity &Entity,
       return;
     }
 
-    Sema::OwningExprResult Result =
+    ExprResult Result =
       SemaRef.PerformCopyInitialization(Entity, expr->getLocStart(),
                                         SemaRef.Owned(expr));
 
@@ -1367,7 +1365,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
         if (SemaRef.CorrectTypo(R, /*Scope=*/0, /*SS=*/0, RT->getDecl(), false,
                                 Sema::CTC_NoKeywords) && 
             (ReplacementField = R.getAsSingle<FieldDecl>()) &&
-            ReplacementField->getDeclContext()->getLookupContext()
+            ReplacementField->getDeclContext()->getRedeclContext()
                                                       ->Equals(RT->getDecl())) {
           SemaRef.Diag(D->getFieldLoc(), 
                        diag::err_field_designator_unknown_suggest)
@@ -1813,10 +1811,10 @@ CheckArrayDesignatorExpr(Sema &S, Expr *Index, llvm::APSInt &Value) {
   return false;
 }
 
-Sema::OwningExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
+ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
                                                         SourceLocation Loc,
                                                         bool GNUSyntax,
-                                                        OwningExprResult Init) {
+                                                        ExprResult Init) {
   typedef DesignatedInitExpr::Designator ASTDesignator;
 
   bool Invalid = false;
@@ -2021,18 +2019,21 @@ void InitializationSequence::Step::Destroy() {
   switch (Kind) {
   case SK_ResolveAddressOfOverloadedFunction:
   case SK_CastDerivedToBaseRValue:
+  case SK_CastDerivedToBaseXValue:
   case SK_CastDerivedToBaseLValue:
   case SK_BindReference:
   case SK_BindReferenceToTemporary:
   case SK_ExtraneousCopyToTemporary:
   case SK_UserConversion:
   case SK_QualificationConversionRValue:
+  case SK_QualificationConversionXValue:
   case SK_QualificationConversionLValue:
   case SK_ListInitialization:
   case SK_ConstructorInitialization:
   case SK_ZeroInitialization:
   case SK_CAssignment:
   case SK_StringInit:
+  case SK_ObjCObjectConversion:
     break;
     
   case SK_ConversionSequence:
@@ -2091,9 +2092,14 @@ void InitializationSequence::AddAddressOverloadResolutionStep(
 }
 
 void InitializationSequence::AddDerivedToBaseCastStep(QualType BaseType, 
-                                                      bool IsLValue) {
+                                                      ExprValueKind VK) {
   Step S;
-  S.Kind = IsLValue? SK_CastDerivedToBaseLValue : SK_CastDerivedToBaseRValue;
+  switch (VK) {
+  case VK_RValue: S.Kind = SK_CastDerivedToBaseRValue; break;
+  case VK_XValue: S.Kind = SK_CastDerivedToBaseXValue; break;
+  case VK_LValue: S.Kind = SK_CastDerivedToBaseLValue; break;
+  default: llvm_unreachable("No such category");
+  }
   S.Type = BaseType;
   Steps.push_back(S);
 }
@@ -2125,10 +2131,20 @@ void InitializationSequence::AddUserConversionStep(FunctionDecl *Function,
 }
 
 void InitializationSequence::AddQualificationConversionStep(QualType Ty,
-                                                            bool IsLValue) {
+                                                            ExprValueKind VK) {
   Step S;
-  S.Kind = IsLValue? SK_QualificationConversionLValue 
-                   : SK_QualificationConversionRValue;
+  S.Kind = SK_QualificationConversionRValue; // work around a gcc warning
+  switch (VK) {
+  case VK_RValue:
+    S.Kind = SK_QualificationConversionRValue;
+    break;
+  case VK_XValue:
+    S.Kind = SK_QualificationConversionXValue;
+    break;
+  case VK_LValue:
+    S.Kind = SK_QualificationConversionLValue;
+    break;
+  }
   S.Type = Ty;
   Steps.push_back(S);
 }
@@ -2184,6 +2200,13 @@ void InitializationSequence::AddStringInitStep(QualType T) {
   Steps.push_back(S);
 }
 
+void InitializationSequence::AddObjCObjectConversionStep(QualType T) {
+  Step S;
+  S.Kind = SK_ObjCObjectConversion;
+  S.Type = T;
+  Steps.push_back(S);
+}
+
 void InitializationSequence::SetOverloadFailure(FailureKind Failure, 
                                                 OverloadingResult Result) {
   SequenceKind = FailedSequence;
@@ -2258,10 +2281,13 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
   QualType T2 = cv2T2.getUnqualifiedType();
 
   bool DerivedToBase;
+  bool ObjCConversion;
   assert(!S.CompareReferenceRelationship(Initializer->getLocStart(), 
-                                         T1, T2, DerivedToBase) &&
+                                         T1, T2, DerivedToBase,
+                                         ObjCConversion) &&
          "Must have incompatible references when binding via conversion");
   (void)DerivedToBase;
+  (void)ObjCConversion;
 
   // Build the candidate set directly in the initialization sequence
   // structure, so that it will persist if we fail.
@@ -2278,6 +2304,7 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
     // The type we're converting to is a class type. Enumerate its constructors
     // to see if there is a suitable conversion.
     CXXRecordDecl *T1RecordDecl = cast<CXXRecordDecl>(T1RecordType->getDecl());
+
     DeclContext::lookup_iterator Con, ConEnd;
     for (llvm::tie(Con, ConEnd) = S.LookupConstructors(T1RecordDecl);
          Con != ConEnd; ++Con) {
@@ -2305,6 +2332,8 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
       }
     }    
   }
+  if (T1RecordType && T1RecordType->getDecl()->isInvalidDecl())
+    return OR_No_Viable_Function;
   
   const RecordType *T2RecordType = 0;
   if ((T2RecordType = T2->getAs<RecordType>()) &&
@@ -2352,13 +2381,15 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
       }
     }
   }
+  if (T2RecordType && T2RecordType->getDecl()->isInvalidDecl())
+    return OR_No_Viable_Function;
   
   SourceLocation DeclLoc = Initializer->getLocStart();
 
   // Perform overload resolution. If it fails, return the failed result.  
   OverloadCandidateSet::iterator Best;
   if (OverloadingResult Result 
-        = S.BestViableFunction(CandidateSet, DeclLoc, Best))
+        = CandidateSet.BestViableFunction(S, DeclLoc, Best))
     return Result;
 
   FunctionDecl *Function = Best->Function;
@@ -2375,11 +2406,18 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
 
   // Determine whether we need to perform derived-to-base or 
   // cv-qualification adjustments.
+  ExprValueKind VK = VK_RValue;
+  if (T2->isLValueReferenceType())
+    VK = VK_LValue;
+  else if (const RValueReferenceType *RRef = T2->getAs<RValueReferenceType>())
+    VK = RRef->getPointeeType()->isFunctionType() ? VK_LValue : VK_XValue;
+
   bool NewDerivedToBase = false;
+  bool NewObjCConversion = false;
   Sema::ReferenceCompareResult NewRefRelationship
     = S.CompareReferenceRelationship(DeclLoc, T1, 
                                      T2.getNonLValueExprType(S.Context),
-                                     NewDerivedToBase);
+                                     NewDerivedToBase, NewObjCConversion);
   if (NewRefRelationship == Sema::Ref_Incompatible) {
     // If the type we've converted to is not reference-related to the
     // type we're looking for, then there is another conversion step
@@ -2394,10 +2432,14 @@ static OverloadingResult TryRefInitWithConversionFunction(Sema &S,
     Sequence.AddDerivedToBaseCastStep(
                                 S.Context.getQualifiedType(T1,
                                   T2.getNonReferenceType().getQualifiers()), 
-                                  /*isLValue=*/true);
-  
+                                      VK);
+  else if (NewObjCConversion)
+    Sequence.AddObjCObjectConversionStep(
+                                S.Context.getQualifiedType(T1,
+                                  T2.getNonReferenceType().getQualifiers()));
+
   if (cv1T1.getQualifiers() != T2.getNonReferenceType().getQualifiers())
-    Sequence.AddQualificationConversionStep(cv1T1, T2->isReferenceType());
+    Sequence.AddQualificationConversionStep(cv1T1, VK);
   
   Sequence.AddReferenceBindingStep(cv1T1, !T2->isReferenceType());
   return OR_Success;
@@ -2443,9 +2485,11 @@ static void TryReferenceInitialization(Sema &S,
   bool isLValueRef = DestType->isLValueReferenceType();
   bool isRValueRef = !isLValueRef;
   bool DerivedToBase = false;
+  bool ObjCConversion = false;
   Expr::Classification InitCategory = Initializer->Classify(S.Context);
   Sema::ReferenceCompareResult RefRelationship
-    = S.CompareReferenceRelationship(DeclLoc, cv1T1, cv2T2, DerivedToBase);
+    = S.CompareReferenceRelationship(DeclLoc, cv1T1, cv2T2, DerivedToBase,
+                                     ObjCConversion);
 
   // C++0x [dcl.init.ref]p5:
   //   A reference to type "cv1 T1" is initialized by an expression of type 
@@ -2472,9 +2516,13 @@ static void TryReferenceInitialization(Sema &S,
       if (DerivedToBase)
         Sequence.AddDerivedToBaseCastStep(
                          S.Context.getQualifiedType(T1, T2Quals), 
-                         /*isLValue=*/true);
+                         VK_LValue);
+      else if (ObjCConversion)
+        Sequence.AddObjCObjectConversionStep(
+                                     S.Context.getQualifiedType(T1, T2Quals));
+
       if (T1Quals != T2Quals)
-        Sequence.AddQualificationConversionStep(cv1T1, /*IsLValue=*/true);
+        Sequence.AddQualificationConversionStep(cv1T1, VK_LValue);
       bool BindingTemporary = T1Quals.hasConst() && !T1Quals.hasVolatile() &&
         (Initializer->getBitField() || Initializer->refersToVectorElement());
       Sequence.AddReferenceBindingStep(cv1T1, BindingTemporary);
@@ -2531,6 +2579,7 @@ static void TryReferenceInitialization(Sema &S,
 
   //       - [If T1 is not a function type], if T2 is a class type and
   if (!T1Function && T2->isRecordType()) {
+    bool isXValue = InitCategory.isXValue();
     //       - the initializer expression is an rvalue and "cv1 T1" is 
     //         reference-compatible with "cv2 T2", or
     if (InitCategory.isRValue() && 
@@ -2550,10 +2599,15 @@ static void TryReferenceInitialization(Sema &S,
       if (DerivedToBase)
         Sequence.AddDerivedToBaseCastStep(
                          S.Context.getQualifiedType(T1, T2Quals), 
-                         /*isLValue=*/false);
+                         isXValue ? VK_XValue : VK_RValue);
+      else if (ObjCConversion)
+        Sequence.AddObjCObjectConversionStep(
+                                     S.Context.getQualifiedType(T1, T2Quals));
+
       if (T1Quals != T2Quals)
-        Sequence.AddQualificationConversionStep(cv1T1, /*IsLValue=*/false);
-      Sequence.AddReferenceBindingStep(cv1T1, /*bindingTemporary=*/true);
+        Sequence.AddQualificationConversionStep(cv1T1,
+                                            isXValue ? VK_XValue : VK_RValue);
+      Sequence.AddReferenceBindingStep(cv1T1, /*bindingTemporary=*/!isXValue);
       return;
     }
 
@@ -2717,7 +2771,7 @@ static void TryConstructorInitialization(Sema &S,
   // Perform overload resolution. If it fails, return the failed result.  
   OverloadCandidateSet::iterator Best;
   if (OverloadingResult Result 
-        = S.BestViableFunction(CandidateSet, DeclLoc, Best)) {
+        = CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
     Sequence.SetOverloadFailure(
                           InitializationSequence::FK_ConstructorOverloadFailed, 
                                 Result);
@@ -2803,8 +2857,8 @@ static void TryDefaultInitialization(Sema &S,
   //       constructor for T is called (and the initialization is ill-formed if
   //       T has no accessible default constructor);
   if (DestType->isRecordType() && S.getLangOptions().CPlusPlus) {
-    return TryConstructorInitialization(S, Entity, Kind, 0, 0, DestType,
-                                        Sequence);
+    TryConstructorInitialization(S, Entity, Kind, 0, 0, DestType, Sequence);
+    return;
   }
   
   //     - otherwise, no initialization is performed.
@@ -2927,7 +2981,7 @@ static void TryUserDefinedConversion(Sema &S,
   // Perform overload resolution. If it fails, return the failed result.  
   OverloadCandidateSet::iterator Best;
   if (OverloadingResult Result
-        = S.BestViableFunction(CandidateSet, DeclLoc, Best)) {
+        = CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
     Sequence.SetOverloadFailure(
                         InitializationSequence::FK_UserConversionOverloadFailed, 
                                 Result);
@@ -2969,24 +3023,6 @@ static void TryUserDefinedConversion(Sema &S,
   }
 }
 
-bool Sema::TryImplicitConversion(InitializationSequence &Sequence,
-                                 const InitializedEntity &Entity,
-                                 Expr *Initializer,
-                                 bool SuppressUserConversions,
-                                 bool AllowExplicitConversions,
-                                 bool InOverloadResolution) {
-  ImplicitConversionSequence ICS
-    = TryImplicitConversion(Initializer, Entity.getType(),
-                            SuppressUserConversions,
-                            AllowExplicitConversions, 
-                            InOverloadResolution);
-  if (ICS.isBad()) return true;
-
-  // Perform the actual conversion.
-  Sequence.AddConversionSequenceStep(ICS, Entity.getType());
-  return false;
-}
-
 InitializationSequence::InitializationSequence(Sema &S,
                                                const InitializedEntity &Entity,
                                                const InitializationKind &Kind,
@@ -3239,10 +3275,10 @@ static bool shouldDestroyTemporary(const InitializedEntity &Entity) {
 /// \returns An expression that copies the initializer expression into
 /// a temporary object, or an error expression if a copy could not be
 /// created.
-static Sema::OwningExprResult CopyObject(Sema &S,
+static ExprResult CopyObject(Sema &S,
                                          QualType T,
                                          const InitializedEntity &Entity,
-                                         Sema::OwningExprResult CurInit,
+                                         ExprResult CurInit,
                                          bool IsExtraneousCopy) {
   // Determine which class type we're copying to.
   Expr *CurInitExpr = (Expr *)CurInit.get();
@@ -3318,7 +3354,7 @@ static Sema::OwningExprResult CopyObject(Sema &S,
   }
   
   OverloadCandidateSet::iterator Best;
-  switch (S.BestViableFunction(CandidateSet, Loc, Best)) {
+  switch (CandidateSet.BestViableFunction(S, Loc, Best)) {
   case OR_Success:
     break;
       
@@ -3328,19 +3364,17 @@ static Sema::OwningExprResult CopyObject(Sema &S,
            : diag::err_temp_copy_no_viable)
       << (int)Entity.getKind() << CurInitExpr->getType()
       << CurInitExpr->getSourceRange();
-    S.PrintOverloadCandidates(CandidateSet, Sema::OCD_AllCandidates,
-                              &CurInitExpr, 1);
+    CandidateSet.NoteCandidates(S, OCD_AllCandidates, &CurInitExpr, 1);
     if (!IsExtraneousCopy || S.isSFINAEContext())
-      return S.ExprError();
+      return ExprError();
     return move(CurInit);
       
   case OR_Ambiguous:
     S.Diag(Loc, diag::err_temp_copy_ambiguous)
       << (int)Entity.getKind() << CurInitExpr->getType()
       << CurInitExpr->getSourceRange();
-    S.PrintOverloadCandidates(CandidateSet, Sema::OCD_ViableCandidates,
-                              &CurInitExpr, 1);
-    return S.ExprError();
+    CandidateSet.NoteCandidates(S, OCD_ViableCandidates, &CurInitExpr, 1);
+    return ExprError();
     
   case OR_Deleted:
     S.Diag(Loc, diag::err_temp_copy_deleted)
@@ -3348,11 +3382,11 @@ static Sema::OwningExprResult CopyObject(Sema &S,
       << CurInitExpr->getSourceRange();
     S.Diag(Best->Function->getLocation(), diag::note_unavailable_here)
       << Best->Function->isDeleted();
-    return S.ExprError();
+    return ExprError();
   }
 
   CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(Best->Function);
-  ASTOwningVector<&ActionBase::DeleteExpr> ConstructorArgs(S);
+  ASTOwningVector<Expr*> ConstructorArgs(S);
   CurInit.release(); // Ownership transferred into MultiExprArg, below.
 
   S.CheckConstructorAccess(Loc, Constructor, Entity,
@@ -3387,16 +3421,15 @@ static Sema::OwningExprResult CopyObject(Sema &S,
   // Determine the arguments required to actually perform the
   // constructor call (we might have derived-to-base conversions, or
   // the copy constructor may have default arguments).
-  if (S.CompleteConstructorCall(Constructor,
-                                Sema::MultiExprArg(S, 
-                                                   (void **)&CurInitExpr,
-                                                   1),
+  if (S.CompleteConstructorCall(Constructor, MultiExprArg(&CurInitExpr, 1),
                                 Loc, ConstructorArgs))
-    return S.ExprError();
+    return ExprError();
 
   // Actually perform the constructor call.
   CurInit = S.BuildCXXConstructExpr(Loc, T, Constructor, Elidable,
-                                    move_arg(ConstructorArgs));
+                                    move_arg(ConstructorArgs),
+                                    /*ZeroInit*/ false,
+                                    CXXConstructExpr::CK_Complete);
   
   // If we're supposed to bind temporaries, do so.
   if (!CurInit.isInvalid() && shouldBindAsTemporary(Entity))
@@ -3418,16 +3451,16 @@ void InitializationSequence::PrintInitLocationNote(Sema &S,
   }
 }
 
-Action::OwningExprResult 
+ExprResult 
 InitializationSequence::Perform(Sema &S,
                                 const InitializedEntity &Entity,
                                 const InitializationKind &Kind,
-                                Action::MultiExprArg Args,
+                                MultiExprArg Args,
                                 QualType *ResultType) {
   if (SequenceKind == FailedSequence) {
     unsigned NumArgs = Args.size();
     Diagnose(S, Entity, Kind, (Expr **)Args.release(), NumArgs);
-    return S.ExprError();
+    return ExprError();
   }
   
   if (SequenceKind == DependentSequence) {
@@ -3471,7 +3504,7 @@ InitializationSequence::Perform(Sema &S,
     }
 
     if (Kind.getKind() == InitializationKind::IK_Copy || Kind.isExplicitCast())
-      return Sema::OwningExprResult(S, Args.release()[0]);
+      return ExprResult(Args.release()[0]);
 
     if (Args.size() == 0)
       return S.Owned((Expr *)0);
@@ -3495,7 +3528,7 @@ InitializationSequence::Perform(Sema &S,
     *ResultType = Entity.getDecl() ? Entity.getDecl()->getType() :
                                      Entity.getType();
 
-  Sema::OwningExprResult CurInit = S.Owned((Expr *)0);
+  ExprResult CurInit = S.Owned((Expr *)0);
   
   assert(!Steps.empty() && "Cannot have an empty initialization sequence");
   
@@ -3505,21 +3538,24 @@ InitializationSequence::Perform(Sema &S,
   switch (Steps.front().Kind) {
   case SK_ResolveAddressOfOverloadedFunction:
   case SK_CastDerivedToBaseRValue:
+  case SK_CastDerivedToBaseXValue:
   case SK_CastDerivedToBaseLValue:
   case SK_BindReference:
   case SK_BindReferenceToTemporary:
   case SK_ExtraneousCopyToTemporary:
   case SK_UserConversion:
   case SK_QualificationConversionLValue:
+  case SK_QualificationConversionXValue:
   case SK_QualificationConversionRValue:
   case SK_ConversionSequence:
   case SK_ListInitialization:
   case SK_CAssignment:
   case SK_StringInit:
+  case SK_ObjCObjectConversion:
     assert(Args.size() == 1);
-    CurInit = Sema::OwningExprResult(S, ((Expr **)(Args.get()))[0]->Retain());
+    CurInit = ExprResult(((Expr **)(Args.get()))[0]->Retain());
     if (CurInit.isInvalid())
-      return S.ExprError();
+      return ExprError();
     break;
     
   case SK_ConstructorInitialization:
@@ -3533,7 +3569,7 @@ InitializationSequence::Perform(Sema &S,
   for (step_iterator Step = step_begin(), StepEnd = step_end();
        Step != StepEnd; ++Step) {
     if (CurInit.isInvalid())
-      return S.ExprError();
+      return ExprError();
     
     Expr *CurInitExpr = (Expr *)CurInit.get();
     QualType SourceType = CurInitExpr? CurInitExpr->getType() : QualType();
@@ -3550,11 +3586,12 @@ InitializationSequence::Perform(Sema &S,
       break;
         
     case SK_CastDerivedToBaseRValue:
+    case SK_CastDerivedToBaseXValue:
     case SK_CastDerivedToBaseLValue: {
       // We have a derived-to-base cast that produces either an rvalue or an
       // lvalue. Perform that cast.
       
-      CXXBaseSpecifierArray BasePath;
+      CXXCastPath BasePath;
 
       // Casts to inaccessible base classes are allowed with C-style casts.
       bool IgnoreBaseAccess = Kind.isCStyleOrFunctionalCast();
@@ -3562,7 +3599,7 @@ InitializationSequence::Perform(Sema &S,
                                          CurInitExpr->getLocStart(),
                                          CurInitExpr->getSourceRange(), 
                                          &BasePath, IgnoreBaseAccess))
-        return S.ExprError();
+        return ExprError();
         
       if (S.BasePathInvolvesVirtualBase(BasePath)) {
         QualType T = SourceType;
@@ -3573,11 +3610,17 @@ InitializationSequence::Perform(Sema &S,
                            cast<CXXRecordDecl>(RecordTy->getDecl()));
       }
 
-      CurInit = S.Owned(new (S.Context) ImplicitCastExpr(Step->Type,
-                                                    CastExpr::CK_DerivedToBase,
-                                                    (Expr*)CurInit.release(),
-                                                    BasePath,
-                                     Step->Kind == SK_CastDerivedToBaseLValue));
+      ExprValueKind VK =
+          Step->Kind == SK_CastDerivedToBaseLValue ?
+              VK_LValue :
+              (Step->Kind == SK_CastDerivedToBaseXValue ?
+                   VK_XValue :
+                   VK_RValue);
+      CurInit = S.Owned(ImplicitCastExpr::Create(S.Context,
+                                                 Step->Type,
+                                                 CK_DerivedToBase,
+                                                 CurInit.get(),
+                                                 &BasePath, VK));
       break;
     }
         
@@ -3589,7 +3632,7 @@ InitializationSequence::Perform(Sema &S,
           << BitField->getDeclName()
           << CurInitExpr->getSourceRange();
         S.Diag(BitField->getLocation(), diag::note_bitfield_decl);
-        return S.ExprError();
+        return ExprError();
       }
 
       if (CurInitExpr->refersToVectorElement()) {
@@ -3598,14 +3641,14 @@ InitializationSequence::Perform(Sema &S,
           << Entity.getType().isVolatileQualified()
           << CurInitExpr->getSourceRange();
         PrintInitLocationNote(S, Entity);
-        return S.ExprError();
+        return ExprError();
       }
         
       // Reference binding does not have any corresponding ASTs.
 
       // Check exception specifications
       if (S.CheckExceptionSpecCompatibility(CurInitExpr, DestType))
-        return S.ExprError();
+        return ExprError();
 
       break;
 
@@ -3614,7 +3657,7 @@ InitializationSequence::Perform(Sema &S,
 
       // Check exception specifications
       if (S.CheckExceptionSpecCompatibility(CurInitExpr, DestType))
-        return S.ExprError();
+        return ExprError();
 
       break;
         
@@ -3626,7 +3669,7 @@ InitializationSequence::Perform(Sema &S,
     case SK_UserConversion: {
       // We have a user-defined conversion that invokes either a constructor
       // or a conversion function.
-      CastExpr::CastKind CastKind = CastExpr::CK_Unknown;
+      CastKind CastKind = CK_Unknown;
       bool IsCopy = false;
       FunctionDecl *Fn = Step->Function.Function;
       DeclAccessPair FoundFn = Step->Function.FoundDecl;
@@ -3634,30 +3677,30 @@ InitializationSequence::Perform(Sema &S,
       bool IsLvalue = false;
       if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(Fn)) {
         // Build a call to the selected constructor.
-        ASTOwningVector<&ActionBase::DeleteExpr> ConstructorArgs(S);
+        ASTOwningVector<Expr*> ConstructorArgs(S);
         SourceLocation Loc = CurInitExpr->getLocStart();
         CurInit.release(); // Ownership transferred into MultiExprArg, below.
 
         // Determine the arguments required to actually perform the constructor
         // call.
         if (S.CompleteConstructorCall(Constructor,
-                                      Sema::MultiExprArg(S, 
-                                                         (void **)&CurInitExpr,
-                                                         1),
+                                      MultiExprArg(&CurInitExpr, 1),
                                       Loc, ConstructorArgs))
-          return S.ExprError();
+          return ExprError();
         
         // Build the an expression that constructs a temporary.
         CurInit = S.BuildCXXConstructExpr(Loc, Step->Type, Constructor, 
-                                          move_arg(ConstructorArgs));
+                                          move_arg(ConstructorArgs),
+                                          /*ZeroInit*/ false,
+                                          CXXConstructExpr::CK_Complete);
         if (CurInit.isInvalid())
-          return S.ExprError();
+          return ExprError();
 
         S.CheckConstructorAccess(Kind.getLocation(), Constructor, Entity,
                                  FoundFn.getAccess());
         S.DiagnoseUseOfDecl(FoundFn, Kind.getLocation());
         
-        CastKind = CastExpr::CK_ConstructorConversion;
+        CastKind = CK_ConstructorConversion;
         QualType Class = S.Context.getTypeDeclType(Constructor->getParent());
         if (S.Context.hasSameUnqualifiedType(SourceType, Class) ||
             S.IsDerivedFrom(SourceType, Class))
@@ -3677,7 +3720,7 @@ InitializationSequence::Perform(Sema &S,
         // we don't want to turn off access control here for c-style casts.
         if (S.PerformObjectArgumentInitialization(CurInitExpr, /*Qualifier=*/0,
                                                   FoundFn, Conversion))
-          return S.ExprError();
+          return ExprError();
 
         // Do a little dance to make sure that CurInit has the proper
         // pointer.
@@ -3687,9 +3730,9 @@ InitializationSequence::Perform(Sema &S,
         CurInit = S.Owned(S.BuildCXXMemberCallExpr(CurInitExpr, FoundFn,
                                                    Conversion));
         if (CurInit.isInvalid() || !CurInit.get())
-          return S.ExprError();
+          return ExprError();
         
-        CastKind = CastExpr::CK_UserDefinedConversion;
+        CastKind = CK_UserDefinedConversion;
         
         CreatedObject = Conversion->getResultType()->isRecordType();
       }
@@ -3711,35 +3754,41 @@ InitializationSequence::Perform(Sema &S,
       }
       
       CurInitExpr = CurInit.takeAs<Expr>();
-      CurInit = S.Owned(new (S.Context) ImplicitCastExpr(CurInitExpr->getType(),
-                                                         CastKind, 
-                                                         CurInitExpr,
-                                                        CXXBaseSpecifierArray(),
-                                                         IsLvalue));
+      // FIXME: xvalues
+      CurInit = S.Owned(ImplicitCastExpr::Create(S.Context,
+                                                 CurInitExpr->getType(),
+                                                 CastKind, CurInitExpr, 0,
+                                           IsLvalue ? VK_LValue : VK_RValue));
       
       if (RequiresCopy)
         CurInit = CopyObject(S, Entity.getType().getNonReferenceType(), Entity,
                              move(CurInit), /*IsExtraneousCopy=*/false);
-      
+
       break;
     }
-        
+
     case SK_QualificationConversionLValue:
-    case SK_QualificationConversionRValue:
+    case SK_QualificationConversionXValue:
+    case SK_QualificationConversionRValue: {
       // Perform a qualification conversion; these can never go wrong.
-      S.ImpCastExprToType(CurInitExpr, Step->Type,
-                          CastExpr::CK_NoOp,
-                          Step->Kind == SK_QualificationConversionLValue);
+      ExprValueKind VK =
+          Step->Kind == SK_QualificationConversionLValue ?
+              VK_LValue :
+              (Step->Kind == SK_QualificationConversionXValue ?
+                   VK_XValue :
+                   VK_RValue);
+      S.ImpCastExprToType(CurInitExpr, Step->Type, CK_NoOp, VK);
       CurInit.release();
       CurInit = S.Owned(CurInitExpr);
       break;
-        
+    }
+
     case SK_ConversionSequence: {
       bool IgnoreBaseAccess = Kind.isCStyleOrFunctionalCast();
 
       if (S.PerformImplicitConversion(CurInitExpr, Step->Type, *Step->ICS,
                                       Sema::AA_Converting, IgnoreBaseAccess))
-        return S.ExprError();
+        return ExprError();
         
       CurInit.release();
       CurInit = S.Owned(CurInitExpr);
@@ -3750,7 +3799,7 @@ InitializationSequence::Perform(Sema &S,
       InitListExpr *InitList = cast<InitListExpr>(CurInitExpr);
       QualType Ty = Step->Type;
       if (S.CheckInitList(Entity, InitList, ResultType? *ResultType : Ty))
-        return S.ExprError();
+        return ExprError();
 
       CurInit.release();
       CurInit = S.Owned(InitList);
@@ -3763,16 +3812,29 @@ InitializationSequence::Perform(Sema &S,
         = cast<CXXConstructorDecl>(Step->Function.Function);
       
       // Build a call to the selected constructor.
-      ASTOwningVector<&ActionBase::DeleteExpr> ConstructorArgs(S);
-      SourceLocation Loc = Kind.getLocation();
-          
+      ASTOwningVector<Expr*> ConstructorArgs(S);
+      SourceLocation Loc = (Kind.isCopyInit() && Kind.getEqualLoc().isValid())
+                             ? Kind.getEqualLoc()
+                             : Kind.getLocation();
+
+      if (Kind.getKind() == InitializationKind::IK_Default) {
+        // Force even a trivial, implicit default constructor to be
+        // semantically checked. We do this explicitly because we don't build
+        // the definition for completely trivial constructors.
+        CXXRecordDecl *ClassDecl = Constructor->getParent();
+        assert(ClassDecl && "No parent class for constructor.");
+        if (Constructor->isImplicit() && Constructor->isDefaultConstructor() &&
+            ClassDecl->hasTrivialConstructor() && !Constructor->isUsed(false))
+          S.DefineImplicitDefaultConstructor(Loc, Constructor);
+      }
+
       // Determine the arguments required to actually perform the constructor
       // call.
       if (S.CompleteConstructorCall(Constructor, move(Args), 
                                     Loc, ConstructorArgs))
-        return S.ExprError();
+        return ExprError();
           
-      // Build the expression that constructs a temporary.
+      
       if (Entity.getKind() == InitializedEntity::EK_Temporary &&
           NumArgs != 1 && // FIXME: Hack to work around cast weirdness
           (Kind.getKind() == InitializationKind::IK_Direct ||
@@ -3780,11 +3842,11 @@ InitializationSequence::Perform(Sema &S,
         // An explicitly-constructed temporary, e.g., X(1, 2).
         unsigned NumExprs = ConstructorArgs.size();
         Expr **Exprs = (Expr **)ConstructorArgs.take();
-        S.MarkDeclarationReferenced(Kind.getLocation(), Constructor);
+        S.MarkDeclarationReferenced(Loc, Constructor);
         CurInit = S.Owned(new (S.Context) CXXTemporaryObjectExpr(S.Context,
                                                                  Constructor,
                                                               Entity.getType(),
-                                                            Kind.getLocation(),
+                                                                 Loc,
                                                                  Exprs, 
                                                                  NumExprs,
                                                 Kind.getParenRange().getEnd(),
@@ -3815,7 +3877,7 @@ InitializationSequence::Perform(Sema &S,
                                             ConstructKind);
       }
       if (CurInit.isInvalid())
-        return S.ExprError();
+        return ExprError();
 
       // Only check access if all of that succeeded.
       S.CheckConstructorAccess(Loc, Constructor, Entity,
@@ -3867,7 +3929,7 @@ InitializationSequence::Perform(Sema &S,
                                      getAssignmentAction(Entity),
                                      &Complained)) {
         PrintInitLocationNote(S, Entity);
-        return S.ExprError();
+        return ExprError();
       } else if (Complained)
         PrintInitLocationNote(S, Entity);
 
@@ -3881,6 +3943,14 @@ InitializationSequence::Perform(Sema &S,
       CheckStringInit(CurInitExpr, ResultType ? *ResultType : Ty, S);
       break;
     }
+
+    case SK_ObjCObjectConversion:
+      S.ImpCastExprToType(CurInitExpr, Step->Type, 
+                          CK_ObjCObjectLValueCast,
+                          S.CastCategory(CurInitExpr));
+      CurInit.release();
+      CurInit = S.Owned(CurInitExpr);
+      break;
     }
   }
   
@@ -3937,16 +4007,14 @@ bool InitializationSequence::Diagnose(Sema &S,
           << DestType << Args[0]->getType()
           << Args[0]->getSourceRange();
 
-      S.PrintOverloadCandidates(FailedCandidateSet, Sema::OCD_ViableCandidates,
-                                Args, NumArgs);
+      FailedCandidateSet.NoteCandidates(S, OCD_ViableCandidates, Args, NumArgs);
       break;
         
     case OR_No_Viable_Function:
       S.Diag(Kind.getLocation(), diag::err_typecheck_nonviable_condition)
         << Args[0]->getType() << DestType.getNonReferenceType()
         << Args[0]->getSourceRange();
-      S.PrintOverloadCandidates(FailedCandidateSet, Sema::OCD_AllCandidates,
-                                Args, NumArgs);
+      FailedCandidateSet.NoteCandidates(S, OCD_AllCandidates, Args, NumArgs);
       break;
         
     case OR_Deleted: {
@@ -3954,9 +4022,8 @@ bool InitializationSequence::Diagnose(Sema &S,
         << Args[0]->getType() << DestType.getNonReferenceType()
         << Args[0]->getSourceRange();
       OverloadCandidateSet::iterator Best;
-      OverloadingResult Ovl = S.BestViableFunction(FailedCandidateSet,
-                                                   Kind.getLocation(),
-                                                   Best);
+      OverloadingResult Ovl
+        = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
       if (Ovl == OR_Deleted) {
         S.Diag(Best->Function->getLocation(), diag::note_unavailable_here)
           << Best->Function->isDeleted();
@@ -4049,8 +4116,8 @@ bool InitializationSequence::Diagnose(Sema &S,
       case OR_Ambiguous:
         S.Diag(Kind.getLocation(), diag::err_ovl_ambiguous_init)
           << DestType << ArgsRange;
-        S.PrintOverloadCandidates(FailedCandidateSet,
-                                  Sema::OCD_ViableCandidates, Args, NumArgs);
+        FailedCandidateSet.NoteCandidates(S, OCD_ViableCandidates,
+                                          Args, NumArgs);
         break;
         
       case OR_No_Viable_Function:
@@ -4095,17 +4162,15 @@ bool InitializationSequence::Diagnose(Sema &S,
 
         S.Diag(Kind.getLocation(), diag::err_ovl_no_viable_function_in_init)
           << DestType << ArgsRange;
-        S.PrintOverloadCandidates(FailedCandidateSet, Sema::OCD_AllCandidates,
-                                  Args, NumArgs);
+        FailedCandidateSet.NoteCandidates(S, OCD_AllCandidates, Args, NumArgs);
         break;
         
       case OR_Deleted: {
         S.Diag(Kind.getLocation(), diag::err_ovl_deleted_init)
           << true << DestType << ArgsRange;
         OverloadCandidateSet::iterator Best;
-        OverloadingResult Ovl = S.BestViableFunction(FailedCandidateSet,
-                                                     Kind.getLocation(),
-                                                     Best);
+        OverloadingResult Ovl
+          = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
         if (Ovl == OR_Deleted) {
           S.Diag(Best->Function->getLocation(), diag::note_unavailable_here)
             << Best->Function->isDeleted();
@@ -4288,6 +4353,10 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
       OS << "derived-to-base case (rvalue" << S->Type.getAsString() << ")";
       break;
       
+    case SK_CastDerivedToBaseXValue:
+      OS << "derived-to-base case (xvalue" << S->Type.getAsString() << ")";
+      break;
+      
     case SK_CastDerivedToBaseLValue:
       OS << "derived-to-base case (lvalue" << S->Type.getAsString() << ")";
       break;
@@ -4307,10 +4376,13 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
     case SK_UserConversion:
       OS << "user-defined conversion via " << S->Function.Function;
       break;
-      
+
     case SK_QualificationConversionRValue:
       OS << "qualification conversion (rvalue)";
 
+    case SK_QualificationConversionXValue:
+      OS << "qualification conversion (xvalue)";
+
     case SK_QualificationConversionLValue:
       OS << "qualification conversion (lvalue)";
       break;
@@ -4340,6 +4412,10 @@ void InitializationSequence::dump(llvm::raw_ostream &OS) const {
     case SK_StringInit:
       OS << "string initialization";
       break;
+
+    case SK_ObjCObjectConversion:
+      OS << "Objective-C object conversion";
+      break;
     }
   }
 }
@@ -4351,10 +4427,10 @@ void InitializationSequence::dump() const {
 //===----------------------------------------------------------------------===//
 // Initialization helper functions
 //===----------------------------------------------------------------------===//
-Sema::OwningExprResult 
+ExprResult 
 Sema::PerformCopyInitialization(const InitializedEntity &Entity,
                                 SourceLocation EqualLoc,
-                                OwningExprResult Init) {
+                                ExprResult Init) {
   if (Init.isInvalid())
     return ExprError();
 
@@ -4368,6 +4444,5 @@ Sema::PerformCopyInitialization(const InitializedEntity &Entity,
                                                            EqualLoc);
   InitializationSequence Seq(*this, Entity, Kind, &InitE, 1);
   Init.release();
-  return Seq.Perform(*this, Entity, Kind, 
-                     MultiExprArg(*this, (void**)&InitE, 1));
+  return Seq.Perform(*this, Entity, Kind, MultiExprArg(&InitE, 1));
 }
diff --git a/lib/Sema/SemaInit.h b/lib/Sema/SemaInit.h
deleted file mode 100644
index 44c36a735bc8..000000000000
--- a/lib/Sema/SemaInit.h
+++ /dev/null
@@ -1,765 +0,0 @@
-//===--- SemaInit.h - Semantic Analysis for Initializers --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file provides supporting data types for initialization of objects.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_CLANG_SEMA_INIT_H
-#define LLVM_CLANG_SEMA_INIT_H
-
-#include "SemaOverload.h"
-#include "clang/AST/Type.h"
-#include "clang/AST/UnresolvedSet.h"
-#include "clang/Parse/Action.h"
-#include "clang/Basic/SourceLocation.h"
-#include "llvm/ADT/PointerIntPair.h"
-#include "llvm/ADT/SmallVector.h"
-#include <cassert>
-
-namespace llvm {
-  class raw_ostream;
-}
-
-namespace clang {
-  
-class CXXBaseSpecifier;
-class DeclaratorDecl;
-class DeclaratorInfo;
-class FieldDecl;
-class FunctionDecl;
-class ParmVarDecl;
-class Sema;
-class TypeLoc;
-class VarDecl;
-  
-/// \brief Describes an entity that is being initialized.
-class InitializedEntity {
-public:
-  /// \brief Specifies the kind of entity being initialized.
-  enum EntityKind {
-    /// \brief The entity being initialized is a variable.
-    EK_Variable,
-    /// \brief The entity being initialized is a function parameter.
-    EK_Parameter,
-    /// \brief The entity being initialized is the result of a function call.
-    EK_Result,
-    /// \brief The entity being initialized is an exception object that
-    /// is being thrown.
-    EK_Exception,
-    /// \brief The entity being initialized is a non-static data member 
-    /// subobject.
-    EK_Member,
-    /// \brief The entity being initialized is an element of an array.
-    EK_ArrayElement,
-    /// \brief The entity being initialized is an object (or array of
-    /// objects) allocated via new.
-    EK_New,
-    /// \brief The entity being initialized is a temporary object.
-    EK_Temporary,
-    /// \brief The entity being initialized is a base member subobject.
-    EK_Base,
-    /// \brief The entity being initialized is an element of a vector.
-    /// or vector.
-    EK_VectorElement,
-    /// \brief The entity being initialized is a field of block descriptor for
-    /// the copied-in c++ object.
-    EK_BlockElement
-  };
-  
-private:
-  /// \brief The kind of entity being initialized.
-  EntityKind Kind;
-
-  /// \brief If non-NULL, the parent entity in which this
-  /// initialization occurs.
-  const InitializedEntity *Parent;
-
-  /// \brief The type of the object or reference being initialized.
-  QualType Type;
-  
-  union {
-    /// \brief When Kind == EK_Variable, EK_Parameter, or EK_Member, 
-    /// the VarDecl, ParmVarDecl, or FieldDecl, respectively.
-    DeclaratorDecl *VariableOrMember;
-    
-    struct {
-      /// \brief When Kind == EK_Result, EK_Exception, or EK_New, the
-      /// location of the 'return', 'throw', or 'new' keyword,
-      /// respectively. When Kind == EK_Temporary, the location where
-      /// the temporary is being created.
-      unsigned Location;
-      
-      /// \brief Whether the 
-      bool NRVO;
-    } LocAndNRVO;
-    
-    /// \brief When Kind == EK_Base, the base specifier that provides the 
-    /// base class. The lower bit specifies whether the base is an inherited
-    /// virtual base.
-    uintptr_t Base;
-
-    /// \brief When Kind == EK_ArrayElement or EK_VectorElement, the
-    /// index of the array or vector element being initialized. 
-    unsigned Index;
-  };
-
-  InitializedEntity() { }
-
-  /// \brief Create the initialization entity for a variable.
-  InitializedEntity(VarDecl *Var)
-    : Kind(EK_Variable), Parent(0), Type(Var->getType()),
-      VariableOrMember(reinterpret_cast<DeclaratorDecl*>(Var)) { }
-  
-  /// \brief Create the initialization entity for a parameter.
-  InitializedEntity(ParmVarDecl *Parm)
-    : Kind(EK_Parameter), Parent(0), Type(Parm->getType().getUnqualifiedType()),
-      VariableOrMember(reinterpret_cast<DeclaratorDecl*>(Parm)) { }
-  
-  /// \brief Create the initialization entity for the result of a
-  /// function, throwing an object, performing an explicit cast, or
-  /// initializing a parameter for which there is no declaration.
-  InitializedEntity(EntityKind Kind, SourceLocation Loc, QualType Type,
-                    bool NRVO = false)
-    : Kind(Kind), Parent(0), Type(Type)
-  {
-    LocAndNRVO.Location = Loc.getRawEncoding();
-    LocAndNRVO.NRVO = NRVO;
-  }
-  
-  /// \brief Create the initialization entity for a member subobject.
-  InitializedEntity(FieldDecl *Member, const InitializedEntity *Parent) 
-    : Kind(EK_Member), Parent(Parent), Type(Member->getType()),
-      VariableOrMember(reinterpret_cast<DeclaratorDecl*>(Member)) { }
-  
-  /// \brief Create the initialization entity for an array element.
-  InitializedEntity(ASTContext &Context, unsigned Index, 
-                    const InitializedEntity &Parent);
-
-public:
-  /// \brief Create the initialization entity for a variable.
-  static InitializedEntity InitializeVariable(VarDecl *Var) {
-    return InitializedEntity(Var);
-  }
-  
-  /// \brief Create the initialization entity for a parameter.
-  static InitializedEntity InitializeParameter(ParmVarDecl *Parm) {
-    return InitializedEntity(Parm);
-  }
-
-  /// \brief Create the initialization entity for a parameter that is
-  /// only known by its type.
-  static InitializedEntity InitializeParameter(QualType Type) {
-    InitializedEntity Entity;
-    Entity.Kind = EK_Parameter;
-    Entity.Type = Type;
-    Entity.Parent = 0;
-    Entity.VariableOrMember = 0;
-    return Entity;
-  }
-
-  /// \brief Create the initialization entity for the result of a function.
-  static InitializedEntity InitializeResult(SourceLocation ReturnLoc,
-                                            QualType Type, bool NRVO) {
-    return InitializedEntity(EK_Result, ReturnLoc, Type, NRVO);
-  }
-
-  static InitializedEntity InitializeBlock(SourceLocation BlockVarLoc,
-                                           QualType Type, bool NRVO) {
-    return InitializedEntity(EK_BlockElement, BlockVarLoc, Type, NRVO);
-  }
-  
-  /// \brief Create the initialization entity for an exception object.
-  static InitializedEntity InitializeException(SourceLocation ThrowLoc,
-                                               QualType Type, bool NRVO) {
-    return InitializedEntity(EK_Exception, ThrowLoc, Type, NRVO);
-  }
-
-  /// \brief Create the initialization entity for an object allocated via new.
-  static InitializedEntity InitializeNew(SourceLocation NewLoc, QualType Type) {
-    return InitializedEntity(EK_New, NewLoc, Type);
-  }
-  
-  /// \brief Create the initialization entity for a temporary.
-  static InitializedEntity InitializeTemporary(QualType Type) {
-    return InitializedEntity(EK_Temporary, SourceLocation(), Type);
-  }
-  
-  /// \brief Create the initialization entity for a base class subobject.
-  static InitializedEntity InitializeBase(ASTContext &Context,
-                                          CXXBaseSpecifier *Base,
-                                          bool IsInheritedVirtualBase);
-  
-  /// \brief Create the initialization entity for a member subobject.
-  static InitializedEntity InitializeMember(FieldDecl *Member,
-                                          const InitializedEntity *Parent = 0) {
-    return InitializedEntity(Member, Parent);
-  }
-  
-  /// \brief Create the initialization entity for an array element.
-  static InitializedEntity InitializeElement(ASTContext &Context, 
-                                             unsigned Index, 
-                                             const InitializedEntity &Parent) {
-    return InitializedEntity(Context, Index, Parent);
-  }
-
-  /// \brief Determine the kind of initialization.
-  EntityKind getKind() const { return Kind; }
-  
-  /// \brief Retrieve the parent of the entity being initialized, when
-  /// the initialization itself is occuring within the context of a
-  /// larger initialization.
-  const InitializedEntity *getParent() const { return Parent; }
-
-  /// \brief Retrieve type being initialized.
-  QualType getType() const { return Type; }
-  
-  /// \brief Retrieve the name of the entity being initialized.
-  DeclarationName getName() const;
-
-  /// \brief Retrieve the variable, parameter, or field being
-  /// initialized.
-  DeclaratorDecl *getDecl() const;
-
-  /// \brief Determine whether this initialization allows the named return 
-  /// value optimization, which also applies to thrown objects.
-  bool allowsNRVO() const;
-                                  
-  /// \brief Retrieve the base specifier.
-  CXXBaseSpecifier *getBaseSpecifier() const {
-    assert(getKind() == EK_Base && "Not a base specifier");
-    return reinterpret_cast<CXXBaseSpecifier *>(Base & ~0x1);
-  }
-
-  /// \brief Return whether the base is an inherited virtual base.
-  bool isInheritedVirtualBase() const {
-    assert(getKind() == EK_Base && "Not a base specifier");
-    return Base & 0x1;
-  }
-
-  /// \brief Determine the location of the 'return' keyword when initializing
-  /// the result of a function call.
-  SourceLocation getReturnLoc() const {
-    assert(getKind() == EK_Result && "No 'return' location!");
-    return SourceLocation::getFromRawEncoding(LocAndNRVO.Location);
-  }
-  
-  /// \brief Determine the location of the 'throw' keyword when initializing
-  /// an exception object.
-  SourceLocation getThrowLoc() const {
-    assert(getKind() == EK_Exception && "No 'throw' location!");
-    return SourceLocation::getFromRawEncoding(LocAndNRVO.Location);
-  }
-
-  /// \brief If this is already the initializer for an array or vector
-  /// element, sets the element index.
-  void setElementIndex(unsigned Index) {
-    assert(getKind() == EK_ArrayElement || getKind() == EK_VectorElement);
-    this->Index = Index;
-  }
-};
-  
-/// \brief Describes the kind of initialization being performed, along with 
-/// location information for tokens related to the initialization (equal sign,
-/// parentheses).
-class InitializationKind {
-public:
-  /// \brief The kind of initialization being performed.
-  enum InitKind {
-    IK_Direct,  ///< Direct initialization
-    IK_Copy,    ///< Copy initialization
-    IK_Default, ///< Default initialization
-    IK_Value    ///< Value initialization
-  };
-  
-private:
-  /// \brief The kind of initialization that we're storing.
-  enum StoredInitKind {
-    SIK_Direct = IK_Direct,   ///< Direct initialization
-    SIK_Copy = IK_Copy,       ///< Copy initialization
-    SIK_Default = IK_Default, ///< Default initialization
-    SIK_Value = IK_Value,     ///< Value initialization
-    SIK_ImplicitValue,        ///< Implicit value initialization
-    SIK_DirectCast,  ///< Direct initialization due to a cast
-    /// \brief Direct initialization due to a C-style or functional cast.
-    SIK_DirectCStyleOrFunctionalCast
-  };
-  
-  /// \brief The kind of initialization being performed.
-  StoredInitKind Kind;
-  
-  /// \brief The source locations involved in the initialization.
-  SourceLocation Locations[3];
-  
-  InitializationKind(StoredInitKind Kind, SourceLocation Loc1, 
-                     SourceLocation Loc2, SourceLocation Loc3)
-    : Kind(Kind) 
-  {
-    Locations[0] = Loc1;
-    Locations[1] = Loc2;
-    Locations[2] = Loc3;
-  }
-  
-public:
-  /// \brief Create a direct initialization.
-  static InitializationKind CreateDirect(SourceLocation InitLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation RParenLoc) {
-    return InitializationKind(SIK_Direct, InitLoc, LParenLoc, RParenLoc);
-  }
-
-  /// \brief Create a direct initialization due to a cast.
-  static InitializationKind CreateCast(SourceRange TypeRange,
-                                       bool IsCStyleCast) {
-    return InitializationKind(IsCStyleCast? SIK_DirectCStyleOrFunctionalCast
-                                          : SIK_DirectCast,
-                              TypeRange.getBegin(), TypeRange.getBegin(), 
-                              TypeRange.getEnd());
-  }
-  
-  /// \brief Create a copy initialization.
-  static InitializationKind CreateCopy(SourceLocation InitLoc,
-                                       SourceLocation EqualLoc) {
-    return InitializationKind(SIK_Copy, InitLoc, EqualLoc, EqualLoc);
-  }
-  
-  /// \brief Create a default initialization.
-  static InitializationKind CreateDefault(SourceLocation InitLoc) {
-    return InitializationKind(SIK_Default, InitLoc, InitLoc, InitLoc);
-  }
-  
-  /// \brief Create a value initialization.
-  static InitializationKind CreateValue(SourceLocation InitLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation RParenLoc,
-                                        bool isImplicit = false) {
-    return InitializationKind(isImplicit? SIK_ImplicitValue : SIK_Value, 
-                              InitLoc, LParenLoc, RParenLoc);
-  }
-  
-  /// \brief Determine the initialization kind.
-  InitKind getKind() const {
-    if (Kind > SIK_ImplicitValue)
-      return IK_Direct;
-    if (Kind == SIK_ImplicitValue)
-      return IK_Value;
-
-    return (InitKind)Kind;
-  }
-  
-  /// \brief Determine whether this initialization is an explicit cast.
-  bool isExplicitCast() const {
-    return Kind == SIK_DirectCast || Kind == SIK_DirectCStyleOrFunctionalCast;
-  }
-  
-  /// \brief Determine whether this initialization is a C-style cast.
-  bool isCStyleOrFunctionalCast() const { 
-    return Kind == SIK_DirectCStyleOrFunctionalCast; 
-  }
-
-  /// \brief Determine whether this initialization is an implicit
-  /// value-initialization, e.g., as occurs during aggregate
-  /// initialization.
-  bool isImplicitValueInit() const { return Kind == SIK_ImplicitValue; }
-
-  /// \brief Retrieve the location at which initialization is occurring.
-  SourceLocation getLocation() const { return Locations[0]; }
-  
-  /// \brief Retrieve the source range that covers the initialization.
-  SourceRange getRange() const { 
-    return SourceRange(Locations[0], Locations[2]);
-  }
-  
-  /// \brief Retrieve the location of the equal sign for copy initialization
-  /// (if present).
-  SourceLocation getEqualLoc() const {
-    assert(Kind == SIK_Copy && "Only copy initialization has an '='");
-    return Locations[1];
-  }
-  
-  /// \brief Retrieve the source range containing the locations of the open
-  /// and closing parentheses for value and direct initializations.
-  SourceRange getParenRange() const {
-    assert((getKind() == IK_Direct || Kind == SIK_Value) &&
-           "Only direct- and value-initialization have parentheses");
-    return SourceRange(Locations[1], Locations[2]);
-  }
-};
-
-/// \brief Describes the sequence of initializations required to initialize
-/// a given object or reference with a set of arguments.
-class InitializationSequence {
-public:
-  /// \brief Describes the kind of initialization sequence computed.
-  ///
-  /// FIXME: Much of this information is in the initialization steps... why is
-  /// it duplicated here?
-  enum SequenceKind {
-    /// \brief A failed initialization sequence. The failure kind tells what
-    /// happened.
-    FailedSequence = 0,
-    
-    /// \brief A dependent initialization, which could not be
-    /// type-checked due to the presence of dependent types or
-    /// dependently-type expressions.
-    DependentSequence,
-
-    /// \brief A user-defined conversion sequence.
-    UserDefinedConversion,
-    
-    /// \brief A constructor call.
-    ConstructorInitialization,
-    
-    /// \brief A reference binding.
-    ReferenceBinding,
-
-    /// \brief List initialization
-    ListInitialization,
-    
-    /// \brief Zero-initialization.
-    ZeroInitialization,
-    
-    /// \brief No initialization required.
-    NoInitialization,
-    
-    /// \brief Standard conversion sequence.
-    StandardConversion,
-
-    /// \brief C conversion sequence.
-    CAssignment,
-
-    /// \brief String initialization
-    StringInit
-  };
-  
-  /// \brief Describes the kind of a particular step in an initialization
-  /// sequence.
-  enum StepKind {
-    /// \brief Resolve the address of an overloaded function to a specific
-    /// function declaration.
-    SK_ResolveAddressOfOverloadedFunction,
-    /// \brief Perform a derived-to-base cast, producing an rvalue.
-    SK_CastDerivedToBaseRValue,
-    /// \brief Perform a derived-to-base cast, producing an lvalue.
-    SK_CastDerivedToBaseLValue,
-    /// \brief Reference binding to an lvalue.
-    SK_BindReference,
-    /// \brief Reference binding to a temporary.
-    SK_BindReferenceToTemporary,
-    /// \brief An optional copy of a temporary object to another
-    /// temporary object, which is permitted (but not required) by
-    /// C++98/03 but not C++0x.
-    SK_ExtraneousCopyToTemporary,
-    /// \brief Perform a user-defined conversion, either via a conversion
-    /// function or via a constructor.
-    SK_UserConversion,
-    /// \brief Perform a qualification conversion, producing an rvalue.
-    SK_QualificationConversionRValue,
-    /// \brief Perform a qualification conversion, producing an lvalue.
-    SK_QualificationConversionLValue,
-    /// \brief Perform an implicit conversion sequence.
-    SK_ConversionSequence,
-    /// \brief Perform list-initialization
-    SK_ListInitialization,
-    /// \brief Perform initialization via a constructor.
-    SK_ConstructorInitialization,
-    /// \brief Zero-initialize the object
-    SK_ZeroInitialization,
-    /// \brief C assignment
-    SK_CAssignment,
-    /// \brief Initialization by string
-    SK_StringInit
-  };
-  
-  /// \brief A single step in the initialization sequence.
-  class Step {
-  public:
-    /// \brief The kind of conversion or initialization step we are taking.
-    StepKind Kind;
-    
-    // \brief The type that results from this initialization.
-    QualType Type;
-    
-    union {
-      /// \brief When Kind == SK_ResolvedOverloadedFunction or Kind ==
-      /// SK_UserConversion, the function that the expression should be 
-      /// resolved to or the conversion function to call, respectively.
-      ///
-      /// Always a FunctionDecl.
-      /// For conversion decls, the naming class is the source type.
-      /// For construct decls, the naming class is the target type.
-      struct {
-        FunctionDecl *Function;
-        DeclAccessPair FoundDecl;
-      } Function;
-      
-      /// \brief When Kind = SK_ConversionSequence, the implicit conversion
-      /// sequence 
-      ImplicitConversionSequence *ICS;
-    };
-    
-    void Destroy();
-  };
-  
-private:
-  /// \brief The kind of initialization sequence computed.
-  enum SequenceKind SequenceKind;
-  
-  /// \brief Steps taken by this initialization.
-  llvm::SmallVector<Step, 4> Steps;
-  
-public:
-  /// \brief Describes why initialization failed.
-  enum FailureKind {
-    /// \brief Too many initializers provided for a reference.
-    FK_TooManyInitsForReference,
-    /// \brief Array must be initialized with an initializer list.
-    FK_ArrayNeedsInitList,
-    /// \brief Array must be initialized with an initializer list or a 
-    /// string literal.
-    FK_ArrayNeedsInitListOrStringLiteral,
-    /// \brief Cannot resolve the address of an overloaded function.
-    FK_AddressOfOverloadFailed,
-    /// \brief Overloading due to reference initialization failed.
-    FK_ReferenceInitOverloadFailed,
-    /// \brief Non-const lvalue reference binding to a temporary.
-    FK_NonConstLValueReferenceBindingToTemporary,
-    /// \brief Non-const lvalue reference binding to an lvalue of unrelated
-    /// type.
-    FK_NonConstLValueReferenceBindingToUnrelated,
-    /// \brief Rvalue reference binding to an lvalue.
-    FK_RValueReferenceBindingToLValue,
-    /// \brief Reference binding drops qualifiers.
-    FK_ReferenceInitDropsQualifiers,
-    /// \brief Reference binding failed.
-    FK_ReferenceInitFailed,
-    /// \brief Implicit conversion failed.
-    FK_ConversionFailed,
-    /// \brief Too many initializers for scalar
-    FK_TooManyInitsForScalar,
-    /// \brief Reference initialization from an initializer list
-    FK_ReferenceBindingToInitList,
-    /// \brief Initialization of some unused destination type with an
-    /// initializer list.
-    FK_InitListBadDestinationType,
-    /// \brief Overloading for a user-defined conversion failed.
-    FK_UserConversionOverloadFailed,
-    /// \brief Overloaded for initialization by constructor failed.
-    FK_ConstructorOverloadFailed,
-    /// \brief Default-initialization of a 'const' object.
-    FK_DefaultInitOfConst,
-    /// \brief Initialization of an incomplete type.
-    FK_Incomplete
-  };
-  
-private:
-  /// \brief The reason why initialization failued.
-  FailureKind Failure;
-
-  /// \brief The failed result of overload resolution.
-  OverloadingResult FailedOverloadResult;
-  
-  /// \brief The candidate set created when initialization failed.
-  OverloadCandidateSet FailedCandidateSet;
-
-  /// \brief Prints a follow-up note that highlights the location of
-  /// the initialized entity, if it's remote.
-  void PrintInitLocationNote(Sema &S, const InitializedEntity &Entity);
-
-public:
-  /// \brief Try to perform initialization of the given entity, creating a 
-  /// record of the steps required to perform the initialization.
-  ///
-  /// The generated initialization sequence will either contain enough
-  /// information to diagnose 
-  ///
-  /// \param S the semantic analysis object.
-  ///
-  /// \param Entity the entity being initialized.
-  ///
-  /// \param Kind the kind of initialization being performed.
-  ///
-  /// \param Args the argument(s) provided for initialization.
-  ///
-  /// \param NumArgs the number of arguments provided for initialization.
-  InitializationSequence(Sema &S, 
-                         const InitializedEntity &Entity,
-                         const InitializationKind &Kind,
-                         Expr **Args,
-                         unsigned NumArgs);
-  
-  ~InitializationSequence();
-  
-  /// \brief Perform the actual initialization of the given entity based on
-  /// the computed initialization sequence.
-  ///
-  /// \param S the semantic analysis object.
-  ///
-  /// \param Entity the entity being initialized.
-  ///
-  /// \param Kind the kind of initialization being performed.
-  ///
-  /// \param Args the argument(s) provided for initialization, ownership of
-  /// which is transfered into the routine.
-  ///
-  /// \param ResultType if non-NULL, will be set to the type of the
-  /// initialized object, which is the type of the declaration in most
-  /// cases. However, when the initialized object is a variable of
-  /// incomplete array type and the initializer is an initializer
-  /// list, this type will be set to the completed array type.
-  ///
-  /// \returns an expression that performs the actual object initialization, if
-  /// the initialization is well-formed. Otherwise, emits diagnostics
-  /// and returns an invalid expression.
-  Action::OwningExprResult Perform(Sema &S,
-                                   const InitializedEntity &Entity,
-                                   const InitializationKind &Kind,
-                                   Action::MultiExprArg Args,
-                                   QualType *ResultType = 0);
-  
-  /// \brief Diagnose an potentially-invalid initialization sequence.
-  ///
-  /// \returns true if the initialization sequence was ill-formed, 
-  /// false otherwise.
-  bool Diagnose(Sema &S, 
-                const InitializedEntity &Entity,
-                const InitializationKind &Kind,
-                Expr **Args, unsigned NumArgs);
-  
-  /// \brief Determine the kind of initialization sequence computed.
-  enum SequenceKind getKind() const { return SequenceKind; }
-  
-  /// \brief Set the kind of sequence computed.
-  void setSequenceKind(enum SequenceKind SK) { SequenceKind = SK; }
-  
-  /// \brief Determine whether the initialization sequence is valid.
-  operator bool() const { return SequenceKind != FailedSequence; }
-  
-  typedef llvm::SmallVector<Step, 4>::const_iterator step_iterator;
-  step_iterator step_begin() const { return Steps.begin(); }
-  step_iterator step_end()   const { return Steps.end(); }
-
-  /// \brief Determine whether this initialization is a direct reference 
-  /// binding (C++ [dcl.init.ref]).
-  bool isDirectReferenceBinding() const;
-  
-  /// \brief Determine whether this initialization failed due to an ambiguity.
-  bool isAmbiguous() const;
-  
-  /// \brief Determine whether this initialization is direct call to a 
-  /// constructor.
-  bool isConstructorInitialization() const;
-  
-  /// \brief Add a new step in the initialization that resolves the address
-  /// of an overloaded function to a specific function declaration.
-  ///
-  /// \param Function the function to which the overloaded function reference
-  /// resolves.
-  void AddAddressOverloadResolutionStep(FunctionDecl *Function,
-                                        DeclAccessPair Found);
-  
-  /// \brief Add a new step in the initialization that performs a derived-to-
-  /// base cast.
-  ///
-  /// \param BaseType the base type to which we will be casting.
-  ///
-  /// \param IsLValue true if the result of this cast will be treated as 
-  /// an lvalue.
-  void AddDerivedToBaseCastStep(QualType BaseType, bool IsLValue);
-     
-  /// \brief Add a new step binding a reference to an object.
-  ///
-  /// \param BindingTemporary True if we are binding a reference to a temporary
-  /// object (thereby extending its lifetime); false if we are binding to an
-  /// lvalue or an lvalue treated as an rvalue.
-  ///
-  /// \param UnnecessaryCopy True if we should check for a copy
-  /// constructor for a completely unnecessary but
-  void AddReferenceBindingStep(QualType T, bool BindingTemporary);
-
-  /// \brief Add a new step that makes an extraneous copy of the input
-  /// to a temporary of the same class type.
-  ///
-  /// This extraneous copy only occurs during reference binding in
-  /// C++98/03, where we are permitted (but not required) to introduce
-  /// an extra copy. At a bare minimum, we must check that we could
-  /// call the copy constructor, and produce a diagnostic if the copy
-  /// constructor is inaccessible or no copy constructor matches.
-  //
-  /// \param T The type of the temporary being created.
-  void AddExtraneousCopyToTemporary(QualType T);
-
-  /// \brief Add a new step invoking a conversion function, which is either
-  /// a constructor or a conversion function.
-  void AddUserConversionStep(FunctionDecl *Function,
-                             DeclAccessPair FoundDecl,
-                             QualType T);
-  
-  /// \brief Add a new step that performs a qualification conversion to the
-  /// given type.
-  void AddQualificationConversionStep(QualType Ty, bool IsLValue);
-  
-  /// \brief Add a new step that applies an implicit conversion sequence.
-  void AddConversionSequenceStep(const ImplicitConversionSequence &ICS,
-                                 QualType T);
-
-  /// \brief Add a list-initialiation step  
-  void AddListInitializationStep(QualType T);
-
-  /// \brief Add a constructor-initialization step.
-  void AddConstructorInitializationStep(CXXConstructorDecl *Constructor,
-                                        AccessSpecifier Access,
-                                        QualType T);
-
-  /// \brief Add a zero-initialization step.
-  void AddZeroInitializationStep(QualType T);
-  
-  /// \brief Add a C assignment step.
-  //
-  // FIXME: It isn't clear whether this should ever be needed;
-  // ideally, we would handle everything needed in C in the common
-  // path. However, that isn't the case yet.
-  void AddCAssignmentStep(QualType T);
-
-  /// \brief Add a string init step.
-  void AddStringInitStep(QualType T);
-
-  /// \brief Note that this initialization sequence failed.
-  void SetFailed(FailureKind Failure) {
-    SequenceKind = FailedSequence;
-    this->Failure = Failure;
-  }
-  
-  /// \brief Note that this initialization sequence failed due to failed
-  /// overload resolution.
-  void SetOverloadFailure(FailureKind Failure, OverloadingResult Result);
-  
-  /// \brief Retrieve a reference to the candidate set when overload
-  /// resolution fails.
-  OverloadCandidateSet &getFailedCandidateSet() {
-    return FailedCandidateSet;
-  }
-
-  /// \brief Determine why initialization failed.
-  FailureKind getFailureKind() const {
-    assert(getKind() == FailedSequence && "Not an initialization failure!");
-    return Failure;
-  }
-  
-  /// \brief Dump a representation of this initialization sequence to 
-  /// the given stream, for debugging purposes.
-  void dump(llvm::raw_ostream &OS) const;
-  
-  /// \brief Dump a representation of this initialization sequence to 
-  /// standard error, for debugging purposes.
-  void dump() const;
-};
-  
-} // end namespace clang
-
-#endif // LLVM_CLANG_SEMA_INIT_H
diff --git a/lib/Sema/SemaLookup.cpp b/lib/Sema/SemaLookup.cpp
index 2e651838df95..306e95a497e8 100644
--- a/lib/Sema/SemaLookup.cpp
+++ b/lib/Sema/SemaLookup.cpp
@@ -11,8 +11,13 @@
 //  Objective-C++.
 //
 //===----------------------------------------------------------------------===//
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/Decl.h"
@@ -21,9 +26,9 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
-#include "clang/Parse/DeclSpec.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/LangOptions.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -35,6 +40,7 @@
 #include <algorithm>
 
 using namespace clang;
+using namespace sema;
 
 namespace {
   class UnqualUsingEntry {
@@ -100,7 +106,7 @@ namespace {
                              End = S->using_directives_end();
           
           for (; I != End; ++I)
-            visit(I->getAs<UsingDirectiveDecl>(), InnermostFileDC);
+            visit(*I, InnermostFileDC);
         }
       }
     }
@@ -254,6 +260,12 @@ static inline unsigned getIDNS(Sema::LookupNameKind NameKind,
   case Sema::LookupObjCProtocolName:
     IDNS = Decl::IDNS_ObjCProtocol;
     break;
+      
+  case Sema::LookupAnyName:
+    IDNS = Decl::IDNS_Ordinary | Decl::IDNS_Tag | Decl::IDNS_Member 
+      | Decl::IDNS_Using | Decl::IDNS_Namespace | Decl::IDNS_ObjCProtocol
+      | Decl::IDNS_Type;
+    break;
   }
   return IDNS;
 }
@@ -267,7 +279,7 @@ void LookupResult::configure() {
   // operators, make sure that the implicitly-declared new and delete
   // operators can be found.
   if (!isForRedeclaration()) {
-    switch (Name.getCXXOverloadedOperator()) {
+    switch (NameInfo.getName().getCXXOverloadedOperator()) {
     case OO_New:
     case OO_Delete:
     case OO_Array_New:
@@ -281,6 +293,22 @@ void LookupResult::configure() {
   }
 }
 
+#ifndef NDEBUG
+void LookupResult::sanity() const {
+  assert(ResultKind != NotFound || Decls.size() == 0);
+  assert(ResultKind != Found || Decls.size() == 1);
+  assert(ResultKind != FoundOverloaded || Decls.size() > 1 ||
+         (Decls.size() == 1 &&
+          isa<FunctionTemplateDecl>((*begin())->getUnderlyingDecl())));
+  assert(ResultKind != FoundUnresolvedValue || sanityCheckUnresolved());
+  assert(ResultKind != Ambiguous || Decls.size() > 1 ||
+         (Decls.size() == 1 && Ambiguity == AmbiguousBaseSubobjects));
+  assert((Paths != NULL) == (ResultKind == Ambiguous &&
+                             (Ambiguity == AmbiguousBaseSubobjectTypes ||
+                              Ambiguity == AmbiguousBaseSubobjects)));
+}
+#endif
+
 // Necessary because CXXBasePaths is not complete in Sema.h
 void LookupResult::deletePaths(CXXBasePaths *Paths) {
   delete Paths;
@@ -311,7 +339,8 @@ void LookupResult::resolveKind() {
   if (ResultKind == Ambiguous) return;
 
   llvm::SmallPtrSet<NamedDecl*, 16> Unique;
-
+  llvm::SmallPtrSet<QualType, 16> UniqueTypes;
+  
   bool Ambiguous = false;
   bool HasTag = false, HasFunction = false, HasNonFunction = false;
   bool HasFunctionTemplate = false, HasUnresolved = false;
@@ -323,32 +352,49 @@ void LookupResult::resolveKind() {
     NamedDecl *D = Decls[I]->getUnderlyingDecl();
     D = cast<NamedDecl>(D->getCanonicalDecl());
 
+    // Redeclarations of types via typedef can occur both within a scope
+    // and, through using declarations and directives, across scopes. There is
+    // no ambiguity if they all refer to the same type, so unique based on the
+    // canonical type.
+    if (TypeDecl *TD = dyn_cast<TypeDecl>(D)) {
+      if (!TD->getDeclContext()->isRecord()) {
+        QualType T = SemaRef.Context.getTypeDeclType(TD);
+        if (!UniqueTypes.insert(SemaRef.Context.getCanonicalType(T))) {
+          // The type is not unique; pull something off the back and continue
+          // at this index.
+          Decls[I] = Decls[--N];
+          continue;
+        }
+      }
+    }
+    
     if (!Unique.insert(D)) {
       // If it's not unique, pull something off the back (and
       // continue at this index).
       Decls[I] = Decls[--N];
+      continue;
+    } 
+    
+    // Otherwise, do some decl type analysis and then continue.
+
+    if (isa<UnresolvedUsingValueDecl>(D)) {
+      HasUnresolved = true;
+    } else if (isa<TagDecl>(D)) {
+      if (HasTag)
+        Ambiguous = true;
+      UniqueTagIndex = I;
+      HasTag = true;
+    } else if (isa<FunctionTemplateDecl>(D)) {
+      HasFunction = true;
+      HasFunctionTemplate = true;
+    } else if (isa<FunctionDecl>(D)) {
+      HasFunction = true;
     } else {
-      // Otherwise, do some decl type analysis and then continue.
-
-      if (isa<UnresolvedUsingValueDecl>(D)) {
-        HasUnresolved = true;
-      } else if (isa<TagDecl>(D)) {
-        if (HasTag)
-          Ambiguous = true;
-        UniqueTagIndex = I;
-        HasTag = true;
-      } else if (isa<FunctionTemplateDecl>(D)) {
-        HasFunction = true;
-        HasFunctionTemplate = true;
-      } else if (isa<FunctionDecl>(D)) {
-        HasFunction = true;
-      } else {
-        if (HasNonFunction)
-          Ambiguous = true;
-        HasNonFunction = true;
-      }
-      I++;
+      if (HasNonFunction)
+        Ambiguous = true;
+      HasNonFunction = true;
     }
+    I++;
   }
 
   // C++ [basic.scope.hiding]p2:
@@ -451,6 +497,10 @@ static bool LookupBuiltin(Sema &S, LookupResult &R) {
 /// the class at this point.
 static bool CanDeclareSpecialMemberFunction(ASTContext &Context,
                                             const CXXRecordDecl *Class) {
+  // Don't do it if the class is invalid.
+  if (Class->isInvalidDecl())
+    return false;
+  
   // We need to have a definition for the class.
   if (!Class->getDefinition() || Class->isDependentContext())
     return false;
@@ -608,7 +658,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
     // result), perform template argument deduction and place the 
     // specialization into the result set. We do this to avoid forcing all
     // callers to perform special deduction for conversion functions.
-    Sema::TemplateDeductionInfo Info(R.getSema().Context, R.getNameLoc());
+    TemplateDeductionInfo Info(R.getSema().Context, R.getNameLoc());
     FunctionDecl *Specialization = 0;
     
     const FunctionProtoType *ConvProto        
@@ -783,7 +833,7 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
 
     // Check whether the IdResolver has anything in this scope.
     bool Found = false;
-    for (; I != IEnd && S->isDeclScope(DeclPtrTy::make(*I)); ++I) {
+    for (; I != IEnd && S->isDeclScope(*I); ++I) {
       if (R.isAcceptableDecl(*I)) {
         Found = true;
         R.addDecl(*I);
@@ -881,7 +931,7 @@ bool Sema::CppLookupName(LookupResult &R, Scope *S) {
   for (; S; S = S->getParent()) {
     // Check whether the IdResolver has anything in this scope.
     bool Found = false;
-    for (; I != IEnd && S->isDeclScope(DeclPtrTy::make(*I)); ++I) {
+    for (; I != IEnd && S->isDeclScope(*I); ++I) {
       if (R.isAcceptableDecl(*I)) {
         // We found something.  Look for anything else in our scope
         // with this same name and in an acceptable identifier
@@ -1017,7 +1067,7 @@ bool Sema::LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation) {
         if (NameKind == LookupRedeclarationWithLinkage) {
           // Determine whether this (or a previous) declaration is
           // out-of-scope.
-          if (!LeftStartingScope && !S->isDeclScope(DeclPtrTy::make(*I)))
+          if (!LeftStartingScope && !S->isDeclScope(*I))
             LeftStartingScope = true;
 
           // If we found something outside of our starting scope that
@@ -1034,14 +1084,14 @@ bool Sema::LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation) {
 
           // Figure out what scope the identifier is in.
           while (!(S->getFlags() & Scope::DeclScope) ||
-                 !S->isDeclScope(DeclPtrTy::make(*I)))
+                 !S->isDeclScope(*I))
             S = S->getParent();
 
           // Find the last declaration in this scope (with the same
           // name, naturally).
           IdentifierResolver::iterator LastI = I;
           for (++LastI; LastI != IEnd; ++LastI) {
-            if (!S->isDeclScope(DeclPtrTy::make(*LastI)))
+            if (!S->isDeclScope(*LastI))
               break;
             R.addDecl(*LastI);
           }
@@ -1177,6 +1227,17 @@ static bool LookupQualifiedNameInUsingDirectives(Sema &S, LookupResult &R,
   return Found;
 }
 
+/// \brief Callback that looks for any member of a class with the given name.
+static bool LookupAnyMember(const CXXBaseSpecifier *Specifier, 
+                            CXXBasePath &Path,
+                            void *Name) {
+  RecordDecl *BaseRecord = Specifier->getType()->getAs<RecordType>()->getDecl();
+  
+  DeclarationName N = DeclarationName::getFromOpaquePtr(Name);
+  Path.Decls = BaseRecord->lookup(N);
+  return Path.Decls.first != Path.Decls.second;
+}
+
 /// \brief Perform qualified name lookup into a given context.
 ///
 /// Qualified name lookup (C++ [basic.lookup.qual]) is used to find
@@ -1272,6 +1333,10 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
       BaseCallback = &CXXRecordDecl::FindTagMember;
       break;
 
+    case LookupAnyName:
+      BaseCallback = &LookupAnyMember;
+      break;
+      
     case LookupUsingDeclName:
       // This lookup is for redeclarations only.
       
@@ -1554,7 +1619,11 @@ static void CollectEnclosingNamespace(Sema::AssociatedNamespaceSet &Namespaces,
   // We don't use DeclContext::getEnclosingNamespaceContext() as this may
   // be a locally scoped record.
 
-  while (Ctx->isRecord() || Ctx->isTransparentContext())
+  // We skip out of inline namespaces. The innermost non-inline namespace
+  // contains all names of all its nested inline namespaces anyway, so we can
+  // replace the entire inline namespace tree with its root.
+  while (Ctx->isRecord() || Ctx->isTransparentContext() ||
+         Ctx->isInlineNamespace())
     Ctx = Ctx->getParent();
 
   if (Ctx->isFileContext())
@@ -1894,7 +1963,7 @@ Sema::FindAssociatedClassesAndNamespaces(Expr **Args, unsigned NumArgs,
     // parameter types and return type.
     Arg = Arg->IgnoreParens();
     if (UnaryOperator *unaryOp = dyn_cast<UnaryOperator>(Arg))
-      if (unaryOp->getOpcode() == UnaryOperator::AddrOf)
+      if (unaryOp->getOpcode() == UO_AddrOf)
         Arg = unaryOp->getSubExpr();
 
     UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(Arg);
@@ -2201,6 +2270,10 @@ public:
     return !VisitedContexts.insert(Ctx);
   }
 
+  bool alreadyVisitedContext(DeclContext *Ctx) {
+    return VisitedContexts.count(Ctx);
+  }
+
   /// \brief Determine whether the given declaration is hidden in the
   /// current scope.
   ///
@@ -2354,9 +2427,9 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
           Visited.add(ND);
         }
 
-      // Visit transparent contexts inside this context.
+      // Visit transparent contexts and inline namespaces inside this context.
       if (DeclContext *InnerCtx = dyn_cast<DeclContext>(*D)) {
-        if (InnerCtx->isTransparentContext())
+        if (InnerCtx->isTransparentContext() || InnerCtx->isInlineNamespace())
           LookupVisibleDecls(InnerCtx, Result, QualifiedNameLookup, InBaseClass,
                              Consumer, Visited);
       }
@@ -2429,8 +2502,9 @@ static void LookupVisibleDecls(DeclContext *Ctx, LookupResult &Result,
     }
 
     // Traverse protocols.
-    for (ObjCInterfaceDecl::protocol_iterator I = IFace->protocol_begin(),
-         E = IFace->protocol_end(); I != E; ++I) {
+    for (ObjCInterfaceDecl::all_protocol_iterator
+         I = IFace->all_referenced_protocol_begin(),
+         E = IFace->all_referenced_protocol_end(); I != E; ++I) {
       ShadowContextRAII Shadow(Visited);
       LookupVisibleDecls(*I, Result, QualifiedNameLookup, false, Consumer, 
                          Visited);
@@ -2481,12 +2555,14 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
   if (!S)
     return;
 
-  if (!S->getEntity() || !S->getParent() ||
+  if (!S->getEntity() || 
+      (!S->getParent() && 
+       !Visited.alreadyVisitedContext((DeclContext *)S->getEntity())) ||
       ((DeclContext *)S->getEntity())->isFunctionOrMethod()) {
     // Walk through the declarations in this Scope.
     for (Scope::decl_iterator D = S->decl_begin(), DEnd = S->decl_end();
          D != DEnd; ++D) {
-      if (NamedDecl *ND = dyn_cast<NamedDecl>((Decl *)((*D).get())))
+      if (NamedDecl *ND = dyn_cast<NamedDecl>(*D))
         if (Result.isAcceptableDecl(ND)) {
           Consumer.FoundDecl(ND, Visited.checkHidden(ND), false);
           Visited.add(ND);
@@ -2559,7 +2635,8 @@ static void LookupVisibleDecls(Scope *S, LookupResult &Result,
 }
 
 void Sema::LookupVisibleDecls(Scope *S, LookupNameKind Kind,
-                              VisibleDeclConsumer &Consumer) {
+                              VisibleDeclConsumer &Consumer,
+                              bool IncludeGlobalScope) {
   // Determine the set of using directives available during
   // unqualified name lookup.
   Scope *Initial = S;
@@ -2576,14 +2653,19 @@ void Sema::LookupVisibleDecls(Scope *S, LookupNameKind Kind,
   // Look for visible declarations.
   LookupResult Result(*this, DeclarationName(), SourceLocation(), Kind);
   VisibleDeclsRecord Visited;
+  if (!IncludeGlobalScope)
+    Visited.visitedContext(Context.getTranslationUnitDecl());
   ShadowContextRAII Shadow(Visited);
   ::LookupVisibleDecls(Initial, Result, UDirs, Consumer, Visited);
 }
 
 void Sema::LookupVisibleDecls(DeclContext *Ctx, LookupNameKind Kind,
-                              VisibleDeclConsumer &Consumer) {
+                              VisibleDeclConsumer &Consumer,
+                              bool IncludeGlobalScope) {
   LookupResult Result(*this, DeclarationName(), SourceLocation(), Kind);
   VisibleDeclsRecord Visited;
+  if (!IncludeGlobalScope)
+    Visited.visitedContext(Context.getTranslationUnitDecl());
   ShadowContextRAII Shadow(Visited);
   ::LookupVisibleDecls(Ctx, Result, /*QualifiedNameLookup=*/true, 
                        /*InBaseClass=*/false, Consumer, Visited);
@@ -2911,7 +2993,7 @@ DeclarationName Sema::CorrectTypo(LookupResult &Res, Scope *S, CXXScopeSpec *SS,
       if (S && S->getContinueParent())
         Consumer.addKeywordResult(Context, "continue");
       
-      if (!getSwitchStack().empty()) {
+      if (!getCurFunction()->SwitchStack.empty()) {
         Consumer.addKeywordResult(Context, "case");
         Consumer.addKeywordResult(Context, "default");
       }
diff --git a/lib/Sema/SemaObjCProperty.cpp b/lib/Sema/SemaObjCProperty.cpp
index ff60599b8510..7181d58f7fb0 100644
--- a/lib/Sema/SemaObjCProperty.cpp
+++ b/lib/Sema/SemaObjCProperty.cpp
@@ -12,9 +12,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprObjC.h"
+#include "llvm/ADT/DenseSet.h"
 
 using namespace clang;
 
@@ -22,14 +24,14 @@ using namespace clang;
 // Grammar actions.
 //===----------------------------------------------------------------------===//
 
-Sema::DeclPtrTy Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
-                                    FieldDeclarator &FD,
-                                    ObjCDeclSpec &ODS,
-                                    Selector GetterSel,
-                                    Selector SetterSel,
-                                    DeclPtrTy ClassCategory,
-                                    bool *isOverridingProperty,
-                                    tok::ObjCKeywordKind MethodImplKind) {
+Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
+                          FieldDeclarator &FD,
+                          ObjCDeclSpec &ODS,
+                          Selector GetterSel,
+                          Selector SetterSel,
+                          Decl *ClassCategory,
+                          bool *isOverridingProperty,
+                          tok::ObjCKeywordKind MethodImplKind) {
   unsigned Attributes = ODS.getPropertyAttributes();
   bool isReadWrite = ((Attributes & ObjCDeclSpec::DQ_PR_readwrite) ||
                       // default is readwrite!
@@ -45,15 +47,15 @@ Sema::DeclPtrTy Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
   QualType T = TSI->getType();
   if (T->isReferenceType()) {
     Diag(AtLoc, diag::error_reference_property);
-    return DeclPtrTy();
+    return 0;
   }
   // Proceed with constructing the ObjCPropertDecls.
   ObjCContainerDecl *ClassDecl =
-    cast<ObjCContainerDecl>(ClassCategory.getAs<Decl>());
+    cast<ObjCContainerDecl>(ClassCategory);
 
   if (ObjCCategoryDecl *CDecl = dyn_cast<ObjCCategoryDecl>(ClassDecl))
     if (CDecl->IsClassExtension()) {
-      DeclPtrTy Res = HandlePropertyInClassExtension(S, CDecl, AtLoc,
+      Decl *Res = HandlePropertyInClassExtension(S, CDecl, AtLoc,
                                            FD, GetterSel, SetterSel,
                                            isAssign, isReadWrite,
                                            Attributes,
@@ -64,16 +66,16 @@ Sema::DeclPtrTy Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
       return Res;
     }
   
-  DeclPtrTy Res =  DeclPtrTy::make(CreatePropertyDecl(S, ClassDecl, AtLoc, FD,
-                                              GetterSel, SetterSel,
-                                              isAssign, isReadWrite,
-                                              Attributes, TSI, MethodImplKind));
+  Decl *Res = CreatePropertyDecl(S, ClassDecl, AtLoc, FD,
+                                 GetterSel, SetterSel,
+                                 isAssign, isReadWrite,
+                                 Attributes, TSI, MethodImplKind);
   // Validate the attributes on the @property.
   CheckObjCPropertyAttributes(Res, AtLoc, Attributes);
   return Res;
 }
 
-Sema::DeclPtrTy
+Decl *
 Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
                                      SourceLocation AtLoc, FieldDeclarator &FD,
                                      Selector GetterSel, Selector SetterSel,
@@ -92,7 +94,7 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
         ObjCPropertyDecl::findPropertyDecl(DC, PropertyId)) {
     Diag(AtLoc, diag::err_duplicate_property);
     Diag(prevDecl->getLocation(), diag::note_property_declare);
-    return DeclPtrTy();
+    return 0;
   }
 
   // Create a new ObjCPropertyDecl with the DeclContext being
@@ -113,7 +115,7 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
   if (!CCPrimary) {
     Diag(CDecl->getLocation(), diag::err_continuation_class);
     *isOverridingProperty = true;
-    return DeclPtrTy();
+    return 0;
   }
 
   // Find the property in continuation class's primary class only.
@@ -136,7 +138,7 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
     // is not what it was meant for. However, gcc supports it and so should we.
     // Make sure setter/getters are declared here.
     ProcessPropertyDecl(PDecl, CCPrimary);
-    return DeclPtrTy::make(PDecl);
+    return PDecl;
 
   }
 
@@ -165,13 +167,13 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
       setPropertyAttributes((ObjCDeclSpec::ObjCPropertyAttributeKind)
                             PIkind);
 
-      DeclPtrTy ProtocolPtrTy =
+      Decl *ProtocolPtrTy =
         ActOnProperty(S, AtLoc, FD, ProtocolPropertyODS,
                       PIDecl->getGetterName(),
                       PIDecl->getSetterName(),
-                      DeclPtrTy::make(CCPrimary), isOverridingProperty,
+                      CCPrimary, isOverridingProperty,
                       MethodImplKind);
-      PIDecl = cast<ObjCPropertyDecl>(ProtocolPtrTy.getAs<Decl>());
+      PIDecl = cast<ObjCPropertyDecl>(ProtocolPtrTy);
     }
     PIDecl->makeitReadWriteAttribute();
     if (Attributes & ObjCDeclSpec::DQ_PR_retain)
@@ -187,7 +189,7 @@ Sema::HandlePropertyInClassExtension(Scope *S, ObjCCategoryDecl *CDecl,
   *isOverridingProperty = true;
   // Make sure setter decl is synthesized, and added to primary class's list.
   ProcessPropertyDecl(PIDecl, CCPrimary);
-  return DeclPtrTy();
+  return 0;
 }
 
 ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
@@ -289,19 +291,19 @@ ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
 /// builds the AST node for a property implementation declaration; declared
 /// as @synthesize or @dynamic.
 ///
-Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
-                                            SourceLocation AtLoc,
-                                            SourceLocation PropertyLoc,
-                                            bool Synthesize,
-                                            DeclPtrTy ClassCatImpDecl,
-                                            IdentifierInfo *PropertyId,
-                                            IdentifierInfo *PropertyIvar) {
+Decl *Sema::ActOnPropertyImplDecl(Scope *S,
+                                  SourceLocation AtLoc,
+                                  SourceLocation PropertyLoc,
+                                  bool Synthesize,
+                                  Decl *ClassCatImpDecl,
+                                  IdentifierInfo *PropertyId,
+                                  IdentifierInfo *PropertyIvar) {
   ObjCContainerDecl *ClassImpDecl =
-    cast_or_null<ObjCContainerDecl>(ClassCatImpDecl.getAs<Decl>());
+    cast_or_null<ObjCContainerDecl>(ClassCatImpDecl);
   // Make sure we have a context for the property implementation declaration.
   if (!ClassImpDecl) {
     Diag(AtLoc, diag::error_missing_property_context);
-    return DeclPtrTy();
+    return 0;
   }
   ObjCPropertyDecl *property = 0;
   ObjCInterfaceDecl* IDecl = 0;
@@ -320,25 +322,25 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
     property = IDecl->FindPropertyDeclaration(PropertyId);
     if (!property) {
       Diag(PropertyLoc, diag::error_bad_property_decl) << IDecl->getDeclName();
-      return DeclPtrTy();
+      return 0;
     }
     if (const ObjCCategoryDecl *CD =
         dyn_cast<ObjCCategoryDecl>(property->getDeclContext())) {
       if (!CD->IsClassExtension()) {
         Diag(PropertyLoc, diag::error_category_property) << CD->getDeclName();
         Diag(property->getLocation(), diag::note_property_declare);
-        return DeclPtrTy();
+        return 0;
       }
     }
   } else if ((CatImplClass = dyn_cast<ObjCCategoryImplDecl>(ClassImpDecl))) {
     if (Synthesize) {
       Diag(AtLoc, diag::error_synthesize_category_decl);
-      return DeclPtrTy();
+      return 0;
     }
     IDecl = CatImplClass->getClassInterface();
     if (!IDecl) {
       Diag(AtLoc, diag::error_missing_property_interface);
-      return DeclPtrTy();
+      return 0;
     }
     ObjCCategoryDecl *Category =
     IDecl->FindCategoryDeclaration(CatImplClass->getIdentifier());
@@ -346,17 +348,17 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
     // If category for this implementation not found, it is an error which
     // has already been reported eralier.
     if (!Category)
-      return DeclPtrTy();
+      return 0;
     // Look for this property declaration in @implementation's category
     property = Category->FindPropertyDeclaration(PropertyId);
     if (!property) {
       Diag(PropertyLoc, diag::error_bad_category_property_decl)
       << Category->getDeclName();
-      return DeclPtrTy();
+      return 0;
     }
   } else {
     Diag(AtLoc, diag::error_bad_property_context);
-    return DeclPtrTy();
+    return 0;
   }
   ObjCIvarDecl *Ivar = 0;
   // Check that we have a valid, previously declared ivar for @synthesize
@@ -372,7 +374,7 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
       Ivar = ObjCIvarDecl::Create(Context, ClassImpDecl, PropertyLoc,
                                   PropertyIvar, PropType, /*Dinfo=*/0,
                                   ObjCIvarDecl::Protected,
-                                  (Expr *)0);
+                                  (Expr *)0, true);
       ClassImpDecl->addDecl(Ivar);
       IDecl->makeDeclVisibleInContext(Ivar, false);
       property->setPropertyIvarDecl(Ivar);
@@ -387,7 +389,7 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
       << property->getDeclName() << Ivar->getDeclName()
       << ClassDeclared->getDeclName();
       Diag(Ivar->getLocation(), diag::note_previous_access_declaration)
-      << Ivar << Ivar->getNameAsCString();
+      << Ivar << Ivar->getName();
       // Note! I deliberately want it to fall thru so more errors are caught.
     }
     QualType IvarType = Context.getCanonicalType(Ivar->getType());
@@ -464,7 +466,7 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
       Expr *IvarRefExpr =
         new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), AtLoc,
                                       SelfExpr, true, true);
-      OwningExprResult Res = 
+      ExprResult Res = 
         PerformCopyInitialization(InitializedEntity::InitializeResult(
                                     SourceLocation(),
                                     getterMethod->getResultType(),
@@ -494,8 +496,8 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
       ParmVarDecl *Param = (*P);
       Expr *rhs = new (Context) DeclRefExpr(Param,Param->getType(),
                                             SourceLocation());
-      OwningExprResult Res = BuildBinOp(S, SourceLocation(), 
-                                        BinaryOperator::Assign, lhs, rhs);
+      ExprResult Res = BuildBinOp(S, SourceLocation(), 
+                                  BO_Assign, lhs, rhs);
       PIDecl->setSetterCXXAssignment(Res.takeAs<Expr>());
     }
   }
@@ -514,9 +516,29 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
         = IC->FindPropertyImplDecl(PropertyId)) {
       Diag(PropertyLoc, diag::error_property_implemented) << PropertyId;
       Diag(PPIDecl->getLocation(), diag::note_previous_declaration);
-      return DeclPtrTy();
+      return 0;
     }
     IC->addPropertyImplementation(PIDecl);
+    if (getLangOptions().ObjCNonFragileABI2) {
+      // Diagnose if an ivar was lazily synthesdized due to a previous
+      // use and if 1) property is @dynamic or 2) property is synthesized
+      // but it requires an ivar of different name.
+      ObjCInterfaceDecl *ClassDeclared;
+      ObjCIvarDecl *Ivar = 0;
+      if (!Synthesize)
+        Ivar = IDecl->lookupInstanceVariable(PropertyId, ClassDeclared);
+      else {
+        if (PropertyIvar && PropertyIvar != PropertyId)
+          Ivar = IDecl->lookupInstanceVariable(PropertyId, ClassDeclared);
+      }
+      // Issue diagnostics only if Ivar belongs to current class.
+      if (Ivar && Ivar->getSynthesize() && 
+          IC->getClassInterface() == ClassDeclared) {
+        Diag(Ivar->getLocation(), diag::err_undeclared_var_use) 
+        << PropertyId;
+        Ivar->setInvalidDecl();
+      }
+    }
   } else {
     if (Synthesize)
       if (ObjCPropertyImplDecl *PPIDecl =
@@ -531,12 +553,12 @@ Sema::DeclPtrTy Sema::ActOnPropertyImplDecl(Scope *S,
         CatImplClass->FindPropertyImplDecl(PropertyId)) {
       Diag(PropertyLoc, diag::error_property_implemented) << PropertyId;
       Diag(PPIDecl->getLocation(), diag::note_previous_declaration);
-      return DeclPtrTy();
+      return 0;
     }
     CatImplClass->addPropertyImplementation(PIDecl);
   }
 
-  return DeclPtrTy::make(PIDecl);
+  return PIDecl;
 }
 
 //===----------------------------------------------------------------------===//
@@ -680,9 +702,8 @@ Sema::MatchOneProtocolPropertiesInClass(Decl *CDecl,
 /// declared in 'ClassOrProtocol' objects (which can be a class or an
 /// inherited protocol with the list of properties for class/category 'CDecl'
 ///
-void Sema::CompareProperties(Decl *CDecl,
-                             DeclPtrTy ClassOrProtocol) {
-  Decl *ClassDecl = ClassOrProtocol.getAs<Decl>();
+void Sema::CompareProperties(Decl *CDecl, Decl *ClassOrProtocol) {
+  Decl *ClassDecl = ClassOrProtocol;
   ObjCInterfaceDecl *IDecl = dyn_cast_or_null<ObjCInterfaceDecl>(CDecl);
 
   if (!IDecl) {
@@ -699,7 +720,7 @@ void Sema::CompareProperties(Decl *CDecl,
       // their properties with those in the category.
       for (ObjCCategoryDecl::protocol_iterator P = CatDecl->protocol_begin(),
            E = CatDecl->protocol_end(); P != E; ++P)
-        CompareProperties(CatDecl, DeclPtrTy::make(*P));
+        CompareProperties(CatDecl, *P);
     } else {
       ObjCProtocolDecl *MD = cast<ObjCProtocolDecl>(ClassDecl);
       for (ObjCProtocolDecl::protocol_iterator P = MD->protocol_begin(),
@@ -710,16 +731,18 @@ void Sema::CompareProperties(Decl *CDecl,
   }
 
   if (ObjCInterfaceDecl *MDecl = dyn_cast<ObjCInterfaceDecl>(ClassDecl)) {
-    for (ObjCInterfaceDecl::protocol_iterator P = MDecl->protocol_begin(),
-         E = MDecl->protocol_end(); P != E; ++P)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+          P = MDecl->all_referenced_protocol_begin(),
+          E = MDecl->all_referenced_protocol_end(); P != E; ++P)
       // Match properties of class IDecl with those of protocol (*P).
       MatchOneProtocolPropertiesInClass(IDecl, *P);
 
     // Go thru the list of protocols for this class and recursively match
     // their properties with those declared in the class.
-    for (ObjCInterfaceDecl::protocol_iterator P = IDecl->protocol_begin(),
-         E = IDecl->protocol_end(); P != E; ++P)
-      CompareProperties(IDecl, DeclPtrTy::make(*P));
+    for (ObjCInterfaceDecl::all_protocol_iterator
+          P = IDecl->all_referenced_protocol_begin(),
+          E = IDecl->all_referenced_protocol_end(); P != E; ++P)
+      CompareProperties(IDecl, *P);
   } else {
     ObjCProtocolDecl *MD = cast<ObjCProtocolDecl>(ClassDecl);
     for (ObjCProtocolDecl::protocol_iterator P = MD->protocol_begin(),
@@ -791,8 +814,9 @@ void Sema::CollectImmediateProperties(ObjCContainerDecl *CDecl,
       PropMap[Prop->getIdentifier()] = Prop;
     }
     // scan through class's protocols.
-    for (ObjCInterfaceDecl::protocol_iterator PI = IDecl->protocol_begin(),
-         E = IDecl->protocol_end(); PI != E; ++PI)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+         PI = IDecl->all_referenced_protocol_begin(),
+         E = IDecl->all_referenced_protocol_end(); PI != E; ++PI)
         CollectImmediateProperties((*PI), PropMap, SuperPropMap);
   }
   if (ObjCCategoryDecl *CATDecl = dyn_cast<ObjCCategoryDecl>(CDecl)) {
@@ -803,7 +827,7 @@ void Sema::CollectImmediateProperties(ObjCContainerDecl *CDecl,
         PropMap[Prop->getIdentifier()] = Prop;
       }
     // scan through class's protocols.
-    for (ObjCInterfaceDecl::protocol_iterator PI = CATDecl->protocol_begin(),
+    for (ObjCCategoryDecl::protocol_iterator PI = CATDecl->protocol_begin(),
          E = CATDecl->protocol_end(); PI != E; ++PI)
       CollectImmediateProperties((*PI), PropMap, SuperPropMap);
   }
@@ -838,8 +862,9 @@ static void CollectClassPropertyImplementations(ObjCContainerDecl *CDecl,
       ObjCPropertyDecl *Prop = (*P);
       PropMap[Prop->getIdentifier()] = Prop;
     }
-    for (ObjCInterfaceDecl::protocol_iterator PI = IDecl->protocol_begin(),
-         E = IDecl->protocol_end(); PI != E; ++PI)
+    for (ObjCInterfaceDecl::all_protocol_iterator
+         PI = IDecl->all_referenced_protocol_begin(),
+         E = IDecl->all_referenced_protocol_end(); PI != E; ++PI)
       CollectClassPropertyImplementations((*PI), PropMap);
   }
   else if (ObjCProtocolDecl *PDecl = dyn_cast<ObjCProtocolDecl>(CDecl)) {
@@ -881,8 +906,9 @@ ObjCPropertyDecl *Sema::LookupPropertyDecl(const ObjCContainerDecl *CDecl,
         return Prop;
     }
     // scan through class's protocols.
-    for (ObjCInterfaceDecl::protocol_iterator PI = IDecl->protocol_begin(),
-         E = IDecl->protocol_end(); PI != E; ++PI) {
+    for (ObjCInterfaceDecl::all_protocol_iterator
+         PI = IDecl->all_referenced_protocol_begin(),
+         E = IDecl->all_referenced_protocol_end(); PI != E; ++PI) {
       ObjCPropertyDecl *Prop = LookupPropertyDecl((*PI), II);
       if (Prop)
         return Prop;
@@ -933,9 +959,15 @@ void Sema::DefaultSynthesizeProperties (Scope *S, ObjCImplDecl* IMPDecl,
     // Property may have been synthesized by user.
     if (IMPDecl->FindPropertyImplDecl(Prop->getIdentifier()))
       continue;
+    if (IMPDecl->getInstanceMethod(Prop->getGetterName())) {
+      if (Prop->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_readonly)
+        continue;
+      if (IMPDecl->getInstanceMethod(Prop->getSetterName()))
+        continue;
+    }
 
     ActOnPropertyImplDecl(S, IMPDecl->getLocation(), IMPDecl->getLocation(),
-                          true, DeclPtrTy::make(IMPDecl),
+                          true, IMPDecl,
                           Prop->getIdentifier(), Prop->getIdentifier());
   }    
 }
@@ -1066,7 +1098,8 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
     // for this class.
     GetterMethod = ObjCMethodDecl::Create(Context, property->getLocation(),
                              property->getLocation(), property->getGetterName(),
-                             property->getType(), 0, CD, true, false, true,
+                             property->getType(), 0, CD, true, false, true, 
+                             false,
                              (property->getPropertyImplementation() ==
                               ObjCPropertyDecl::Optional) ?
                              ObjCMethodDecl::Optional :
@@ -1094,6 +1127,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
                                property->getLocation(),
                                property->getSetterName(),
                                Context.VoidTy, 0, CD, true, false, true,
+                               false,
                                (property->getPropertyImplementation() ==
                                 ObjCPropertyDecl::Optional) ?
                                ObjCMethodDecl::Optional :
@@ -1105,8 +1139,8 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
                                                   property->getIdentifier(),
                                                   property->getType(),
                                                   /*TInfo=*/0,
-                                                  VarDecl::None,
-                                                  VarDecl::None,
+                                                  SC_None,
+                                                  SC_None,
                                                   0);
       SetterMethod->setMethodParams(Context, &Argument, 1, 1);
       CD->addDecl(SetterMethod);
@@ -1138,11 +1172,10 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
     AddInstanceMethodToGlobalPool(SetterMethod);
 }
 
-void Sema::CheckObjCPropertyAttributes(DeclPtrTy PropertyPtrTy,
+void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
                                        SourceLocation Loc,
                                        unsigned &Attributes) {
   // FIXME: Improve the reported location.
-  Decl *PDecl = PropertyPtrTy.getAs<Decl>();
   if (!PDecl)
     return;
 
diff --git a/lib/Sema/SemaOverload.cpp b/lib/Sema/SemaOverload.cpp
index c4ab9061b445..11b4bb3b92c6 100644
--- a/lib/Sema/SemaOverload.cpp
+++ b/lib/Sema/SemaOverload.cpp
@@ -11,13 +11,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "Lookup.h"
-#include "SemaInit.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Template.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/TypeOrdering.h"
@@ -27,6 +30,34 @@
 #include <algorithm>
 
 namespace clang {
+using namespace sema;
+
+static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
+                                 bool InOverloadResolution,
+                                 StandardConversionSequence &SCS);
+static OverloadingResult
+IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
+                        UserDefinedConversionSequence& User,
+                        OverloadCandidateSet& Conversions,
+                        bool AllowExplicit);
+
+
+static ImplicitConversionSequence::CompareKind
+CompareStandardConversionSequences(Sema &S,
+                                   const StandardConversionSequence& SCS1,
+                                   const StandardConversionSequence& SCS2);
+
+static ImplicitConversionSequence::CompareKind
+CompareQualificationConversions(Sema &S,
+                                const StandardConversionSequence& SCS1,
+                                const StandardConversionSequence& SCS2);
+
+static ImplicitConversionSequence::CompareKind
+CompareDerivedToBaseConversions(Sema &S,
+                                const StandardConversionSequence& SCS1,
+                                const StandardConversionSequence& SCS2);
+
+
 
 /// GetConversionCategory - Retrieve the implicit conversion
 /// category corresponding to the given implicit conversion kind.
@@ -298,7 +329,7 @@ namespace {
 OverloadCandidate::DeductionFailureInfo
 static MakeDeductionFailureInfo(ASTContext &Context,
                                 Sema::TemplateDeductionResult TDK,
-                                Sema::TemplateDeductionInfo &Info) {
+                                TemplateDeductionInfo &Info) {
   OverloadCandidate::DeductionFailureInfo Result;
   Result.Result = static_cast<unsigned>(TDK);
   Result.Data = 0;
@@ -315,7 +346,7 @@ static MakeDeductionFailureInfo(ASTContext &Context,
     break;
       
   case Sema::TDK_Inconsistent:
-  case Sema::TDK_InconsistentQuals: {
+  case Sema::TDK_Underqualified: {
     // FIXME: Should allocate from normal heap so that we can free this later.
     DFIParamWithArguments *Saved = new (Context) DFIParamWithArguments;
     Saved->Param = Info.Param;
@@ -348,7 +379,7 @@ void OverloadCandidate::DeductionFailureInfo::Destroy() {
     break;
       
   case Sema::TDK_Inconsistent:
-  case Sema::TDK_InconsistentQuals:
+  case Sema::TDK_Underqualified:
     // FIXME: Destroy the data?
     Data = 0;
     break;
@@ -380,7 +411,7 @@ OverloadCandidate::DeductionFailureInfo::getTemplateParameter() {
     return TemplateParameter::getFromOpaqueValue(Data);    
 
   case Sema::TDK_Inconsistent:
-  case Sema::TDK_InconsistentQuals:
+  case Sema::TDK_Underqualified:
     return static_cast<DFIParamWithArguments*>(Data)->Param;
       
   // Unhandled
@@ -402,7 +433,7 @@ OverloadCandidate::DeductionFailureInfo::getTemplateArgumentList() {
     case Sema::TDK_Incomplete:
     case Sema::TDK_InvalidExplicitArguments:
     case Sema::TDK_Inconsistent:
-    case Sema::TDK_InconsistentQuals:
+    case Sema::TDK_Underqualified:
       return 0;
 
     case Sema::TDK_SubstitutionFailure:
@@ -429,7 +460,7 @@ const TemplateArgument *OverloadCandidate::DeductionFailureInfo::getFirstArg() {
     return 0;
 
   case Sema::TDK_Inconsistent:
-  case Sema::TDK_InconsistentQuals:
+  case Sema::TDK_Underqualified:
     return &static_cast<DFIParamWithArguments*>(Data)->FirstArg;      
 
   // Unhandled
@@ -454,7 +485,7 @@ OverloadCandidate::DeductionFailureInfo::getSecondArg() {
     return 0;
 
   case Sema::TDK_Inconsistent:
-  case Sema::TDK_InconsistentQuals:
+  case Sema::TDK_Underqualified:
     return &static_cast<DFIParamWithArguments*>(Data)->SecondArg;
 
   // Unhandled
@@ -573,6 +604,11 @@ Sema::CheckOverload(Scope *S, FunctionDecl *New, const LookupResult &Old,
 
 bool Sema::IsOverload(FunctionDecl *New, FunctionDecl *Old,
                       bool UseUsingDeclRules) {
+  // If both of the functions are extern "C", then they are not
+  // overloads.
+  if (Old->isExternC() && New->isExternC())
+    return false;
+
   FunctionTemplateDecl *OldTemplate = Old->getDescribedFunctionTemplate();
   FunctionTemplateDecl *NewTemplate = New->getDescribedFunctionTemplate();
 
@@ -669,40 +705,34 @@ bool Sema::IsOverload(FunctionDecl *New, FunctionDecl *Old,
 /// not permitted.
 /// If @p AllowExplicit, then explicit user-defined conversions are
 /// permitted.
-ImplicitConversionSequence
-Sema::TryImplicitConversion(Expr* From, QualType ToType,
-                            bool SuppressUserConversions,
-                            bool AllowExplicit, 
-                            bool InOverloadResolution) {
+static ImplicitConversionSequence
+TryImplicitConversion(Sema &S, Expr *From, QualType ToType,
+                      bool SuppressUserConversions,
+                      bool AllowExplicit, 
+                      bool InOverloadResolution) {
   ImplicitConversionSequence ICS;
-  if (IsStandardConversion(From, ToType, InOverloadResolution, ICS.Standard)) {
+  if (IsStandardConversion(S, From, ToType, InOverloadResolution,
+                           ICS.Standard)) {
     ICS.setStandard();
     return ICS;
   }
 
-  if (!getLangOptions().CPlusPlus) {
+  if (!S.getLangOptions().CPlusPlus) {
     ICS.setBad(BadConversionSequence::no_conversion, From, ToType);
     return ICS;
   }
 
-  if (SuppressUserConversions) {
-    // C++ [over.ics.user]p4:
-    //   A conversion of an expression of class type to the same class
-    //   type is given Exact Match rank, and a conversion of an
-    //   expression of class type to a base class of that type is
-    //   given Conversion rank, in spite of the fact that a copy/move
-    //   constructor (i.e., a user-defined conversion function) is
-    //   called for those cases.
-    QualType FromType = From->getType();
-    if (!ToType->getAs<RecordType>() || !FromType->getAs<RecordType>() ||
-        !(Context.hasSameUnqualifiedType(FromType, ToType) ||
-          IsDerivedFrom(FromType, ToType))) {
-      // We're not in the case above, so there is no conversion that
-      // we can perform.
-      ICS.setBad(BadConversionSequence::no_conversion, From, ToType);
-      return ICS;
-    }
-
+  // C++ [over.ics.user]p4:
+  //   A conversion of an expression of class type to the same class
+  //   type is given Exact Match rank, and a conversion of an
+  //   expression of class type to a base class of that type is
+  //   given Conversion rank, in spite of the fact that a copy/move
+  //   constructor (i.e., a user-defined conversion function) is
+  //   called for those cases.
+  QualType FromType = From->getType();
+  if (ToType->getAs<RecordType>() && FromType->getAs<RecordType>() &&
+      (S.Context.hasSameUnqualifiedType(FromType, ToType) ||
+       S.IsDerivedFrom(FromType, ToType))) {
     ICS.setStandard();
     ICS.Standard.setAsIdentityConversion();
     ICS.Standard.setFromType(FromType);
@@ -713,18 +743,25 @@ Sema::TryImplicitConversion(Expr* From, QualType ToType,
     // exists. When we actually perform initialization, we'll find the
     // appropriate constructor to copy the returned object, if needed.
     ICS.Standard.CopyConstructor = 0;
-
+    
     // Determine whether this is considered a derived-to-base conversion.
-    if (!Context.hasSameUnqualifiedType(FromType, ToType))
+    if (!S.Context.hasSameUnqualifiedType(FromType, ToType))
       ICS.Standard.Second = ICK_Derived_To_Base;
-
+    
+    return ICS;
+  }
+  
+  if (SuppressUserConversions) {
+    // We're not in the case above, so there is no conversion that
+    // we can perform.
+    ICS.setBad(BadConversionSequence::no_conversion, From, ToType);
     return ICS;
   }
 
   // Attempt user-defined conversion.
   OverloadCandidateSet Conversions(From->getExprLoc());
   OverloadingResult UserDefResult
-    = IsUserDefinedConversion(From, ToType, ICS.UserDefined, Conversions,
+    = IsUserDefinedConversion(S, From, ToType, ICS.UserDefined, Conversions,
                               AllowExplicit);
 
   if (UserDefResult == OR_Success) {
@@ -739,10 +776,11 @@ Sema::TryImplicitConversion(Expr* From, QualType ToType,
     if (CXXConstructorDecl *Constructor
           = dyn_cast<CXXConstructorDecl>(ICS.UserDefined.ConversionFunction)) {
       QualType FromCanon
-        = Context.getCanonicalType(From->getType().getUnqualifiedType());
-      QualType ToCanon = Context.getCanonicalType(ToType).getUnqualifiedType();
+        = S.Context.getCanonicalType(From->getType().getUnqualifiedType());
+      QualType ToCanon
+        = S.Context.getCanonicalType(ToType).getUnqualifiedType();
       if (Constructor->isCopyConstructor() &&
-          (FromCanon == ToCanon || IsDerivedFrom(FromCanon, ToCanon))) {
+          (FromCanon == ToCanon || S.IsDerivedFrom(FromCanon, ToCanon))) {
         // Turn this into a "standard" conversion sequence, so that it
         // gets ranked with standard conversion sequences.
         ICS.setStandard();
@@ -780,6 +818,24 @@ Sema::TryImplicitConversion(Expr* From, QualType ToType,
   return ICS;
 }
 
+bool Sema::TryImplicitConversion(InitializationSequence &Sequence,
+                                 const InitializedEntity &Entity,
+                                 Expr *Initializer,
+                                 bool SuppressUserConversions,
+                                 bool AllowExplicitConversions,
+                                 bool InOverloadResolution) {
+  ImplicitConversionSequence ICS
+    = clang::TryImplicitConversion(*this, Initializer, Entity.getType(),
+                                   SuppressUserConversions,
+                                   AllowExplicitConversions, 
+                                   InOverloadResolution);
+  if (ICS.isBad()) return true;
+
+  // Perform the actual conversion.
+  Sequence.AddConversionSequenceStep(ICS, Entity.getType());
+  return false;
+}
+
 /// PerformImplicitConversion - Perform an implicit conversion of the
 /// expression From to the type ToType. Returns true if there was an
 /// error, false otherwise. The expression From is replaced with the
@@ -797,10 +853,10 @@ bool
 Sema::PerformImplicitConversion(Expr *&From, QualType ToType,
                                 AssignmentAction Action, bool AllowExplicit,
                                 ImplicitConversionSequence& ICS) {
-  ICS = TryImplicitConversion(From, ToType,
-                              /*SuppressUserConversions=*/false,
-                              AllowExplicit,
-                              /*InOverloadResolution=*/false);
+  ICS = clang::TryImplicitConversion(*this, From, ToType,
+                                     /*SuppressUserConversions=*/false,
+                                     AllowExplicit,
+                                     /*InOverloadResolution=*/false);
   return PerformImplicitConversion(From, ToType, ICS, Action);
 }
   
@@ -850,16 +906,20 @@ static bool IsVectorConversion(ASTContext &Context, QualType FromType,
       return true;
     }
   }
-  
-  // If lax vector conversions are permitted and the vector types are of the
-  // same size, we can perform the conversion.
-  if (Context.getLangOptions().LaxVectorConversions &&
-      FromType->isVectorType() && ToType->isVectorType() &&
-      Context.getTypeSize(FromType) == Context.getTypeSize(ToType)) {
-    ICK = ICK_Vector_Conversion;
-    return true;
+
+  // We can perform the conversion between vector types in the following cases:
+  // 1)vector types are equivalent AltiVec and GCC vector types
+  // 2)lax vector conversions are permitted and the vector types are of the
+  //   same size
+  if (ToType->isVectorType() && FromType->isVectorType()) {
+    if (Context.areCompatibleVectorTypes(FromType, ToType) ||
+        (Context.getLangOptions().LaxVectorConversions &&
+         (Context.getTypeSize(FromType) == Context.getTypeSize(ToType)))) {
+      ICK = ICK_Vector_Conversion;
+      return true;
+    }
   }
-  
+
   return false;
 }
   
@@ -871,12 +931,11 @@ static bool IsVectorConversion(ASTContext &Context, QualType FromType,
 /// contain the standard conversion sequence required to perform this
 /// conversion and this routine will return true. Otherwise, this
 /// routine will return false and the value of SCS is unspecified.
-bool
-Sema::IsStandardConversion(Expr* From, QualType ToType,
-                           bool InOverloadResolution,
-                           StandardConversionSequence &SCS) {
+static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
+                                 bool InOverloadResolution,
+                                 StandardConversionSequence &SCS) {
   QualType FromType = From->getType();
-
+  
   // Standard conversions (C++ [conv])
   SCS.setAsIdentityConversion();
   SCS.DeprecatedStringLiteralToCharPtr = false;
@@ -887,7 +946,7 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
   // There are no standard conversions for class types in C++, so
   // abort early. When overloading in C, however, we do permit
   if (FromType->isRecordType() || ToType->isRecordType()) {
-    if (getLangOptions().CPlusPlus)
+    if (S.getLangOptions().CPlusPlus)
       return false;
 
     // When we're overloading in C, we allow, as standard conversions,
@@ -897,19 +956,19 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
   // array-to-pointer conversion, or function-to-pointer conversion
   // (C++ 4p1).
 
-  if (FromType == Context.OverloadTy) {
+  if (FromType == S.Context.OverloadTy) {
     DeclAccessPair AccessPair;
     if (FunctionDecl *Fn
-          = ResolveAddressOfOverloadedFunction(From, ToType, false, 
-                                               AccessPair)) {
+          = S.ResolveAddressOfOverloadedFunction(From, ToType, false, 
+                                                 AccessPair)) {
       // We were able to resolve the address of the overloaded function,
       // so we can convert to the type of that function.
       FromType = Fn->getType();
       if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Fn)) {
         if (!Method->isStatic()) {
           Type *ClassType 
-            = Context.getTypeDeclType(Method->getParent()).getTypePtr();
-          FromType = Context.getMemberPointerType(FromType, ClassType);
+            = S.Context.getTypeDeclType(Method->getParent()).getTypePtr();
+          FromType = S.Context.getMemberPointerType(FromType, ClassType);
         }
       }
       
@@ -917,12 +976,12 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
       // function, update the type of the resulting expression accordingly.
       if (FromType->getAs<FunctionType>())
         if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(From->IgnoreParens()))
-          if (UnOp->getOpcode() == UnaryOperator::AddrOf)
-            FromType = Context.getPointerType(FromType);
+          if (UnOp->getOpcode() == UO_AddrOf)
+            FromType = S.Context.getPointerType(FromType);
  
       // Check that we've computed the proper type after overload resolution.
-      assert(Context.hasSameType(FromType,
-              FixOverloadedFunctionReference(From, AccessPair, Fn)->getType()));
+      assert(S.Context.hasSameType(FromType,
+            S.FixOverloadedFunctionReference(From, AccessPair, Fn)->getType()));
     } else {
       return false;
     }
@@ -930,10 +989,10 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
   // Lvalue-to-rvalue conversion (C++ 4.1):
   //   An lvalue (3.10) of a non-function, non-array type T can be
   //   converted to an rvalue.
-  Expr::isLvalueResult argIsLvalue = From->isLvalue(Context);
+  Expr::isLvalueResult argIsLvalue = From->isLvalue(S.Context);
   if (argIsLvalue == Expr::LV_Valid &&
       !FromType->isFunctionType() && !FromType->isArrayType() &&
-      Context.getCanonicalType(FromType) != Context.OverloadTy) {
+      S.Context.getCanonicalType(FromType) != S.Context.OverloadTy) {
     SCS.First = ICK_Lvalue_To_Rvalue;
 
     // If T is a non-class type, the type of the rvalue is the
@@ -948,9 +1007,9 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
     // An lvalue or rvalue of type "array of N T" or "array of unknown
     // bound of T" can be converted to an rvalue of type "pointer to
     // T" (C++ 4.2p1).
-    FromType = Context.getArrayDecayedType(FromType);
+    FromType = S.Context.getArrayDecayedType(FromType);
 
-    if (IsStringLiteralToNonConstPointerConversion(From, ToType)) {
+    if (S.IsStringLiteralToNonConstPointerConversion(From, ToType)) {
       // This conversion is deprecated. (C++ D.4).
       SCS.DeprecatedStringLiteralToCharPtr = true;
 
@@ -970,7 +1029,7 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
     // An lvalue of function type T can be converted to an rvalue of
     // type "pointer to T." The result is a pointer to the
     // function. (C++ 4.3p1).
-    FromType = Context.getPointerType(FromType);
+    FromType = S.Context.getPointerType(FromType);
   } else {
     // We don't require any conversions for the first step.
     SCS.First = ICK_Identity;
@@ -985,24 +1044,24 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
   // conversion.
   bool IncompatibleObjC = false;
   ImplicitConversionKind SecondICK = ICK_Identity;
-  if (Context.hasSameUnqualifiedType(FromType, ToType)) {
+  if (S.Context.hasSameUnqualifiedType(FromType, ToType)) {
     // The unqualified versions of the types are the same: there's no
     // conversion to do.
     SCS.Second = ICK_Identity;
-  } else if (IsIntegralPromotion(From, FromType, ToType)) {
+  } else if (S.IsIntegralPromotion(From, FromType, ToType)) {
     // Integral promotion (C++ 4.5).
     SCS.Second = ICK_Integral_Promotion;
     FromType = ToType.getUnqualifiedType();
-  } else if (IsFloatingPointPromotion(FromType, ToType)) {
+  } else if (S.IsFloatingPointPromotion(FromType, ToType)) {
     // Floating point promotion (C++ 4.6).
     SCS.Second = ICK_Floating_Promotion;
     FromType = ToType.getUnqualifiedType();
-  } else if (IsComplexPromotion(FromType, ToType)) {
+  } else if (S.IsComplexPromotion(FromType, ToType)) {
     // Complex promotion (Clang extension)
     SCS.Second = ICK_Complex_Promotion;
     FromType = ToType.getUnqualifiedType();
   } else if (FromType->isIntegralOrEnumerationType() &&
-             ToType->isIntegralType(Context)) {
+             ToType->isIntegralType(S.Context)) {
     // Integral conversions (C++ 4.7).
     SCS.Second = ICK_Integral_Conversion;
     FromType = ToType.getUnqualifiedType();
@@ -1020,19 +1079,19 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
     SCS.Second = ICK_Floating_Conversion;
     FromType = ToType.getUnqualifiedType();
   } else if ((FromType->isRealFloatingType() && 
-              ToType->isIntegralType(Context) && !ToType->isBooleanType()) ||
+              ToType->isIntegralType(S.Context) && !ToType->isBooleanType()) ||
              (FromType->isIntegralOrEnumerationType() &&
               ToType->isRealFloatingType())) {
     // Floating-integral conversions (C++ 4.9).
     SCS.Second = ICK_Floating_Integral;
     FromType = ToType.getUnqualifiedType();
-  } else if (IsPointerConversion(From, FromType, ToType, InOverloadResolution,
-                                 FromType, IncompatibleObjC)) {
+  } else if (S.IsPointerConversion(From, FromType, ToType, InOverloadResolution,
+                                   FromType, IncompatibleObjC)) {
     // Pointer conversions (C++ 4.10).
     SCS.Second = ICK_Pointer_Conversion;
     SCS.IncompatibleObjC = IncompatibleObjC;
-  } else if (IsMemberPointerConversion(From, FromType, ToType, 
-                                       InOverloadResolution, FromType)) {
+  } else if (S.IsMemberPointerConversion(From, FromType, ToType, 
+                                         InOverloadResolution, FromType)) {
     // Pointer to member conversions (4.11).
     SCS.Second = ICK_Pointer_Member;
   } else if (ToType->isBooleanType() &&
@@ -1044,16 +1103,16 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
               FromType->isNullPtrType())) {
     // Boolean conversions (C++ 4.12).
     SCS.Second = ICK_Boolean_Conversion;
-    FromType = Context.BoolTy;
-  } else if (IsVectorConversion(Context, FromType, ToType, SecondICK)) {
+    FromType = S.Context.BoolTy;
+  } else if (IsVectorConversion(S.Context, FromType, ToType, SecondICK)) {
     SCS.Second = SecondICK;
     FromType = ToType.getUnqualifiedType();
-  } else if (!getLangOptions().CPlusPlus &&
-             Context.typesAreCompatible(ToType, FromType)) {
+  } else if (!S.getLangOptions().CPlusPlus &&
+             S.Context.typesAreCompatible(ToType, FromType)) {
     // Compatible conversions (Clang extension for C function overloading)
     SCS.Second = ICK_Compatible_Conversion;
     FromType = ToType.getUnqualifiedType();
-  } else if (IsNoReturnConversion(Context, FromType, ToType, FromType)) {
+  } else if (IsNoReturnConversion(S.Context, FromType, ToType, FromType)) {
     // Treat a conversion that strips "noreturn" as an identity conversion.
     SCS.Second = ICK_NoReturn_Adjustment;
   } else {
@@ -1065,11 +1124,11 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
   QualType CanonFrom;
   QualType CanonTo;
   // The third conversion can be a qualification conversion (C++ 4p1).
-  if (IsQualificationConversion(FromType, ToType)) {
+  if (S.IsQualificationConversion(FromType, ToType)) {
     SCS.Third = ICK_Qualification;
     FromType = ToType;
-    CanonFrom = Context.getCanonicalType(FromType);
-    CanonTo = Context.getCanonicalType(ToType);
+    CanonFrom = S.Context.getCanonicalType(FromType);
+    CanonTo = S.Context.getCanonicalType(ToType);
   } else {
     // No conversion required
     SCS.Third = ICK_Identity;
@@ -1078,8 +1137,8 @@ Sema::IsStandardConversion(Expr* From, QualType ToType,
     //   [...] Any difference in top-level cv-qualification is
     //   subsumed by the initialization itself and does not constitute
     //   a conversion. [...]
-    CanonFrom = Context.getCanonicalType(FromType);
-    CanonTo = Context.getCanonicalType(ToType);
+    CanonFrom = S.Context.getCanonicalType(FromType);
+    CanonTo = S.Context.getCanonicalType(ToType);
     if (CanonFrom.getLocalUnqualifiedType() 
                                        == CanonTo.getLocalUnqualifiedType() &&
         (CanonFrom.getLocalCVRQualifiers() != CanonTo.getLocalCVRQualifiers()
@@ -1397,10 +1456,16 @@ bool Sema::IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
 
   QualType FromPointeeType = FromTypePtr->getPointeeType();
 
+  // If the unqualified pointee types are the same, this can't be a 
+  // pointer conversion, so don't do all of the work below.
+  if (Context.hasSameUnqualifiedType(FromPointeeType, ToPointeeType))
+    return false;
+
   // An rvalue of type "pointer to cv T," where T is an object type,
   // can be converted to an rvalue of type "pointer to cv void" (C++
   // 4.10p2).
-  if (FromPointeeType->isObjectType() && ToPointeeType->isVoidType()) {
+  if (FromPointeeType->isIncompleteOrObjectType() &&
+      ToPointeeType->isVoidType()) {
     ConvertedType = BuildSimilarlyQualifiedPointerType(FromTypePtr,
                                                        ToPointeeType,
                                                        ToType, Context);
@@ -1657,8 +1722,8 @@ bool Sema::FunctionArgTypesAreEqual(FunctionProtoType*  OldType,
 /// true. It returns true and produces a diagnostic if there was an
 /// error, or returns false otherwise.
 bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
-                                  CastExpr::CastKind &Kind,
-                                  CXXBaseSpecifierArray& BasePath,
+                                  CastKind &Kind,
+                                  CXXCastPath& BasePath,
                                   bool IgnoreBaseAccess) {
   QualType FromType = From->getType();
 
@@ -1684,7 +1749,7 @@ bool Sema::CheckPointerConversion(Expr *From, QualType ToType,
           return true;
         
         // The conversion was successful.
-        Kind = CastExpr::CK_DerivedToBase;
+        Kind = CK_DerivedToBase;
       }
     }
   if (const ObjCObjectPointerType *FromPtrType =
@@ -1749,8 +1814,8 @@ bool Sema::IsMemberPointerConversion(Expr *From, QualType FromType,
 /// true and produces a diagnostic if there was an error, or returns false
 /// otherwise.
 bool Sema::CheckMemberPointerConversion(Expr *From, QualType ToType,
-                                        CastExpr::CastKind &Kind,
-                                        CXXBaseSpecifierArray &BasePath,
+                                        CastKind &Kind,
+                                        CXXCastPath &BasePath,
                                         bool IgnoreBaseAccess) {
   QualType FromType = From->getType();
   const MemberPointerType *FromPtrType = FromType->getAs<MemberPointerType>();
@@ -1759,7 +1824,7 @@ bool Sema::CheckMemberPointerConversion(Expr *From, QualType ToType,
     assert(From->isNullPointerConstant(Context, 
                                        Expr::NPC_ValueDependentIsNull) &&
            "Expr must be null pointer constant!");
-    Kind = CastExpr::CK_NullToMemberPointer;
+    Kind = CK_NullToMemberPointer;
     return false;
   }
 
@@ -1803,7 +1868,7 @@ bool Sema::CheckMemberPointerConversion(Expr *From, QualType ToType,
 
   // Must be a base to derived member conversion.
   BuildBasePathArray(Paths, BasePath);
-  Kind = CastExpr::CK_BaseToDerivedMemberPointer;
+  Kind = CK_BaseToDerivedMemberPointer;
   return false;
 }
 
@@ -1869,10 +1934,11 @@ Sema::IsQualificationConversion(QualType FromType, QualType ToType) {
 /// \param AllowExplicit  true if the conversion should consider C++0x
 /// "explicit" conversion functions as well as non-explicit conversion
 /// functions (C++0x [class.conv.fct]p2).
-OverloadingResult Sema::IsUserDefinedConversion(Expr *From, QualType ToType,
-                                          UserDefinedConversionSequence& User,
-                                           OverloadCandidateSet& CandidateSet,
-                                                bool AllowExplicit) {
+static OverloadingResult
+IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
+                        UserDefinedConversionSequence& User,
+                        OverloadCandidateSet& CandidateSet,
+                        bool AllowExplicit) {
   // Whether we will only visit constructors.
   bool ConstructorsOnly = false;
 
@@ -1887,17 +1953,17 @@ OverloadingResult Sema::IsUserDefinedConversion(Expr *From, QualType ToType,
     //   functions are all the converting constructors (12.3.1) of
     //   that class. The argument list is the expression-list within
     //   the parentheses of the initializer.
-    if (Context.hasSameUnqualifiedType(ToType, From->getType()) ||
+    if (S.Context.hasSameUnqualifiedType(ToType, From->getType()) ||
         (From->getType()->getAs<RecordType>() &&
-         IsDerivedFrom(From->getType(), ToType)))
+         S.IsDerivedFrom(From->getType(), ToType)))
       ConstructorsOnly = true;
 
-    if (RequireCompleteType(From->getLocStart(), ToType, PDiag())) {
+    if (S.RequireCompleteType(From->getLocStart(), ToType, S.PDiag())) {
       // We're not going to find any constructors.
     } else if (CXXRecordDecl *ToRecordDecl
                  = dyn_cast<CXXRecordDecl>(ToRecordType->getDecl())) {
       DeclContext::lookup_iterator Con, ConEnd;
-      for (llvm::tie(Con, ConEnd) = LookupConstructors(ToRecordDecl);
+      for (llvm::tie(Con, ConEnd) = S.LookupConstructors(ToRecordDecl);
            Con != ConEnd; ++Con) {
         NamedDecl *D = *Con;
         DeclAccessPair FoundDecl = DeclAccessPair::make(D, D->getAccess());
@@ -1915,16 +1981,18 @@ OverloadingResult Sema::IsUserDefinedConversion(Expr *From, QualType ToType,
         if (!Constructor->isInvalidDecl() &&
             Constructor->isConvertingConstructor(AllowExplicit)) {
           if (ConstructorTmpl)
-            AddTemplateOverloadCandidate(ConstructorTmpl, FoundDecl,
-                                         /*ExplicitArgs*/ 0,
-                                         &From, 1, CandidateSet, 
-                                 /*SuppressUserConversions=*/!ConstructorsOnly);
+            S.AddTemplateOverloadCandidate(ConstructorTmpl, FoundDecl,
+                                           /*ExplicitArgs*/ 0,
+                                           &From, 1, CandidateSet, 
+                                           /*SuppressUserConversions=*/
+                                             !ConstructorsOnly);
           else
             // Allow one user-defined conversion when user specifies a
             // From->ToType conversion via an static cast (c-style, etc).
-            AddOverloadCandidate(Constructor, FoundDecl,
-                                 &From, 1, CandidateSet,
-                                 /*SuppressUserConversions=*/!ConstructorsOnly);
+            S.AddOverloadCandidate(Constructor, FoundDecl,
+                                   &From, 1, CandidateSet,
+                                   /*SuppressUserConversions=*/
+                                     !ConstructorsOnly);
         }
       }
     }
@@ -1932,8 +2000,8 @@ OverloadingResult Sema::IsUserDefinedConversion(Expr *From, QualType ToType,
 
   // Enumerate conversion functions, if we're allowed to.
   if (ConstructorsOnly) {
-  } else if (RequireCompleteType(From->getLocStart(), From->getType(),
-                          PDiag(0) << From->getSourceRange())) {
+  } else if (S.RequireCompleteType(From->getLocStart(), From->getType(),
+                                   S.PDiag(0) << From->getSourceRange())) {
     // No conversion functions from incomplete types.
   } else if (const RecordType *FromRecordType
                                    = From->getType()->getAs<RecordType>()) {
@@ -1959,80 +2027,79 @@ OverloadingResult Sema::IsUserDefinedConversion(Expr *From, QualType ToType,
 
         if (AllowExplicit || !Conv->isExplicit()) {
           if (ConvTemplate)
-            AddTemplateConversionCandidate(ConvTemplate, FoundDecl,
-                                           ActingContext, From, ToType,
-                                           CandidateSet);
+            S.AddTemplateConversionCandidate(ConvTemplate, FoundDecl,
+                                             ActingContext, From, ToType,
+                                             CandidateSet);
           else
-            AddConversionCandidate(Conv, FoundDecl, ActingContext,
-                                   From, ToType, CandidateSet);
+            S.AddConversionCandidate(Conv, FoundDecl, ActingContext,
+                                     From, ToType, CandidateSet);
         }
       }
     }
   }
 
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, From->getLocStart(), Best)) {
-    case OR_Success:
-      // Record the standard conversion we used and the conversion function.
-      if (CXXConstructorDecl *Constructor
-            = dyn_cast<CXXConstructorDecl>(Best->Function)) {
-        // C++ [over.ics.user]p1:
-        //   If the user-defined conversion is specified by a
-        //   constructor (12.3.1), the initial standard conversion
-        //   sequence converts the source type to the type required by
-        //   the argument of the constructor.
-        //
-        QualType ThisType = Constructor->getThisType(Context);
-        if (Best->Conversions[0].isEllipsis())
-          User.EllipsisConversion = true;
-        else {
-          User.Before = Best->Conversions[0].Standard;
-          User.EllipsisConversion = false;
-        }
-        User.ConversionFunction = Constructor;
-        User.After.setAsIdentityConversion();
-        User.After.setFromType(
-          ThisType->getAs<PointerType>()->getPointeeType());
-        User.After.setAllToTypes(ToType);
-        return OR_Success;
-      } else if (CXXConversionDecl *Conversion
-                   = dyn_cast<CXXConversionDecl>(Best->Function)) {
-        // C++ [over.ics.user]p1:
-        //
-        //   [...] If the user-defined conversion is specified by a
-        //   conversion function (12.3.2), the initial standard
-        //   conversion sequence converts the source type to the
-        //   implicit object parameter of the conversion function.
+  switch (CandidateSet.BestViableFunction(S, From->getLocStart(), Best)) {
+  case OR_Success:
+    // Record the standard conversion we used and the conversion function.
+    if (CXXConstructorDecl *Constructor
+          = dyn_cast<CXXConstructorDecl>(Best->Function)) {
+      // C++ [over.ics.user]p1:
+      //   If the user-defined conversion is specified by a
+      //   constructor (12.3.1), the initial standard conversion
+      //   sequence converts the source type to the type required by
+      //   the argument of the constructor.
+      //
+      QualType ThisType = Constructor->getThisType(S.Context);
+      if (Best->Conversions[0].isEllipsis())
+        User.EllipsisConversion = true;
+      else {
         User.Before = Best->Conversions[0].Standard;
-        User.ConversionFunction = Conversion;
         User.EllipsisConversion = false;
-
-        // C++ [over.ics.user]p2:
-        //   The second standard conversion sequence converts the
-        //   result of the user-defined conversion to the target type
-        //   for the sequence. Since an implicit conversion sequence
-        //   is an initialization, the special rules for
-        //   initialization by user-defined conversion apply when
-        //   selecting the best user-defined conversion for a
-        //   user-defined conversion sequence (see 13.3.3 and
-        //   13.3.3.1).
-        User.After = Best->FinalConversion;
-        return OR_Success;
-      } else {
-        assert(false && "Not a constructor or conversion function?");
-        return OR_No_Viable_Function;
       }
-
-    case OR_No_Viable_Function:
+      User.ConversionFunction = Constructor;
+      User.After.setAsIdentityConversion();
+      User.After.setFromType(ThisType->getAs<PointerType>()->getPointeeType());
+      User.After.setAllToTypes(ToType);
+      return OR_Success;
+    } else if (CXXConversionDecl *Conversion
+                 = dyn_cast<CXXConversionDecl>(Best->Function)) {
+      // C++ [over.ics.user]p1:
+      //
+      //   [...] If the user-defined conversion is specified by a
+      //   conversion function (12.3.2), the initial standard
+      //   conversion sequence converts the source type to the
+      //   implicit object parameter of the conversion function.
+      User.Before = Best->Conversions[0].Standard;
+      User.ConversionFunction = Conversion;
+      User.EllipsisConversion = false;
+
+      // C++ [over.ics.user]p2:
+      //   The second standard conversion sequence converts the
+      //   result of the user-defined conversion to the target type
+      //   for the sequence. Since an implicit conversion sequence
+      //   is an initialization, the special rules for
+      //   initialization by user-defined conversion apply when
+      //   selecting the best user-defined conversion for a
+      //   user-defined conversion sequence (see 13.3.3 and
+      //   13.3.3.1).
+      User.After = Best->FinalConversion;
+      return OR_Success;
+    } else {
+      llvm_unreachable("Not a constructor or conversion function?");
       return OR_No_Viable_Function;
-    case OR_Deleted:
-      // No conversion here! We're done.
-      return OR_Deleted;
-
-    case OR_Ambiguous:
-      return OR_Ambiguous;
     }
 
+  case OR_No_Viable_Function:
+    return OR_No_Viable_Function;
+  case OR_Deleted:
+    // No conversion here! We're done.
+    return OR_Deleted;
+    
+  case OR_Ambiguous:
+    return OR_Ambiguous;
+  }
+
   return OR_No_Viable_Function;
 }
   
@@ -2041,7 +2108,7 @@ Sema::DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType) {
   ImplicitConversionSequence ICS;
   OverloadCandidateSet CandidateSet(From->getExprLoc());
   OverloadingResult OvResult = 
-    IsUserDefinedConversion(From, ToType, ICS.UserDefined,
+    IsUserDefinedConversion(*this, From, ToType, ICS.UserDefined,
                             CandidateSet, false);
   if (OvResult == OR_Ambiguous)
     Diag(From->getSourceRange().getBegin(),
@@ -2053,16 +2120,17 @@ Sema::DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType) {
     << From->getType() << ToType << From->getSourceRange();
   else
     return false;
-  PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, &From, 1);
+  CandidateSet.NoteCandidates(*this, OCD_AllCandidates, &From, 1);
   return true;  
 }
 
 /// CompareImplicitConversionSequences - Compare two implicit
 /// conversion sequences to determine whether one is better than the
 /// other or if they are indistinguishable (C++ 13.3.3.2).
-ImplicitConversionSequence::CompareKind
-Sema::CompareImplicitConversionSequences(const ImplicitConversionSequence& ICS1,
-                                         const ImplicitConversionSequence& ICS2)
+static ImplicitConversionSequence::CompareKind
+CompareImplicitConversionSequences(Sema &S,
+                                   const ImplicitConversionSequence& ICS1,
+                                   const ImplicitConversionSequence& ICS2)
 {
   // (C++ 13.3.3.2p2): When comparing the basic forms of implicit
   // conversion sequences (as defined in 13.3.3.1)
@@ -2092,7 +2160,7 @@ Sema::CompareImplicitConversionSequences(const ImplicitConversionSequence& ICS1,
   // indistinguishable conversion sequences unless one of the
   // following rules apply: (C++ 13.3.3.2p3):
   if (ICS1.isStandard())
-    return CompareStandardConversionSequences(ICS1.Standard, ICS2.Standard);
+    return CompareStandardConversionSequences(S, ICS1.Standard, ICS2.Standard);
   else if (ICS1.isUserDefined()) {
     // User-defined conversion sequence U1 is a better conversion
     // sequence than another user-defined conversion sequence U2 if
@@ -2102,7 +2170,8 @@ Sema::CompareImplicitConversionSequences(const ImplicitConversionSequence& ICS1,
     // U2 (C++ 13.3.3.2p3).
     if (ICS1.UserDefined.ConversionFunction ==
           ICS2.UserDefined.ConversionFunction)
-      return CompareStandardConversionSequences(ICS1.UserDefined.After,
+      return CompareStandardConversionSequences(S,
+                                                ICS1.UserDefined.After,
                                                 ICS2.UserDefined.After);
   }
 
@@ -2168,9 +2237,10 @@ compareStandardConversionSubsets(ASTContext &Context,
 /// CompareStandardConversionSequences - Compare two standard
 /// conversion sequences to determine whether one is better than the
 /// other or if they are indistinguishable (C++ 13.3.3.2p3).
-ImplicitConversionSequence::CompareKind
-Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
-                                         const StandardConversionSequence& SCS2)
+static ImplicitConversionSequence::CompareKind
+CompareStandardConversionSequences(Sema &S,
+                                   const StandardConversionSequence& SCS1,
+                                   const StandardConversionSequence& SCS2)
 {
   // Standard conversion sequence S1 is a better conversion sequence
   // than standard conversion sequence S2 if (C++ 13.3.3.2p3):
@@ -2181,7 +2251,7 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
   //     sequence is considered to be a subsequence of any
   //     non-identity conversion sequence) or, if not that,
   if (ImplicitConversionSequence::CompareKind CK
-        = compareStandardConversionSubsets(Context, SCS1, SCS2))
+        = compareStandardConversionSubsets(S.Context, SCS1, SCS2))
     return CK;
 
   //  -- the rank of S1 is better than the rank of S2 (by the rules
@@ -2212,9 +2282,9 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
   //   void*, and conversion of A* to void* is better than conversion
   //   of B* to void*.
   bool SCS1ConvertsToVoid
-    = SCS1.isPointerConversionToVoidPointer(Context);
+    = SCS1.isPointerConversionToVoidPointer(S.Context);
   bool SCS2ConvertsToVoid
-    = SCS2.isPointerConversionToVoidPointer(Context);
+    = SCS2.isPointerConversionToVoidPointer(S.Context);
   if (SCS1ConvertsToVoid != SCS2ConvertsToVoid) {
     // Exactly one of the conversion sequences is a conversion to
     // a void pointer; it's the worse conversion.
@@ -2224,7 +2294,7 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
     // Neither conversion sequence converts to a void pointer; compare
     // their derived-to-base conversions.
     if (ImplicitConversionSequence::CompareKind DerivedCK
-          = CompareDerivedToBaseConversions(SCS1, SCS2))
+          = CompareDerivedToBaseConversions(S, SCS1, SCS2))
       return DerivedCK;
   } else if (SCS1ConvertsToVoid && SCS2ConvertsToVoid) {
     // Both conversion sequences are conversions to void
@@ -2236,18 +2306,18 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
     // Adjust the types we're converting from via the array-to-pointer
     // conversion, if we need to.
     if (SCS1.First == ICK_Array_To_Pointer)
-      FromType1 = Context.getArrayDecayedType(FromType1);
+      FromType1 = S.Context.getArrayDecayedType(FromType1);
     if (SCS2.First == ICK_Array_To_Pointer)
-      FromType2 = Context.getArrayDecayedType(FromType2);
+      FromType2 = S.Context.getArrayDecayedType(FromType2);
 
     QualType FromPointee1
       = FromType1->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
     QualType FromPointee2
       = FromType2->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
 
-    if (IsDerivedFrom(FromPointee2, FromPointee1))
+    if (S.IsDerivedFrom(FromPointee2, FromPointee1))
       return ImplicitConversionSequence::Better;
-    else if (IsDerivedFrom(FromPointee1, FromPointee2))
+    else if (S.IsDerivedFrom(FromPointee1, FromPointee2))
       return ImplicitConversionSequence::Worse;
 
     // Objective-C++: If one interface is more specific than the
@@ -2255,9 +2325,9 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
     const ObjCObjectType* FromIface1 = FromPointee1->getAs<ObjCObjectType>();
     const ObjCObjectType* FromIface2 = FromPointee2->getAs<ObjCObjectType>();
     if (FromIface1 && FromIface1) {
-      if (Context.canAssignObjCInterfaces(FromIface2, FromIface1))
+      if (S.Context.canAssignObjCInterfaces(FromIface2, FromIface1))
         return ImplicitConversionSequence::Better;
-      else if (Context.canAssignObjCInterfaces(FromIface1, FromIface2))
+      else if (S.Context.canAssignObjCInterfaces(FromIface1, FromIface2))
         return ImplicitConversionSequence::Worse;
     }
   }
@@ -2265,7 +2335,7 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
   // Compare based on qualification conversions (C++ 13.3.3.2p3,
   // bullet 3).
   if (ImplicitConversionSequence::CompareKind QualCK
-        = CompareQualificationConversions(SCS1, SCS2))
+        = CompareQualificationConversions(S, SCS1, SCS2))
     return QualCK;
 
   if (SCS1.ReferenceBinding && SCS2.ReferenceBinding) {
@@ -2289,18 +2359,18 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
     //      to which the reference initialized by S1 refers.
     QualType T1 = SCS1.getToType(2);
     QualType T2 = SCS2.getToType(2);
-    T1 = Context.getCanonicalType(T1);
-    T2 = Context.getCanonicalType(T2);
+    T1 = S.Context.getCanonicalType(T1);
+    T2 = S.Context.getCanonicalType(T2);
     Qualifiers T1Quals, T2Quals;
-    QualType UnqualT1 = Context.getUnqualifiedArrayType(T1, T1Quals);
-    QualType UnqualT2 = Context.getUnqualifiedArrayType(T2, T2Quals);
+    QualType UnqualT1 = S.Context.getUnqualifiedArrayType(T1, T1Quals);
+    QualType UnqualT2 = S.Context.getUnqualifiedArrayType(T2, T2Quals);
     if (UnqualT1 == UnqualT2) {
       // If the type is an array type, promote the element qualifiers to the type
       // for comparison.
       if (isa<ArrayType>(T1) && T1Quals)
-        T1 = Context.getQualifiedType(UnqualT1, T1Quals);
+        T1 = S.Context.getQualifiedType(UnqualT1, T1Quals);
       if (isa<ArrayType>(T2) && T2Quals)
-        T2 = Context.getQualifiedType(UnqualT2, T2Quals);
+        T2 = S.Context.getQualifiedType(UnqualT2, T2Quals);
       if (T2.isMoreQualifiedThan(T1))
         return ImplicitConversionSequence::Better;
       else if (T1.isMoreQualifiedThan(T2))
@@ -2315,8 +2385,9 @@ Sema::CompareStandardConversionSequences(const StandardConversionSequence& SCS1,
 /// sequences to determine whether they can be ranked based on their
 /// qualification conversions (C++ 13.3.3.2p3 bullet 3).
 ImplicitConversionSequence::CompareKind
-Sema::CompareQualificationConversions(const StandardConversionSequence& SCS1,
-                                      const StandardConversionSequence& SCS2) {
+CompareQualificationConversions(Sema &S,
+                                const StandardConversionSequence& SCS1,
+                                const StandardConversionSequence& SCS2) {
   // C++ 13.3.3.2p3:
   //  -- S1 and S2 differ only in their qualification conversion and
   //     yield similar types T1 and T2 (C++ 4.4), respectively, and the
@@ -2331,11 +2402,11 @@ Sema::CompareQualificationConversions(const StandardConversionSequence& SCS1,
   // conversion (!)
   QualType T1 = SCS1.getToType(2);
   QualType T2 = SCS2.getToType(2);
-  T1 = Context.getCanonicalType(T1);
-  T2 = Context.getCanonicalType(T2);
+  T1 = S.Context.getCanonicalType(T1);
+  T2 = S.Context.getCanonicalType(T2);
   Qualifiers T1Quals, T2Quals;
-  QualType UnqualT1 = Context.getUnqualifiedArrayType(T1, T1Quals);
-  QualType UnqualT2 = Context.getUnqualifiedArrayType(T2, T2Quals);
+  QualType UnqualT1 = S.Context.getUnqualifiedArrayType(T1, T1Quals);
+  QualType UnqualT2 = S.Context.getUnqualifiedArrayType(T2, T2Quals);
 
   // If the types are the same, we won't learn anything by unwrapped
   // them.
@@ -2345,13 +2416,13 @@ Sema::CompareQualificationConversions(const StandardConversionSequence& SCS1,
   // If the type is an array type, promote the element qualifiers to the type
   // for comparison.
   if (isa<ArrayType>(T1) && T1Quals)
-    T1 = Context.getQualifiedType(UnqualT1, T1Quals);
+    T1 = S.Context.getQualifiedType(UnqualT1, T1Quals);
   if (isa<ArrayType>(T2) && T2Quals)
-    T2 = Context.getQualifiedType(UnqualT2, T2Quals);
+    T2 = S.Context.getQualifiedType(UnqualT2, T2Quals);
 
   ImplicitConversionSequence::CompareKind Result
     = ImplicitConversionSequence::Indistinguishable;
-  while (Context.UnwrapSimilarPointerTypes(T1, T2)) {
+  while (S.Context.UnwrapSimilarPointerTypes(T1, T2)) {
     // Within each iteration of the loop, we check the qualifiers to
     // determine if this still looks like a qualification
     // conversion. Then, if all is well, we unwrap one more level of
@@ -2386,7 +2457,7 @@ Sema::CompareQualificationConversions(const StandardConversionSequence& SCS1,
     }
 
     // If the types after this point are equivalent, we're done.
-    if (Context.hasSameUnqualifiedType(T1, T2))
+    if (S.Context.hasSameUnqualifiedType(T1, T2))
       break;
   }
 
@@ -2416,8 +2487,9 @@ Sema::CompareQualificationConversions(const StandardConversionSequence& SCS1,
 /// [over.ics.rank]p4b3).  As part of these checks, we also look at
 /// conversions between Objective-C interface types.
 ImplicitConversionSequence::CompareKind
-Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
-                                      const StandardConversionSequence& SCS2) {
+CompareDerivedToBaseConversions(Sema &S,
+                                const StandardConversionSequence& SCS1,
+                                const StandardConversionSequence& SCS2) {
   QualType FromType1 = SCS1.getFromType();
   QualType ToType1 = SCS1.getToType(1);
   QualType FromType2 = SCS2.getFromType();
@@ -2426,15 +2498,15 @@ Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
   // Adjust the types we're converting from via the array-to-pointer
   // conversion, if we need to.
   if (SCS1.First == ICK_Array_To_Pointer)
-    FromType1 = Context.getArrayDecayedType(FromType1);
+    FromType1 = S.Context.getArrayDecayedType(FromType1);
   if (SCS2.First == ICK_Array_To_Pointer)
-    FromType2 = Context.getArrayDecayedType(FromType2);
+    FromType2 = S.Context.getArrayDecayedType(FromType2);
 
   // Canonicalize all of the types.
-  FromType1 = Context.getCanonicalType(FromType1);
-  ToType1 = Context.getCanonicalType(ToType1);
-  FromType2 = Context.getCanonicalType(FromType2);
-  ToType2 = Context.getCanonicalType(ToType2);
+  FromType1 = S.Context.getCanonicalType(FromType1);
+  ToType1 = S.Context.getCanonicalType(ToType1);
+  FromType2 = S.Context.getCanonicalType(FromType2);
+  ToType2 = S.Context.getCanonicalType(ToType2);
 
   // C++ [over.ics.rank]p4b3:
   //
@@ -2466,30 +2538,30 @@ Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
 
     //   -- conversion of C* to B* is better than conversion of C* to A*,
     if (FromPointee1 == FromPointee2 && ToPointee1 != ToPointee2) {
-      if (IsDerivedFrom(ToPointee1, ToPointee2))
+      if (S.IsDerivedFrom(ToPointee1, ToPointee2))
         return ImplicitConversionSequence::Better;
-      else if (IsDerivedFrom(ToPointee2, ToPointee1))
+      else if (S.IsDerivedFrom(ToPointee2, ToPointee1))
         return ImplicitConversionSequence::Worse;
 
       if (ToIface1 && ToIface2) {
-        if (Context.canAssignObjCInterfaces(ToIface2, ToIface1))
+        if (S.Context.canAssignObjCInterfaces(ToIface2, ToIface1))
           return ImplicitConversionSequence::Better;
-        else if (Context.canAssignObjCInterfaces(ToIface1, ToIface2))
+        else if (S.Context.canAssignObjCInterfaces(ToIface1, ToIface2))
           return ImplicitConversionSequence::Worse;
       }
     }
 
     //   -- conversion of B* to A* is better than conversion of C* to A*,
     if (FromPointee1 != FromPointee2 && ToPointee1 == ToPointee2) {
-      if (IsDerivedFrom(FromPointee2, FromPointee1))
+      if (S.IsDerivedFrom(FromPointee2, FromPointee1))
         return ImplicitConversionSequence::Better;
-      else if (IsDerivedFrom(FromPointee1, FromPointee2))
+      else if (S.IsDerivedFrom(FromPointee1, FromPointee2))
         return ImplicitConversionSequence::Worse;
 
       if (FromIface1 && FromIface2) {
-        if (Context.canAssignObjCInterfaces(FromIface1, FromIface2))
+        if (S.Context.canAssignObjCInterfaces(FromIface1, FromIface2))
           return ImplicitConversionSequence::Better;
-        else if (Context.canAssignObjCInterfaces(FromIface2, FromIface1))
+        else if (S.Context.canAssignObjCInterfaces(FromIface2, FromIface1))
           return ImplicitConversionSequence::Worse;
       }
     }
@@ -2517,16 +2589,16 @@ Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
     QualType ToPointee2 = QualType(ToPointeeType2, 0).getUnqualifiedType();
     // conversion of A::* to B::* is better than conversion of A::* to C::*,
     if (FromPointee1 == FromPointee2 && ToPointee1 != ToPointee2) {
-      if (IsDerivedFrom(ToPointee1, ToPointee2))
+      if (S.IsDerivedFrom(ToPointee1, ToPointee2))
         return ImplicitConversionSequence::Worse;
-      else if (IsDerivedFrom(ToPointee2, ToPointee1))
+      else if (S.IsDerivedFrom(ToPointee2, ToPointee1))
         return ImplicitConversionSequence::Better;
     }
     // conversion of B::* to C::* is better than conversion of A::* to C::*
     if (ToPointee1 == ToPointee2 && FromPointee1 != FromPointee2) {
-      if (IsDerivedFrom(FromPointee1, FromPointee2))
+      if (S.IsDerivedFrom(FromPointee1, FromPointee2))
         return ImplicitConversionSequence::Better;
-      else if (IsDerivedFrom(FromPointee2, FromPointee1))
+      else if (S.IsDerivedFrom(FromPointee2, FromPointee1))
         return ImplicitConversionSequence::Worse;
     }
   }
@@ -2536,11 +2608,11 @@ Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
     //   -- binding of an expression of type C to a reference of type
     //      B& is better than binding an expression of type C to a
     //      reference of type A&,
-    if (Context.hasSameUnqualifiedType(FromType1, FromType2) &&
-        !Context.hasSameUnqualifiedType(ToType1, ToType2)) {
-      if (IsDerivedFrom(ToType1, ToType2))
+    if (S.Context.hasSameUnqualifiedType(FromType1, FromType2) &&
+        !S.Context.hasSameUnqualifiedType(ToType1, ToType2)) {
+      if (S.IsDerivedFrom(ToType1, ToType2))
         return ImplicitConversionSequence::Better;
-      else if (IsDerivedFrom(ToType2, ToType1))
+      else if (S.IsDerivedFrom(ToType2, ToType1))
         return ImplicitConversionSequence::Worse;
     }
 
@@ -2548,11 +2620,11 @@ Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
     //   -- binding of an expression of type B to a reference of type
     //      A& is better than binding an expression of type C to a
     //      reference of type A&,
-    if (!Context.hasSameUnqualifiedType(FromType1, FromType2) &&
-        Context.hasSameUnqualifiedType(ToType1, ToType2)) {
-      if (IsDerivedFrom(FromType2, FromType1))
+    if (!S.Context.hasSameUnqualifiedType(FromType1, FromType2) &&
+        S.Context.hasSameUnqualifiedType(ToType1, ToType2)) {
+      if (S.IsDerivedFrom(FromType2, FromType1))
         return ImplicitConversionSequence::Better;
-      else if (IsDerivedFrom(FromType1, FromType2))
+      else if (S.IsDerivedFrom(FromType1, FromType2))
         return ImplicitConversionSequence::Worse;
     }
   }
@@ -2570,7 +2642,8 @@ Sema::CompareDerivedToBaseConversions(const StandardConversionSequence& SCS1,
 Sema::ReferenceCompareResult
 Sema::CompareReferenceRelationship(SourceLocation Loc,
                                    QualType OrigT1, QualType OrigT2,
-                                   bool& DerivedToBase) {
+                                   bool &DerivedToBase,
+                                   bool &ObjCConversion) {
   assert(!OrigT1->isReferenceType() &&
     "T1 must be the pointee type of the reference type");
   assert(!OrigT2->isReferenceType() && "T2 cannot be a reference type");
@@ -2585,11 +2658,17 @@ Sema::CompareReferenceRelationship(SourceLocation Loc,
   //   Given types "cv1 T1" and "cv2 T2," "cv1 T1" is
   //   reference-related to "cv2 T2" if T1 is the same type as T2, or
   //   T1 is a base class of T2.
-  if (UnqualT1 == UnqualT2)
-    DerivedToBase = false;
-  else if (!RequireCompleteType(Loc, OrigT2, PDiag()) &&
+  DerivedToBase = false;
+  ObjCConversion = false;
+  if (UnqualT1 == UnqualT2) {
+    // Nothing to do.
+  } else if (!RequireCompleteType(Loc, OrigT2, PDiag()) &&
            IsDerivedFrom(UnqualT2, UnqualT1))
     DerivedToBase = true;
+  else if (UnqualT1->isObjCObjectOrInterfaceType() &&
+           UnqualT2->isObjCObjectOrInterfaceType() &&
+           Context.canBindObjCObjectType(UnqualT1, UnqualT2))
+    ObjCConversion = true;
   else
     return Ref_Incompatible;
 
@@ -2618,16 +2697,21 @@ Sema::CompareReferenceRelationship(SourceLocation Loc,
     return Ref_Related;
 }
 
-/// \brief Look for a user-defined conversion to an lvalue reference-compatible
+/// \brief Look for a user-defined conversion to an value reference-compatible
 ///        with DeclType. Return true if something definite is found.
 static bool
-FindConversionToLValue(Sema &S, ImplicitConversionSequence &ICS,
-                       QualType DeclType, SourceLocation DeclLoc,
-                       Expr *Init, QualType T2, bool AllowExplicit) {
+FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
+                         QualType DeclType, SourceLocation DeclLoc,
+                         Expr *Init, QualType T2, bool AllowRvalues,
+                         bool AllowExplicit) {
   assert(T2->isRecordType() && "Can only find conversions of record types.");
   CXXRecordDecl *T2RecordDecl
     = dyn_cast<CXXRecordDecl>(T2->getAs<RecordType>()->getDecl());
 
+  QualType ToType
+    = AllowRvalues? DeclType->getAs<ReferenceType>()->getPointeeType()
+                  : DeclType;
+
   OverloadCandidateSet CandidateSet(DeclLoc);
   const UnresolvedSetImpl *Conversions
     = T2RecordDecl->getVisibleConversionFunctions();
@@ -2646,25 +2730,44 @@ FindConversionToLValue(Sema &S, ImplicitConversionSequence &ICS,
     else
       Conv = cast<CXXConversionDecl>(D);
 
-    // If the conversion function doesn't return a reference type,
-    // it can't be considered for this conversion. An rvalue reference
-    // is only acceptable if its referencee is a function type.
-    const ReferenceType *RefType =
-      Conv->getConversionType()->getAs<ReferenceType>();
-    if (RefType && (RefType->isLValueReferenceType() ||
-                    RefType->getPointeeType()->isFunctionType()) &&
-        (AllowExplicit || !Conv->isExplicit())) {
-      if (ConvTemplate)
-        S.AddTemplateConversionCandidate(ConvTemplate, I.getPair(), ActingDC,
-                                       Init, DeclType, CandidateSet);
-      else
-        S.AddConversionCandidate(Conv, I.getPair(), ActingDC, Init,
-                               DeclType, CandidateSet);
+    // If this is an explicit conversion, and we're not allowed to consider 
+    // explicit conversions, skip it.
+    if (!AllowExplicit && Conv->isExplicit())
+      continue;
+    
+    if (AllowRvalues) {
+      bool DerivedToBase = false;
+      bool ObjCConversion = false;
+      if (!ConvTemplate &&
+          S.CompareReferenceRelationship(DeclLoc,
+                                         Conv->getConversionType().getNonReferenceType().getUnqualifiedType(),
+                                         DeclType.getNonReferenceType().getUnqualifiedType(),
+                                         DerivedToBase, ObjCConversion)
+            == Sema::Ref_Incompatible)
+        continue;
+    } else {
+      // If the conversion function doesn't return a reference type,
+      // it can't be considered for this conversion. An rvalue reference
+      // is only acceptable if its referencee is a function type.
+
+      const ReferenceType *RefType =
+        Conv->getConversionType()->getAs<ReferenceType>();
+      if (!RefType ||
+          (!RefType->isLValueReferenceType() &&
+           !RefType->getPointeeType()->isFunctionType()))
+        continue;
     }
+    
+    if (ConvTemplate)
+      S.AddTemplateConversionCandidate(ConvTemplate, I.getPair(), ActingDC,
+                                       Init, ToType, CandidateSet);
+    else
+      S.AddConversionCandidate(Conv, I.getPair(), ActingDC, Init,
+                               ToType, CandidateSet);
   }
 
   OverloadCandidateSet::iterator Best;
-  switch (S.BestViableFunction(CandidateSet, DeclLoc, Best)) {
+  switch (CandidateSet.BestViableFunction(S, DeclLoc, Best)) {
   case OR_Success:
     // C++ [over.ics.ref]p1:
     //
@@ -2736,9 +2839,11 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
   // Compute some basic properties of the types and the initializer.
   bool isRValRef = DeclType->isRValueReferenceType();
   bool DerivedToBase = false;
+  bool ObjCConversion = false;
   Expr::Classification InitCategory = Init->Classify(S.Context);
   Sema::ReferenceCompareResult RefRelationship
-    = S.CompareReferenceRelationship(DeclLoc, T1, T2, DerivedToBase);
+    = S.CompareReferenceRelationship(DeclLoc, T1, T2, DerivedToBase,
+                                     ObjCConversion);
 
 
   // C++0x [dcl.init.ref]p5:
@@ -2764,7 +2869,9 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
       //   derived-to-base Conversion (13.3.3.1).
       ICS.setStandard();
       ICS.Standard.First = ICK_Identity;
-      ICS.Standard.Second = DerivedToBase? ICK_Derived_To_Base : ICK_Identity;
+      ICS.Standard.Second = DerivedToBase? ICK_Derived_To_Base
+                         : ObjCConversion? ICK_Compatible_Conversion
+                         : ICK_Identity;
       ICS.Standard.Third = ICK_Identity;
       ICS.Standard.FromTypePtr = T2.getAsOpaquePtr();
       ICS.Standard.setToType(0, T2);
@@ -2792,8 +2899,9 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
     if (!SuppressUserConversions && T2->isRecordType() &&
         !S.RequireCompleteType(DeclLoc, T2, 0) && 
         RefRelationship == Sema::Ref_Incompatible) {
-      if (FindConversionToLValue(S, ICS, DeclType, DeclLoc,
-                                 Init, T2, AllowExplicit))
+      if (FindConversionForRefInit(S, ICS, DeclType, DeclLoc,
+                                   Init, T2, /*AllowRvalues=*/false,
+                                   AllowExplicit))
         return ICS;
     }
   }
@@ -2845,26 +2953,37 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
   //          that is the result of the conversion in the second case
   //          (or, in either case, to the appropriate base class
   //          subobject of the object).
-  //
-  // We're only checking the first case here, which is a direct
-  // binding in C++0x but not in C++03.
-  if (InitCategory.isRValue() && T2->isRecordType() &&
-      RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification) {
-    ICS.setStandard();
-    ICS.Standard.First = ICK_Identity;
-    ICS.Standard.Second = DerivedToBase? ICK_Derived_To_Base : ICK_Identity;
-    ICS.Standard.Third = ICK_Identity;
-    ICS.Standard.FromTypePtr = T2.getAsOpaquePtr();
-    ICS.Standard.setToType(0, T2);
-    ICS.Standard.setToType(1, T1);
-    ICS.Standard.setToType(2, T1);
-    ICS.Standard.ReferenceBinding = true;
-    ICS.Standard.DirectBinding = S.getLangOptions().CPlusPlus0x;
-    ICS.Standard.RRefBinding = isRValRef;
-    ICS.Standard.CopyConstructor = 0;
-    return ICS;
+  if (T2->isRecordType()) {
+    // First case: "cv1 T1" is reference-compatible with "cv2 T2". This is a
+    // direct binding in C++0x but not in C++03.
+    if (InitCategory.isRValue() && 
+        RefRelationship >= Sema::Ref_Compatible_With_Added_Qualification) {
+      ICS.setStandard();
+      ICS.Standard.First = ICK_Identity;
+      ICS.Standard.Second = DerivedToBase? ICK_Derived_To_Base 
+                          : ObjCConversion? ICK_Compatible_Conversion
+                          : ICK_Identity;
+      ICS.Standard.Third = ICK_Identity;
+      ICS.Standard.FromTypePtr = T2.getAsOpaquePtr();
+      ICS.Standard.setToType(0, T2);
+      ICS.Standard.setToType(1, T1);
+      ICS.Standard.setToType(2, T1);
+      ICS.Standard.ReferenceBinding = true;
+      ICS.Standard.DirectBinding = S.getLangOptions().CPlusPlus0x;
+      ICS.Standard.RRefBinding = isRValRef;
+      ICS.Standard.CopyConstructor = 0;
+      return ICS;
+    }
+    
+    // Second case: not reference-related.
+    if (RefRelationship == Sema::Ref_Incompatible &&
+        !S.RequireCompleteType(DeclLoc, T2, 0) && 
+        FindConversionForRefInit(S, ICS, DeclType, DeclLoc,
+                                 Init, T2, /*AllowRvalues=*/true,
+                                 AllowExplicit))
+      return ICS;
   }
-
+  
   //       -- Otherwise, a temporary of type "cv1 T1" is created and
   //          initialized from the initializer expression using the
   //          rules for a non-reference copy initialization (8.5). The
@@ -2899,9 +3018,9 @@ TryReferenceInit(Sema &S, Expr *&Init, QualType DeclType,
   //   the argument expression. Any difference in top-level
   //   cv-qualification is subsumed by the initialization itself
   //   and does not constitute a conversion.
-  ICS = S.TryImplicitConversion(Init, T1, SuppressUserConversions,
-                                /*AllowExplicit=*/false,
-                                /*InOverloadResolution=*/false);
+  ICS = TryImplicitConversion(S, Init, T1, SuppressUserConversions,
+                              /*AllowExplicit=*/false,
+                              /*InOverloadResolution=*/false);
 
   // Of course, that's still a reference binding.
   if (ICS.isStandard()) {
@@ -2930,25 +3049,25 @@ TryCopyInitialization(Sema &S, Expr *From, QualType ToType,
                             SuppressUserConversions,
                             /*AllowExplicit=*/false);
 
-  return S.TryImplicitConversion(From, ToType,
-                                 SuppressUserConversions,
-                                 /*AllowExplicit=*/false,
-                                 InOverloadResolution);
+  return TryImplicitConversion(S, From, ToType,
+                               SuppressUserConversions,
+                               /*AllowExplicit=*/false,
+                               InOverloadResolution);
 }
 
 /// TryObjectArgumentInitialization - Try to initialize the object
 /// parameter of the given member function (@c Method) from the
 /// expression @p From.
-ImplicitConversionSequence
-Sema::TryObjectArgumentInitialization(QualType OrigFromType,
-                                      CXXMethodDecl *Method,
-                                      CXXRecordDecl *ActingContext) {
-  QualType ClassType = Context.getTypeDeclType(ActingContext);
+static ImplicitConversionSequence
+TryObjectArgumentInitialization(Sema &S, QualType OrigFromType,
+                                CXXMethodDecl *Method,
+                                CXXRecordDecl *ActingContext) {
+  QualType ClassType = S.Context.getTypeDeclType(ActingContext);
   // [class.dtor]p2: A destructor can be invoked for a const, volatile or
   //                 const volatile object.
   unsigned Quals = isa<CXXDestructorDecl>(Method) ?
     Qualifiers::Const | Qualifiers::Volatile : Method->getTypeQualifiers();
-  QualType ImplicitParamType =  Context.getCVRQualifiedType(ClassType, Quals);
+  QualType ImplicitParamType =  S.Context.getCVRQualifiedType(ClassType, Quals);
 
   // Set up the conversion sequence as a "bad" conversion, to allow us
   // to exit early.
@@ -2972,7 +3091,7 @@ Sema::TryObjectArgumentInitialization(QualType OrigFromType,
 
   // First check the qualifiers. We don't care about lvalue-vs-rvalue
   // with the implicit object parameter (C++ [over.match.funcs]p5).
-  QualType FromTypeCanon = Context.getCanonicalType(FromType);
+  QualType FromTypeCanon = S.Context.getCanonicalType(FromType);
   if (ImplicitParamType.getCVRQualifiers() 
                                     != FromTypeCanon.getLocalCVRQualifiers() &&
       !ImplicitParamType.isAtLeastAsQualifiedAs(FromTypeCanon)) {
@@ -2983,11 +3102,11 @@ Sema::TryObjectArgumentInitialization(QualType OrigFromType,
 
   // Check that we have either the same type or a derived type. It
   // affects the conversion rank.
-  QualType ClassTypeCanon = Context.getCanonicalType(ClassType);
+  QualType ClassTypeCanon = S.Context.getCanonicalType(ClassType);
   ImplicitConversionKind SecondKind;
   if (ClassTypeCanon == FromTypeCanon.getLocalUnqualifiedType()) {
     SecondKind = ICK_Identity;
-  } else if (IsDerivedFrom(FromType, ClassType))
+  } else if (S.IsDerivedFrom(FromType, ClassType))
     SecondKind = ICK_Derived_To_Base;
   else {
     ICS.setBad(BadConversionSequence::unrelated_class,
@@ -3030,7 +3149,7 @@ Sema::PerformObjectArgumentInitialization(Expr *&From,
   // Note that we always use the true parent context when performing
   // the actual argument initialization.
   ImplicitConversionSequence ICS
-    = TryObjectArgumentInitialization(From->getType(), Method,
+    = TryObjectArgumentInitialization(*this, From->getType(), Method,
                                       Method->getParent());
   if (ICS.isBad())
     return Diag(From->getSourceRange().getBegin(),
@@ -3041,16 +3160,17 @@ Sema::PerformObjectArgumentInitialization(Expr *&From,
     return PerformObjectMemberConversion(From, Qualifier, FoundDecl, Method);
 
   if (!Context.hasSameType(From->getType(), DestType))
-    ImpCastExprToType(From, DestType, CastExpr::CK_NoOp,
-                      /*isLvalue=*/!From->getType()->isPointerType());
+    ImpCastExprToType(From, DestType, CK_NoOp,
+                      From->getType()->isPointerType() ? VK_RValue : VK_LValue);
   return false;
 }
 
 /// TryContextuallyConvertToBool - Attempt to contextually convert the
 /// expression From to bool (C++0x [conv]p3).
-ImplicitConversionSequence Sema::TryContextuallyConvertToBool(Expr *From) {
+static ImplicitConversionSequence
+TryContextuallyConvertToBool(Sema &S, Expr *From) {
   // FIXME: This is pretty broken.
-  return TryImplicitConversion(From, Context.BoolTy,
+  return TryImplicitConversion(S, From, S.Context.BoolTy,
                                // FIXME: Are these flags correct?
                                /*SuppressUserConversions=*/false,
                                /*AllowExplicit=*/true,
@@ -3060,7 +3180,7 @@ ImplicitConversionSequence Sema::TryContextuallyConvertToBool(Expr *From) {
 /// PerformContextuallyConvertToBool - Perform a contextual conversion
 /// of the expression From to bool (C++0x [conv]p3).
 bool Sema::PerformContextuallyConvertToBool(Expr *&From) {
-  ImplicitConversionSequence ICS = TryContextuallyConvertToBool(From);
+  ImplicitConversionSequence ICS = TryContextuallyConvertToBool(*this, From);
   if (!ICS.isBad())
     return PerformImplicitConversion(From, Context.BoolTy, ICS, AA_Converting);
   
@@ -3073,20 +3193,21 @@ bool Sema::PerformContextuallyConvertToBool(Expr *&From) {
   
 /// TryContextuallyConvertToObjCId - Attempt to contextually convert the
 /// expression From to 'id'.
-ImplicitConversionSequence Sema::TryContextuallyConvertToObjCId(Expr *From) {
-  QualType Ty = Context.getObjCIdType();
-  return TryImplicitConversion(From, Ty,
-                                 // FIXME: Are these flags correct?
-                                 /*SuppressUserConversions=*/false,
-                                 /*AllowExplicit=*/true,
-                                 /*InOverloadResolution=*/false);
+static ImplicitConversionSequence
+TryContextuallyConvertToObjCId(Sema &S, Expr *From) {
+  QualType Ty = S.Context.getObjCIdType();
+  return TryImplicitConversion(S, From, Ty,
+                               // FIXME: Are these flags correct?
+                               /*SuppressUserConversions=*/false,
+                               /*AllowExplicit=*/true,
+                               /*InOverloadResolution=*/false);
 }
-  
+
 /// PerformContextuallyConvertToObjCId - Perform a contextual conversion
 /// of the expression From to 'id'.
 bool Sema::PerformContextuallyConvertToObjCId(Expr *&From) {
   QualType Ty = Context.getObjCIdType();
-  ImplicitConversionSequence ICS = TryContextuallyConvertToObjCId(From);
+  ImplicitConversionSequence ICS = TryContextuallyConvertToObjCId(*this, From);
   if (!ICS.isBad())
     return PerformImplicitConversion(From, Ty, ICS, AA_Converting);
   return true;
@@ -3128,8 +3249,8 @@ bool Sema::PerformContextuallyConvertToObjCId(Expr *&From) {
 ///
 /// \returns The expression, converted to an integral or enumeration type if
 /// successful.
-Sema::OwningExprResult 
-Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
+ExprResult 
+Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, Expr *From,
                                          const PartialDiagnostic &NotIntDiag,
                                        const PartialDiagnostic &IncompleteDiag,
                                      const PartialDiagnostic &ExplicitConvDiag,
@@ -3137,16 +3258,14 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
                                          const PartialDiagnostic &AmbigDiag,
                                          const PartialDiagnostic &AmbigNote,
                                          const PartialDiagnostic &ConvDiag) {
-  Expr *From = static_cast<Expr *>(FromE.get());
-  
   // We can't perform any more checking for type-dependent expressions.
   if (From->isTypeDependent())
-    return move(FromE);
+    return Owned(From);
   
   // If the expression already has integral or enumeration type, we're golden.
   QualType T = From->getType();
   if (T->isIntegralOrEnumerationType())
-    return move(FromE);
+    return Owned(From);
 
   // FIXME: Check for missing '()' if T is a function type?
 
@@ -3156,12 +3275,12 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
   if (!RecordTy || !getLangOptions().CPlusPlus) {
     Diag(Loc, NotIntDiag)
       << T << From->getSourceRange();
-    return move(FromE);
+    return Owned(From);
   }
     
   // We must have a complete class type.
   if (RequireCompleteType(Loc, T, IncompleteDiag))
-    return move(FromE);
+    return Owned(From);
   
   // Look for a conversion to an integral or enumeration type.
   UnresolvedSet<4> ViableConversions;
@@ -3213,8 +3332,7 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
         return ExprError();
       
       CheckMemberOperatorAccess(From->getExprLoc(), From, 0, Found);
-      From = BuildCXXMemberCallExpr(FromE.takeAs<Expr>(), Found, Conversion);
-      FromE = Owned(From);
+      From = BuildCXXMemberCallExpr(From, Found, Conversion);
     }
     
     // We'll complain below about a non-integral condition type.
@@ -3237,9 +3355,8 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
         << T << ConvTy->isEnumeralType() << ConvTy << From->getSourceRange();
     }
     
-    From = BuildCXXMemberCallExpr(FromE.takeAs<Expr>(), Found,
+    From = BuildCXXMemberCallExpr(From, Found,
                           cast<CXXConversionDecl>(Found->getUnderlyingDecl()));
-    FromE = Owned(From);
     break;
   }
     
@@ -3253,14 +3370,14 @@ Sema::ConvertToIntegralOrEnumerationType(SourceLocation Loc, ExprArg FromE,
       Diag(Conv->getLocation(), AmbigNote)
         << ConvTy->isEnumeralType() << ConvTy;
     }
-    return move(FromE);
+    return Owned(From);
   }
   
   if (!From->getType()->isIntegralOrEnumerationType())
     Diag(Loc, NotIntDiag)
       << From->getType() << From->getSourceRange();
 
-  return move(FromE);
+  return Owned(From);
 }
 
 /// AddOverloadCandidate - Adds the given function to the set of
@@ -3306,7 +3423,7 @@ Sema::AddOverloadCandidate(FunctionDecl *Function,
     return;
 
   // Overload resolution is always an unevaluated context.
-  EnterExpressionEvaluationContext Unevaluated(*this, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(*this, Sema::Unevaluated);
 
   if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(Function)){
     // C++ [class.copy]p3:
@@ -3469,7 +3586,7 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
     return;
 
   // Overload resolution is always an unevaluated context.
-  EnterExpressionEvaluationContext Unevaluated(*this, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(*this, Sema::Unevaluated);
 
   // Add this candidate
   CandidateSet.push_back(OverloadCandidate());
@@ -3513,7 +3630,8 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
     // Determine the implicit conversion sequence for the object
     // parameter.
     Candidate.Conversions[0]
-      = TryObjectArgumentInitialization(ObjectType, Method, ActingContext);
+      = TryObjectArgumentInitialization(*this, ObjectType, Method,
+                                        ActingContext);
     if (Candidate.Conversions[0].isBad()) {
       Candidate.Viable = false;
       Candidate.FailureKind = ovl_fail_bad_conversion;
@@ -3666,7 +3784,7 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
     return;
 
   // Overload resolution is always an unevaluated context.
-  EnterExpressionEvaluationContext Unevaluated(*this, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(*this, Sema::Unevaluated);
 
   // Add this candidate
   CandidateSet.push_back(OverloadCandidate());
@@ -3678,25 +3796,32 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   Candidate.FinalConversion.setAsIdentityConversion();
   Candidate.FinalConversion.setFromType(ConvType);
   Candidate.FinalConversion.setAllToTypes(ToType);
+  Candidate.Viable = true;
+  Candidate.Conversions.resize(1);
 
+  // C++ [over.match.funcs]p4:
+  //   For conversion functions, the function is considered to be a member of 
+  //   the class of the implicit implied object argument for the purpose of 
+  //   defining the type of the implicit object parameter.
+  //
   // Determine the implicit conversion sequence for the implicit
   // object parameter.
-  Candidate.Viable = true;
-  Candidate.Conversions.resize(1);
+  QualType ImplicitParamType = From->getType();
+  if (const PointerType *FromPtrType = ImplicitParamType->getAs<PointerType>())
+    ImplicitParamType = FromPtrType->getPointeeType();
+  CXXRecordDecl *ConversionContext
+    = cast<CXXRecordDecl>(ImplicitParamType->getAs<RecordType>()->getDecl());
+  
   Candidate.Conversions[0]
-    = TryObjectArgumentInitialization(From->getType(), Conversion,
-                                      ActingContext);
-  // Conversion functions to a different type in the base class is visible in 
-  // the derived class.  So, a derived to base conversion should not participate
-  // in overload resolution. 
-  if (Candidate.Conversions[0].Standard.Second == ICK_Derived_To_Base)
-    Candidate.Conversions[0].Standard.Second = ICK_Identity;
+    = TryObjectArgumentInitialization(*this, From->getType(), Conversion,
+                                      ConversionContext);
+  
   if (Candidate.Conversions[0].isBad()) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_conversion;
     return;
   }
-  
+
   // We won't go through a user-define type conversion function to convert a 
   // derived to base as such conversions are given Conversion Rank. They only
   // go through a copy constructor. 13.3.3.1.2-p4 [over.ics.user]
@@ -3719,9 +3844,10 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   // well-formed.
   DeclRefExpr ConversionRef(Conversion, Conversion->getType(),
                             From->getLocStart());
-  ImplicitCastExpr ConversionFn(Context.getPointerType(Conversion->getType()),
-                                CastExpr::CK_FunctionToPointerDecay,
-                                &ConversionRef, CXXBaseSpecifierArray(), false);
+  ImplicitCastExpr ConversionFn(ImplicitCastExpr::OnStack,
+                                Context.getPointerType(Conversion->getType()),
+                                CK_FunctionToPointerDecay,
+                                &ConversionRef, VK_RValue);
 
   // Note that it is safe to allocate CallExpr on the stack here because
   // there are 0 arguments (i.e., nothing is allocated using ASTContext's
@@ -3819,7 +3945,7 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
     return;
 
   // Overload resolution is always an unevaluated context.
-  EnterExpressionEvaluationContext Unevaluated(*this, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(*this, Sema::Unevaluated);
 
   CandidateSet.push_back(OverloadCandidate());
   OverloadCandidate& Candidate = CandidateSet.back();
@@ -3834,7 +3960,8 @@ void Sema::AddSurrogateCandidate(CXXConversionDecl *Conversion,
   // Determine the implicit conversion sequence for the implicit
   // object parameter.
   ImplicitConversionSequence ObjectInit
-    = TryObjectArgumentInitialization(ObjectType, Conversion, ActingContext);
+    = TryObjectArgumentInitialization(*this, ObjectType, Conversion,
+                                      ActingContext);
   if (ObjectInit.isBad()) {
     Candidate.Viable = false;
     Candidate.FailureKind = ovl_fail_bad_conversion;
@@ -3925,9 +4052,6 @@ void Sema::AddMemberOperatorCandidates(OverloadedOperatorKind Op,
   //   candidates, non-member candidates and built-in candidates, are
   //   constructed as follows:
   QualType T1 = Args[0]->getType();
-  QualType T2;
-  if (NumArgs > 1)
-    T2 = Args[1]->getType();
 
   //     -- If T1 is a class type, the set of member candidates is the
   //        result of the qualified lookup of T1::operator@
@@ -3966,7 +4090,7 @@ void Sema::AddBuiltinCandidate(QualType ResultTy, QualType *ParamTys,
                                bool IsAssignmentOperator,
                                unsigned NumContextualBoolArguments) {
   // Overload resolution is always an unevaluated context.
-  EnterExpressionEvaluationContext Unevaluated(*this, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(*this, Sema::Unevaluated);
 
   // Add this candidate
   CandidateSet.push_back(OverloadCandidate());
@@ -3999,7 +4123,8 @@ void Sema::AddBuiltinCandidate(QualType ResultTy, QualType *ParamTys,
     if (ArgIdx < NumContextualBoolArguments) {
       assert(ParamTys[ArgIdx] == Context.BoolTy &&
              "Contextual conversion to bool requires bool type");
-      Candidate.Conversions[ArgIdx] = TryContextuallyConvertToBool(Args[ArgIdx]);
+      Candidate.Conversions[ArgIdx]
+        = TryContextuallyConvertToBool(*this, Args[ArgIdx]);
     } else {
       Candidate.Conversions[ArgIdx]
         = TryCopyInitialization(*this, Args[ArgIdx], ParamTys[ArgIdx],
@@ -4100,11 +4225,21 @@ BuiltinCandidateTypeSet::AddPointerWithMoreQualifiedTypeVariants(QualType Ty,
   // Insert this type.
   if (!PointerTypes.insert(Ty))
     return false;
-
+    
+  QualType PointeeTy;
   const PointerType *PointerTy = Ty->getAs<PointerType>();
-  assert(PointerTy && "type was not a pointer type!");
-
-  QualType PointeeTy = PointerTy->getPointeeType();
+  bool buildObjCPtr = false;
+  if (!PointerTy) {
+    if (const ObjCObjectPointerType *PTy = Ty->getAs<ObjCObjectPointerType>()) {
+      PointeeTy = PTy->getPointeeType();
+      buildObjCPtr = true;
+    }
+    else
+      assert(false && "type was not a pointer type!");
+  }
+  else
+    PointeeTy = PointerTy->getPointeeType();
+  
   // Don't add qualified variants of arrays. For one, they're not allowed
   // (the qualifier would sink to the element type), and for another, the
   // only overload situation where it matters is subscript or pointer +- int,
@@ -4125,7 +4260,10 @@ BuiltinCandidateTypeSet::AddPointerWithMoreQualifiedTypeVariants(QualType Ty,
     if ((CVR & Qualifiers::Volatile) && !hasVolatile) continue;
     if ((CVR & Qualifiers::Restrict) && !hasRestrict) continue;
     QualType QPointeeTy = Context.getCVRQualifiedType(PointeeTy, CVR);
-    PointerTypes.insert(Context.getPointerType(QPointeeTy));
+    if (!buildObjCPtr)
+      PointerTypes.insert(Context.getPointerType(QPointeeTy));
+    else
+      PointerTypes.insert(Context.getObjCObjectPointerType(QPointeeTy));
   }
 
   return true;
@@ -4200,10 +4338,9 @@ BuiltinCandidateTypeSet::AddTypesConvertedFrom(QualType Ty,
   // If we're dealing with an array type, decay to the pointer.
   if (Ty->isArrayType())
     Ty = SemaRef.Context.getArrayDecayedType(Ty);
-
-  if (const PointerType *PointerTy = Ty->getAs<PointerType>()) {
-    QualType PointeeTy = PointerTy->getPointeeType();
-
+  if (Ty->isObjCIdType() || Ty->isObjCClassType())
+    PointerTypes.insert(Ty);
+  else if (Ty->getAs<PointerType>() || Ty->getAs<ObjCObjectPointerType>()) {
     // Insert our type, and its more-qualified variants, into the set
     // of types.
     if (!AddPointerWithMoreQualifiedTypeVariants(Ty, VisibleQuals))
@@ -4479,7 +4616,7 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
     for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
          Ptr != CandidateTypes.pointer_end(); ++Ptr) {
       // Skip pointer types that aren't pointers to object types.
-      if (!(*Ptr)->getAs<PointerType>()->getPointeeType()->isObjectType())
+      if (!(*Ptr)->getPointeeType()->isIncompleteOrObjectType())
         continue;
 
       QualType ParamTypes[2] = {
@@ -4519,7 +4656,7 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
     for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
          Ptr != CandidateTypes.pointer_end(); ++Ptr) {
       QualType ParamTy = *Ptr;
-      QualType PointeeTy = ParamTy->getAs<PointerType>()->getPointeeType();
+      QualType PointeeTy = ParamTy->getPointeeType();
       AddBuiltinCandidate(Context.getLValueReferenceType(PointeeTy),
                           &ParamTy, Args, 1, CandidateSet);
     }
@@ -5000,7 +5137,7 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
     for (BuiltinCandidateTypeSet::iterator Ptr = CandidateTypes.pointer_begin();
          Ptr != CandidateTypes.pointer_end(); ++Ptr) {
       QualType ParamTypes[2] = { *Ptr, Context.getPointerDiffType() };
-      QualType PointeeType = (*Ptr)->getAs<PointerType>()->getPointeeType();
+      QualType PointeeType = (*Ptr)->getPointeeType();
       QualType ResultTy = Context.getLValueReferenceType(PointeeType);
 
       // T& operator[](T*, ptrdiff_t)
@@ -5028,18 +5165,16 @@ Sema::AddBuiltinOperatorCandidates(OverloadedOperatorKind Op,
         QualType C1Ty = (*Ptr);
         QualType C1;
         QualifierCollector Q1;
-        if (const PointerType *PointerTy = C1Ty->getAs<PointerType>()) {
-          C1 = QualType(Q1.strip(PointerTy->getPointeeType()), 0);
-          if (!isa<RecordType>(C1))
-            continue;
-          // heuristic to reduce number of builtin candidates in the set.
-          // Add volatile/restrict version only if there are conversions to a
-          // volatile/restrict type.
-          if (!VisibleTypeConversionsQuals.hasVolatile() && Q1.hasVolatile())
-            continue;
-          if (!VisibleTypeConversionsQuals.hasRestrict() && Q1.hasRestrict())
-            continue;
-        }
+        C1 = QualType(Q1.strip(C1Ty->getPointeeType()), 0);
+        if (!isa<RecordType>(C1))
+          continue;
+        // heuristic to reduce number of builtin candidates in the set.
+        // Add volatile/restrict version only if there are conversions to a
+        // volatile/restrict type.
+        if (!VisibleTypeConversionsQuals.hasVolatile() && Q1.hasVolatile())
+          continue;
+        if (!VisibleTypeConversionsQuals.hasRestrict() && Q1.hasRestrict())
+          continue;
         for (BuiltinCandidateTypeSet::iterator
              MemPtr = CandidateTypes.member_pointer_begin(),
              MemPtrEnd = CandidateTypes.member_pointer_end();
@@ -5148,9 +5283,10 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
 /// isBetterOverloadCandidate - Determines whether the first overload
 /// candidate is a better candidate than the second (C++ 13.3.3p1).
 bool
-Sema::isBetterOverloadCandidate(const OverloadCandidate& Cand1,
-                                const OverloadCandidate& Cand2,
-                                SourceLocation Loc) {
+isBetterOverloadCandidate(Sema &S,
+                          const OverloadCandidate& Cand1,
+                          const OverloadCandidate& Cand2,
+                          SourceLocation Loc) {
   // Define viable functions to be better candidates than non-viable
   // functions.
   if (!Cand2.Viable)
@@ -5176,7 +5312,8 @@ Sema::isBetterOverloadCandidate(const OverloadCandidate& Cand1,
   assert(Cand2.Conversions.size() == NumArgs && "Overload candidate mismatch");
   bool HasBetterConversion = false;
   for (unsigned ArgIdx = StartArg; ArgIdx < NumArgs; ++ArgIdx) {
-    switch (CompareImplicitConversionSequences(Cand1.Conversions[ArgIdx],
+    switch (CompareImplicitConversionSequences(S,
+                                               Cand1.Conversions[ArgIdx],
                                                Cand2.Conversions[ArgIdx])) {
     case ImplicitConversionSequence::Better:
       // Cand1 has a better conversion sequence.
@@ -5211,9 +5348,9 @@ Sema::isBetterOverloadCandidate(const OverloadCandidate& Cand1,
   if (Cand1.Function && Cand1.Function->getPrimaryTemplate() &&
       Cand2.Function && Cand2.Function->getPrimaryTemplate())
     if (FunctionTemplateDecl *BetterTemplate
-          = getMoreSpecializedTemplate(Cand1.Function->getPrimaryTemplate(),
-                                       Cand2.Function->getPrimaryTemplate(),
-                                       Loc,
+          = S.getMoreSpecializedTemplate(Cand1.Function->getPrimaryTemplate(),
+                                         Cand2.Function->getPrimaryTemplate(),
+                                         Loc,
                        isa<CXXConversionDecl>(Cand1.Function)? TPOC_Conversion 
                                                              : TPOC_Call))
       return BetterTemplate == Cand1.Function->getPrimaryTemplate();
@@ -5227,7 +5364,8 @@ Sema::isBetterOverloadCandidate(const OverloadCandidate& Cand1,
   if (Cand1.Function && Cand2.Function &&
       isa<CXXConversionDecl>(Cand1.Function) &&
       isa<CXXConversionDecl>(Cand2.Function)) {
-    switch (CompareStandardConversionSequences(Cand1.FinalConversion,
+    switch (CompareStandardConversionSequences(S,
+                                               Cand1.FinalConversion,
                                                Cand2.FinalConversion)) {
     case ImplicitConversionSequence::Better:
       // Cand1 has a better conversion sequence.
@@ -5258,32 +5396,28 @@ Sema::isBetterOverloadCandidate(const OverloadCandidate& Cand1,
 /// function, Best points to the candidate function found.
 ///
 /// \returns The result of overload resolution.
-OverloadingResult Sema::BestViableFunction(OverloadCandidateSet& CandidateSet,
-                                           SourceLocation Loc,
-                                        OverloadCandidateSet::iterator& Best) {
+OverloadingResult
+OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
+                                         iterator& Best) {
   // Find the best viable function.
-  Best = CandidateSet.end();
-  for (OverloadCandidateSet::iterator Cand = CandidateSet.begin();
-       Cand != CandidateSet.end(); ++Cand) {
-    if (Cand->Viable) {
-      if (Best == CandidateSet.end() ||
-          isBetterOverloadCandidate(*Cand, *Best, Loc))
+  Best = end();
+  for (iterator Cand = begin(); Cand != end(); ++Cand) {
+    if (Cand->Viable)
+      if (Best == end() || isBetterOverloadCandidate(S, *Cand, *Best, Loc))
         Best = Cand;
-    }
   }
 
   // If we didn't find any viable functions, abort.
-  if (Best == CandidateSet.end())
+  if (Best == end())
     return OR_No_Viable_Function;
 
   // Make sure that this function is better than every other viable
   // function. If not, we have an ambiguity.
-  for (OverloadCandidateSet::iterator Cand = CandidateSet.begin();
-       Cand != CandidateSet.end(); ++Cand) {
+  for (iterator Cand = begin(); Cand != end(); ++Cand) {
     if (Cand->Viable &&
         Cand != Best &&
-        !isBetterOverloadCandidate(*Best, *Cand, Loc)) {
-      Best = CandidateSet.end();
+        !isBetterOverloadCandidate(S, *Best, *Cand, Loc)) {
+      Best = end();
       return OR_Ambiguous;
     }
   }
@@ -5301,7 +5435,7 @@ OverloadingResult Sema::BestViableFunction(OverloadCandidateSet& CandidateSet,
   //   (clause 13), user-defined conversions (12.3.2), allocation function for
   //   placement new (5.3.4), as well as non-default initialization (8.5).
   if (Best->Function)
-    MarkDeclarationReferenced(Loc, Best->Function);
+    S.MarkDeclarationReferenced(Loc, Best->Function);
   return OR_Success;
 }
 
@@ -5365,14 +5499,15 @@ void Sema::NoteOverloadCandidate(FunctionDecl *Fn) {
 /// Diagnoses an ambiguous conversion.  The partial diagnostic is the
 /// "lead" diagnostic; it will be given two arguments, the source and
 /// target types of the conversion.
-void Sema::DiagnoseAmbiguousConversion(const ImplicitConversionSequence &ICS,
-                                       SourceLocation CaretLoc,
-                                       const PartialDiagnostic &PDiag) {
-  Diag(CaretLoc, PDiag)
-    << ICS.Ambiguous.getFromType() << ICS.Ambiguous.getToType();
+void ImplicitConversionSequence::DiagnoseAmbiguousConversion(
+                                 Sema &S,
+                                 SourceLocation CaretLoc,
+                                 const PartialDiagnostic &PDiag) const {
+  S.Diag(CaretLoc, PDiag)
+    << Ambiguous.getFromType() << Ambiguous.getToType();
   for (AmbiguousConversionSequence::const_iterator
-         I = ICS.Ambiguous.begin(), E = ICS.Ambiguous.end(); I != E; ++I) {
-    NoteOverloadCandidate(*I);
+         I = Ambiguous.begin(), E = Ambiguous.end(); I != E; ++I) {
+    S.NoteOverloadCandidate(*I);
   }
 }
 
@@ -5589,8 +5724,31 @@ void DiagnoseBadDeduction(Sema &S, OverloadCandidate *Cand,
     return;
   }
 
-  case Sema::TDK_Inconsistent:
-  case Sema::TDK_InconsistentQuals: {
+  case Sema::TDK_Underqualified: {
+    assert(ParamD && "no parameter found for bad qualifiers deduction result");
+    TemplateTypeParmDecl *TParam = cast<TemplateTypeParmDecl>(ParamD);
+
+    QualType Param = Cand->DeductionFailure.getFirstArg()->getAsType();
+
+    // Param will have been canonicalized, but it should just be a
+    // qualified version of ParamD, so move the qualifiers to that.
+    QualifierCollector Qs(S.Context);
+    Qs.strip(Param);
+    QualType NonCanonParam = Qs.apply(TParam->getTypeForDecl());
+    assert(S.Context.hasSameType(Param, NonCanonParam));
+
+    // Arg has also been canonicalized, but there's nothing we can do
+    // about that.  It also doesn't matter as much, because it won't
+    // have any template parameters in it (because deduction isn't
+    // done on dependent types).
+    QualType Arg = Cand->DeductionFailure.getSecondArg()->getAsType();
+
+    S.Diag(Fn->getLocation(), diag::note_ovl_candidate_underqualified)
+      << ParamD->getDeclName() << Arg << NonCanonParam;
+    return;
+  }
+
+  case Sema::TDK_Inconsistent: {
     assert(ParamD && "no parameter found for inconsistent deduction result");    
     int which = 0;
     if (isa<TemplateTypeParmDecl>(ParamD))
@@ -5779,7 +5937,7 @@ void NoteAmbiguousUserConversions(Sema &S, SourceLocation OpLoc,
     if (ICS.isBad()) break; // all meaningless after first invalid
     if (!ICS.isAmbiguous()) continue;
 
-    S.DiagnoseAmbiguousConversion(ICS, OpLoc,
+    ICS.DiagnoseAmbiguousConversion(S, OpLoc,
                               S.PDiag(diag::note_ambiguous_type_conversion));
   }
 }
@@ -5808,8 +5966,8 @@ struct CompareOverloadCandidatesForDisplay {
       // TODO: introduce a tri-valued comparison for overload
       // candidates.  Would be more worthwhile if we had a sort
       // that could exploit it.
-      if (S.isBetterOverloadCandidate(*L, *R, SourceLocation())) return true;
-      if (S.isBetterOverloadCandidate(*R, *L, SourceLocation())) return false;
+      if (isBetterOverloadCandidate(S, *L, *R, SourceLocation())) return true;
+      if (isBetterOverloadCandidate(S, *R, *L, SourceLocation())) return false;
     } else if (R->Viable)
       return false;
 
@@ -5838,8 +5996,9 @@ struct CompareOverloadCandidatesForDisplay {
         int leftBetter = 0;
         unsigned I = (L->IgnoreObjectArgument || R->IgnoreObjectArgument);
         for (unsigned E = L->Conversions.size(); I != E; ++I) {
-          switch (S.CompareImplicitConversionSequences(L->Conversions[I],
-                                                       R->Conversions[I])) {
+          switch (CompareImplicitConversionSequences(S,
+                                                     L->Conversions[I],
+                                                     R->Conversions[I])) {
           case ImplicitConversionSequence::Better:
             leftBetter++;
             break;
@@ -5947,23 +6106,20 @@ void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
 /// PrintOverloadCandidates - When overload resolution fails, prints
 /// diagnostic messages containing the candidates in the candidate
 /// set.
-void
-Sema::PrintOverloadCandidates(OverloadCandidateSet& CandidateSet,
-                              OverloadCandidateDisplayKind OCD,
-                              Expr **Args, unsigned NumArgs,
-                              const char *Opc,
-                              SourceLocation OpLoc) {
+void OverloadCandidateSet::NoteCandidates(Sema &S,
+                                          OverloadCandidateDisplayKind OCD,
+                                          Expr **Args, unsigned NumArgs,
+                                          const char *Opc,
+                                          SourceLocation OpLoc) {
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
   llvm::SmallVector<OverloadCandidate*, 32> Cands;
-  if (OCD == OCD_AllCandidates) Cands.reserve(CandidateSet.size());
-  for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
-                                  LastCand = CandidateSet.end();
-       Cand != LastCand; ++Cand) {
+  if (OCD == OCD_AllCandidates) Cands.reserve(size());
+  for (iterator Cand = begin(), LastCand = end(); Cand != LastCand; ++Cand) {
     if (Cand->Viable)
       Cands.push_back(Cand);
     else if (OCD == OCD_AllCandidates) {
-      CompleteNonViableCandidate(*this, Cand, Args, NumArgs);
+      CompleteNonViableCandidate(S, Cand, Args, NumArgs);
       if (Cand->Function || Cand->IsSurrogate)
         Cands.push_back(Cand);
       // Otherwise, this a non-viable builtin candidate.  We do not, in general,
@@ -5972,12 +6128,12 @@ Sema::PrintOverloadCandidates(OverloadCandidateSet& CandidateSet,
   }
 
   std::sort(Cands.begin(), Cands.end(),
-            CompareOverloadCandidatesForDisplay(*this));
+            CompareOverloadCandidatesForDisplay(S));
   
   bool ReportedAmbiguousConversions = false;
 
   llvm::SmallVectorImpl<OverloadCandidate*>::iterator I, E;
-  const Diagnostic::OverloadsShown ShowOverloads = Diags.getShowOverloads();
+  const Diagnostic::OverloadsShown ShowOverloads = S.Diags.getShowOverloads();
   unsigned CandsShown = 0;
   for (I = Cands.begin(), E = Cands.end(); I != E; ++I) {
     OverloadCandidate *Cand = *I;
@@ -5991,9 +6147,9 @@ Sema::PrintOverloadCandidates(OverloadCandidateSet& CandidateSet,
     ++CandsShown;
 
     if (Cand->Function)
-      NoteFunctionCandidate(*this, Cand, Args, NumArgs);
+      NoteFunctionCandidate(S, Cand, Args, NumArgs);
     else if (Cand->IsSurrogate)
-      NoteSurrogateCandidate(*this, Cand);
+      NoteSurrogateCandidate(S, Cand);
     else {
       assert(Cand->Viable &&
              "Non-viable built-in candidates are not added to Cands.");
@@ -6004,17 +6160,17 @@ Sema::PrintOverloadCandidates(OverloadCandidateSet& CandidateSet,
       // FIXME: It's quite possible for different conversions to see
       // different ambiguities, though.
       if (!ReportedAmbiguousConversions) {
-        NoteAmbiguousUserConversions(*this, OpLoc, Cand);
+        NoteAmbiguousUserConversions(S, OpLoc, Cand);
         ReportedAmbiguousConversions = true;
       }
 
       // If this is a viable builtin, print it.
-      NoteBuiltinOperatorCandidate(*this, Opc, OpLoc, Cand);
+      NoteBuiltinOperatorCandidate(S, Opc, OpLoc, Cand);
     }
   }
 
   if (I != E)
-    Diag(OpLoc, diag::note_ovl_too_many_candidates) << int(E - I);
+    S.Diag(OpLoc, diag::note_ovl_too_many_candidates) << int(E - I);
 }
 
 static bool CheckUnresolvedAccess(Sema &S, OverloadExpr *E, DeclAccessPair D) {
@@ -6061,12 +6217,13 @@ Sema::ResolveAddressOfOverloadedFunction(Expr *From, QualType ToType,
   // C++ [over.over]p1:
   //   [...] The overloaded function name can be preceded by the &
   //   operator.
-  OverloadExpr *OvlExpr = OverloadExpr::find(From).getPointer();
-  TemplateArgumentListInfo ETABuffer, *ExplicitTemplateArgs = 0;
-  if (OvlExpr->hasExplicitTemplateArgs()) {
-    OvlExpr->getExplicitTemplateArgs().copyInto(ETABuffer);
-    ExplicitTemplateArgs = &ETABuffer;
-  }
+  // However, remember whether the expression has member-pointer form:
+  // C++ [expr.unary.op]p4:
+  //     A pointer to member is only formed when an explicit & is used
+  //     and its operand is a qualified-id not enclosed in
+  //     parentheses.
+  OverloadExpr::FindResult Ovl = OverloadExpr::find(From);
+  OverloadExpr *OvlExpr = Ovl.Expression;
   
   // We expect a pointer or reference to function, or a function pointer.
   FunctionType = Context.getCanonicalType(FunctionType).getUnqualifiedType();
@@ -6078,6 +6235,25 @@ Sema::ResolveAddressOfOverloadedFunction(Expr *From, QualType ToType,
     return 0;
   }
 
+  // If the overload expression doesn't have the form of a pointer to
+  // member, don't try to convert it to a pointer-to-member type.
+  if (IsMember && !Ovl.HasFormOfMemberPointer) {
+    if (!Complain) return 0;
+
+    // TODO: Should we condition this on whether any functions might
+    // have matched, or is it more appropriate to do that in callers?
+    // TODO: a fixit wouldn't hurt.
+    Diag(OvlExpr->getNameLoc(), diag::err_addr_ovl_no_qualifier)
+      << ToType << OvlExpr->getSourceRange();
+    return 0;
+  }
+
+  TemplateArgumentListInfo ETABuffer, *ExplicitTemplateArgs = 0;
+  if (OvlExpr->hasExplicitTemplateArgs()) {
+    OvlExpr->getExplicitTemplateArgs().copyInto(ETABuffer);
+    ExplicitTemplateArgs = &ETABuffer;
+  }
+
   assert(From->getType() == Context.OverloadTy);
 
   // Look through all of the overloaded functions, searching for one
@@ -6267,7 +6443,7 @@ FunctionDecl *Sema::ResolveSingleFunctionTemplateSpecialization(Expr *From) {
   if (From->getType() != Context.OverloadTy)
     return 0;
 
-  OverloadExpr *OvlExpr = OverloadExpr::find(From).getPointer();
+  OverloadExpr *OvlExpr = OverloadExpr::find(From).Expression;
   
   // If we didn't actually find any template-ids, we're done.
   if (!OvlExpr->hasExplicitTemplateArgs())
@@ -6405,18 +6581,10 @@ void Sema::AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
                                          PartialOverloading);  
 }
 
-static Sema::OwningExprResult Destroy(Sema &SemaRef, Expr *Fn,
-                                      Expr **Args, unsigned NumArgs) {
-  Fn->Destroy(SemaRef.Context);
-  for (unsigned Arg = 0; Arg < NumArgs; ++Arg)
-    Args[Arg]->Destroy(SemaRef.Context);
-  return SemaRef.ExprError();
-}
-
 /// Attempts to recover from a call where no functions were found.
 ///
 /// Returns true if new candidates were found.
-static Sema::OwningExprResult
+static ExprResult
 BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
                       UnresolvedLookupExpr *ULE,
                       SourceLocation LParenLoc,
@@ -6440,13 +6608,13 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
   LookupResult R(SemaRef, ULE->getName(), ULE->getNameLoc(),
                  Sema::LookupOrdinaryName);
   if (SemaRef.DiagnoseEmptyLookup(S, SS, R, Sema::CTC_Expression))
-    return Destroy(SemaRef, Fn, Args, NumArgs);
+    return ExprError();
 
   assert(!R.empty() && "lookup results empty despite recovery");
 
   // Build an implicit member call if appropriate.  Just drop the
   // casts and such from the call, we don't really care.
-  Sema::OwningExprResult NewFn = SemaRef.ExprError();
+  ExprResult NewFn = ExprError();
   if ((*R.begin())->isCXXClassMember())
     NewFn = SemaRef.BuildPossibleImplicitMemberExpr(SS, R, ExplicitTemplateArgs);
   else if (ExplicitTemplateArgs)
@@ -6455,15 +6623,13 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
     NewFn = SemaRef.BuildDeclarationNameExpr(SS, R, false);
 
   if (NewFn.isInvalid())
-    return Destroy(SemaRef, Fn, Args, NumArgs);
-
-  Fn->Destroy(SemaRef.Context);
+    return ExprError();
 
   // This shouldn't cause an infinite loop because we're giving it
   // an expression with non-empty lookup results, which should never
   // end up here.
-  return SemaRef.ActOnCallExpr(/*Scope*/ 0, move(NewFn), LParenLoc,
-                         Sema::MultiExprArg(SemaRef, (void**) Args, NumArgs),
+  return SemaRef.ActOnCallExpr(/*Scope*/ 0, NewFn.take(), LParenLoc,
+                               MultiExprArg(Args, NumArgs),
                                CommaLocs, RParenLoc);
 }
 
@@ -6474,7 +6640,7 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
 /// the function declaration produced by overload
 /// resolution. Otherwise, emits diagnostics, deletes all of the
 /// arguments and Fn, and returns NULL.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
                               SourceLocation LParenLoc,
                               Expr **Args, unsigned NumArgs,
@@ -6512,7 +6678,7 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
                                  CommaLocs, RParenLoc);
 
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, Fn->getLocStart(), Best)) {
+  switch (CandidateSet.BestViableFunction(*this, Fn->getLocStart(), Best)) {
   case OR_Success: {
     FunctionDecl *FDecl = Best->Function;
     CheckUnresolvedLookupAccess(ULE, Best->FoundDecl);
@@ -6525,13 +6691,13 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
     Diag(Fn->getSourceRange().getBegin(),
          diag::err_ovl_no_viable_function_in_call)
       << ULE->getName() << Fn->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
     break;
 
   case OR_Ambiguous:
     Diag(Fn->getSourceRange().getBegin(), diag::err_ovl_ambiguous_call)
       << ULE->getName() << Fn->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_ViableCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, NumArgs);
     break;
 
   case OR_Deleted:
@@ -6539,15 +6705,11 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
       << Best->Function->isDeleted()
       << ULE->getName()
       << Fn->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
     break;
   }
 
-  // Overload resolution failed. Destroy all of the subexpressions and
-  // return NULL.
-  Fn->Destroy(Context);
-  for (unsigned Arg = 0; Arg < NumArgs; ++Arg)
-    Args[Arg]->Destroy(Context);
+  // Overload resolution failed.
   return ExprError();
 }
 
@@ -6572,16 +6734,17 @@ static bool IsOverloaded(const UnresolvedSetImpl &Functions) {
 /// by CreateOverloadedUnaryOp().
 ///
 /// \param input The input argument.
-Sema::OwningExprResult
+ExprResult
 Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
                               const UnresolvedSetImpl &Fns,
-                              ExprArg input) {
+                              Expr *Input) {
   UnaryOperator::Opcode Opc = static_cast<UnaryOperator::Opcode>(OpcIn);
-  Expr *Input = (Expr *)input.get();
 
   OverloadedOperatorKind Op = UnaryOperator::getOverloadedOperator(Opc);
   assert(Op != OO_None && "Invalid opcode for overloaded unary operator");
   DeclarationName OpName = Context.DeclarationNames.getCXXOperatorName(Op);
+  // TODO: provide better source location info.
+  DeclarationNameInfo OpNameInfo(OpName, OpLoc);
 
   Expr *Args[2] = { Input, 0 };
   unsigned NumArgs = 1;
@@ -6589,16 +6752,16 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
   // For post-increment and post-decrement, add the implicit '0' as
   // the second argument, so that we know this is a post-increment or
   // post-decrement.
-  if (Opc == UnaryOperator::PostInc || Opc == UnaryOperator::PostDec) {
+  if (Opc == UO_PostInc || Opc == UO_PostDec) {
     llvm::APSInt Zero(Context.getTypeSize(Context.IntTy), false);
-    Args[1] = new (Context) IntegerLiteral(Zero, Context.IntTy,
-                                           SourceLocation());
+    Args[1] = IntegerLiteral::Create(Context, Zero, Context.IntTy,
+                                     SourceLocation());
     NumArgs = 2;
   }
 
   if (Input->isTypeDependent()) {
     if (Fns.empty())
-      return Owned(new (Context) UnaryOperator(input.takeAs<Expr>(),
+      return Owned(new (Context) UnaryOperator(Input,
                                                Opc, 
                                                Context.DependentTy,
                                                OpLoc));
@@ -6606,10 +6769,9 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
     CXXRecordDecl *NamingClass = 0; // because lookup ignores member operators
     UnresolvedLookupExpr *Fn
       = UnresolvedLookupExpr::Create(Context, /*Dependent*/ true, NamingClass,
-                                     0, SourceRange(), OpName, OpLoc,
+                                     0, SourceRange(), OpNameInfo,
                                      /*ADL*/ true, IsOverloaded(Fns),
                                      Fns.begin(), Fns.end());
-    input.release();
     return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn,
                                                    &Args[0], NumArgs,
                                                    Context.DependentTy,
@@ -6636,7 +6798,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
 
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, OpLoc, Best)) {
+  switch (CandidateSet.BestViableFunction(*this, OpLoc, Best)) {
   case OR_Success: {
     // We found a built-in operator or an overloaded operator.
     FunctionDecl *FnDecl = Best->Function;
@@ -6654,16 +6816,14 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
           return ExprError();
       } else {
         // Convert the arguments.
-        OwningExprResult InputInit
+        ExprResult InputInit
           = PerformCopyInitialization(InitializedEntity::InitializeParameter(
                                                       FnDecl->getParamDecl(0)),
                                       SourceLocation(), 
-                                      move(input));
+                                      Input);
         if (InputInit.isInvalid())
           return ExprError();
-        
-        input = move(InputInit);
-        Input = (Expr *)input.get();
+        Input = InputInit.take();
       }
 
       DiagnoseUseOfDecl(Best->FoundDecl, OpLoc);
@@ -6676,17 +6836,16 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
                                                SourceLocation());
       UsualUnaryConversions(FnExpr);
 
-      input.release();
       Args[0] = Input;
-      ExprOwningPtr<CallExpr> TheCall(this,
+      CallExpr *TheCall =
         new (Context) CXXOperatorCallExpr(Context, Op, FnExpr,
-                                          Args, NumArgs, ResultTy, OpLoc));
+                                          Args, NumArgs, ResultTy, OpLoc);
 
-      if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall.get(), 
+      if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall, 
                               FnDecl))
         return ExprError();
 
-      return MaybeBindToTemporary(TheCall.release());
+      return MaybeBindToTemporary(TheCall);
     } else {
       // We matched a built-in operator. Convert the arguments, then
       // break out so that we will build the appropriate built-in
@@ -6708,8 +6867,9 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
       Diag(OpLoc,  diag::err_ovl_ambiguous_oper)
           << UnaryOperator::getOpcodeStr(Opc)
           << Input->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_ViableCandidates, Args, NumArgs,
-                              UnaryOperator::getOpcodeStr(Opc), OpLoc);
+      CandidateSet.NoteCandidates(*this, OCD_ViableCandidates,
+                                  Args, NumArgs,
+                                  UnaryOperator::getOpcodeStr(Opc), OpLoc);
       return ExprError();
 
     case OR_Deleted:
@@ -6717,15 +6877,14 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
         << Best->Function->isDeleted()
         << UnaryOperator::getOpcodeStr(Opc)
         << Input->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
       return ExprError();
     }
 
   // Either we found no viable overloaded operator or we matched a
   // built-in operator. In either case, fall through to trying to
   // build a built-in operation.
-  input.release();
-  return CreateBuiltinUnaryOp(OpLoc, Opc, Owned(Input));
+  return CreateBuiltinUnaryOp(OpLoc, Opc, Input);
 }
 
 /// \brief Create a binary operation that may resolve to an overloaded
@@ -6745,7 +6904,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
 ///
 /// \param LHS Left-hand argument.
 /// \param RHS Right-hand argument.
-Sema::OwningExprResult
+ExprResult
 Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
                             unsigned OpcIn,
                             const UnresolvedSetImpl &Fns,
@@ -6763,7 +6922,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
     if (Fns.empty()) {
       // If there are no functions to store, just build a dependent 
       // BinaryOperator or CompoundAssignment.
-      if (Opc <= BinaryOperator::Assign || Opc > BinaryOperator::OrAssign)
+      if (Opc <= BO_Assign || Opc > BO_OrAssign)
         return Owned(new (Context) BinaryOperator(Args[0], Args[1], Opc,
                                                   Context.DependentTy, OpLoc));
       
@@ -6776,9 +6935,11 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
 
     // FIXME: save results of ADL from here?
     CXXRecordDecl *NamingClass = 0; // because lookup ignores member operators
+    // TODO: provide better source location info in DNLoc component.
+    DeclarationNameInfo OpNameInfo(OpName, OpLoc);
     UnresolvedLookupExpr *Fn
       = UnresolvedLookupExpr::Create(Context, /*Dependent*/ true, NamingClass,
-                                     0, SourceRange(), OpName, OpLoc,
+                                     0, SourceRange(), OpNameInfo,
                                      /*ADL*/ true, IsOverloaded(Fns),
                                      Fns.begin(), Fns.end());
     return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn,
@@ -6789,7 +6950,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
 
   // If this is the .* operator, which is not overloadable, just
   // create a built-in binary operator.
-  if (Opc == BinaryOperator::PtrMemD)
+  if (Opc == BO_PtrMemD)
     return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
 
   // If this is the assignment operator, we only perform overload resolution
@@ -6798,7 +6959,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
   // various built-in candidates, but as DR507 points out, this can lead to
   // problems. So we do it this way, which pretty much follows what GCC does.
   // Note that we go the traditional code path for compound assignment forms.
-  if (Opc==BinaryOperator::Assign && !Args[0]->getType()->isOverloadableType())
+  if (Opc == BO_Assign && !Args[0]->getType()->isOverloadableType())
     return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
 
   // Build an empty overload set.
@@ -6821,7 +6982,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
 
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, OpLoc, Best)) {
+  switch (CandidateSet.BestViableFunction(*this, OpLoc, Best)) {
     case OR_Success: {
       // We found a built-in operator or an overloaded operator.
       FunctionDecl *FnDecl = Best->Function;
@@ -6835,7 +6996,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           // Best->Access is only meaningful for class members.
           CheckMemberOperatorAccess(OpLoc, Args[0], Args[1], Best->FoundDecl);
 
-          OwningExprResult Arg1
+          ExprResult Arg1
             = PerformCopyInitialization(
                                         InitializedEntity::InitializeParameter(
                                                         FnDecl->getParamDecl(0)),
@@ -6851,7 +7012,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           Args[1] = RHS = Arg1.takeAs<Expr>();
         } else {
           // Convert the arguments.
-          OwningExprResult Arg0
+          ExprResult Arg0
             = PerformCopyInitialization(
                                         InitializedEntity::InitializeParameter(
                                                         FnDecl->getParamDecl(0)),
@@ -6860,7 +7021,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           if (Arg0.isInvalid())
             return ExprError();
 
-          OwningExprResult Arg1
+          ExprResult Arg1
             = PerformCopyInitialization(
                                         InitializedEntity::InitializeParameter(
                                                         FnDecl->getParamDecl(1)),
@@ -6884,16 +7045,15 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
                                                  OpLoc);
         UsualUnaryConversions(FnExpr);
 
-        ExprOwningPtr<CXXOperatorCallExpr> 
-          TheCall(this, new (Context) CXXOperatorCallExpr(Context, Op, FnExpr,
-                                                          Args, 2, ResultTy, 
-                                                          OpLoc));
+        CXXOperatorCallExpr *TheCall =
+          new (Context) CXXOperatorCallExpr(Context, Op, FnExpr,
+                                            Args, 2, ResultTy, OpLoc);
         
-        if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall.get(), 
+        if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall, 
                                 FnDecl))
           return ExprError();
 
-        return MaybeBindToTemporary(TheCall.release());
+        return MaybeBindToTemporary(TheCall);
       } else {
         // We matched a built-in operator. Convert the arguments, then
         // break out so that we will build the appropriate built-in
@@ -6913,15 +7073,15 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
       //   If the operator is the operator , [...] and there are no
       //   viable functions, then the operator is assumed to be the
       //   built-in operator and interpreted according to clause 5.
-      if (Opc == BinaryOperator::Comma)
+      if (Opc == BO_Comma)
         break;
 
       // For class as left operand for assignment or compound assigment operator
       // do not fall through to handling in built-in, but report that no overloaded
       // assignment operator found
-      OwningExprResult Result = ExprError();
+      ExprResult Result = ExprError();
       if (Args[0]->getType()->isRecordType() && 
-          Opc >= BinaryOperator::Assign && Opc <= BinaryOperator::OrAssign) {
+          Opc >= BO_Assign && Opc <= BO_OrAssign) {
         Diag(OpLoc,  diag::err_ovl_no_viable_oper)
              << BinaryOperator::getOpcodeStr(Opc)
              << Args[0]->getSourceRange() << Args[1]->getSourceRange();
@@ -6933,8 +7093,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
       assert(Result.isInvalid() && 
              "C++ binary operator overloading is missing candidates!");
       if (Result.isInvalid())
-        PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, 2,
-                                BinaryOperator::getOpcodeStr(Opc), OpLoc);
+        CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2,
+                                    BinaryOperator::getOpcodeStr(Opc), OpLoc);
       return move(Result);
     }
 
@@ -6942,8 +7102,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
       Diag(OpLoc,  diag::err_ovl_ambiguous_oper)
           << BinaryOperator::getOpcodeStr(Opc)
           << Args[0]->getSourceRange() << Args[1]->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_ViableCandidates, Args, 2,
-                              BinaryOperator::getOpcodeStr(Opc), OpLoc);
+      CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, 2,
+                                  BinaryOperator::getOpcodeStr(Opc), OpLoc);
       return ExprError();
 
     case OR_Deleted:
@@ -6951,7 +7111,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         << Best->Function->isDeleted()
         << BinaryOperator::getOpcodeStr(Opc)
         << Args[0]->getSourceRange() << Args[1]->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, 2);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2);
       return ExprError();
   }
 
@@ -6959,12 +7119,11 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
   return CreateBuiltinBinOp(OpLoc, Opc, Args[0], Args[1]);
 }
 
-Action::OwningExprResult
+ExprResult
 Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
                                          SourceLocation RLoc,
-                                         ExprArg Base, ExprArg Idx) {
-  Expr *Args[2] = { static_cast<Expr*>(Base.get()),
-                    static_cast<Expr*>(Idx.get()) };
+                                         Expr *Base, Expr *Idx) {
+  Expr *Args[2] = { Base, Idx };
   DeclarationName OpName =
       Context.DeclarationNames.getCXXOperatorName(OO_Subscript);
 
@@ -6973,16 +7132,17 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
   if (Args[0]->isTypeDependent() || Args[1]->isTypeDependent()) {
 
     CXXRecordDecl *NamingClass = 0; // because lookup ignores member operators
+    // CHECKME: no 'operator' keyword?
+    DeclarationNameInfo OpNameInfo(OpName, LLoc);
+    OpNameInfo.setCXXOperatorNameRange(SourceRange(LLoc, RLoc));
     UnresolvedLookupExpr *Fn
       = UnresolvedLookupExpr::Create(Context, /*Dependent*/ true, NamingClass,
-                                     0, SourceRange(), OpName, LLoc,
+                                     0, SourceRange(), OpNameInfo,
                                      /*ADL*/ true, /*Overloaded*/ false,
                                      UnresolvedSetIterator(),
                                      UnresolvedSetIterator());
     // Can't add any actual overloads yet
 
-    Base.release();
-    Idx.release();
     return Owned(new (Context) CXXOperatorCallExpr(Context, OO_Subscript, Fn,
                                                    Args, 2,
                                                    Context.DependentTy,
@@ -7002,7 +7162,7 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, LLoc, Best)) {
+  switch (CandidateSet.BestViableFunction(*this, LLoc, Best)) {
     case OR_Success: {
       // We found a built-in operator or an overloaded operator.
       FunctionDecl *FnDecl = Best->Function;
@@ -7021,7 +7181,7 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
           return ExprError();
 
         // Convert the arguments.
-        OwningExprResult InputInit
+        ExprResult InputInit
           = PerformCopyInitialization(InitializedEntity::InitializeParameter(
                                                       FnDecl->getParamDecl(0)),
                                       SourceLocation(), 
@@ -7041,18 +7201,16 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
                                                  LLoc);
         UsualUnaryConversions(FnExpr);
 
-        Base.release();
-        Idx.release();
-        ExprOwningPtr<CXXOperatorCallExpr>
-          TheCall(this, new (Context) CXXOperatorCallExpr(Context, OO_Subscript,
-                                                          FnExpr, Args, 2,
-                                                          ResultTy, RLoc));
+        CXXOperatorCallExpr *TheCall =
+          new (Context) CXXOperatorCallExpr(Context, OO_Subscript,
+                                            FnExpr, Args, 2,
+                                            ResultTy, RLoc);
 
-        if (CheckCallReturnType(FnDecl->getResultType(), LLoc, TheCall.get(),
+        if (CheckCallReturnType(FnDecl->getResultType(), LLoc, TheCall,
                                 FnDecl))
           return ExprError();
 
-        return MaybeBindToTemporary(TheCall.release());
+        return MaybeBindToTemporary(TheCall);
       } else {
         // We matched a built-in operator. Convert the arguments, then
         // break out so that we will build the appropriate built-in
@@ -7076,32 +7234,29 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
         Diag(LLoc, diag::err_ovl_no_viable_subscript)
           << Args[0]->getType()
           << Args[0]->getSourceRange() << Args[1]->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, 2,
-                              "[]", LLoc);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2,
+                                  "[]", LLoc);
       return ExprError();
     }
 
     case OR_Ambiguous:
       Diag(LLoc,  diag::err_ovl_ambiguous_oper)
           << "[]" << Args[0]->getSourceRange() << Args[1]->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_ViableCandidates, Args, 2,
-                              "[]", LLoc);
+      CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, 2,
+                                  "[]", LLoc);
       return ExprError();
 
     case OR_Deleted:
       Diag(LLoc, diag::err_ovl_deleted_oper)
         << Best->Function->isDeleted() << "[]"
         << Args[0]->getSourceRange() << Args[1]->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, 2,
-                              "[]", LLoc);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, 2,
+                                  "[]", LLoc);
       return ExprError();
     }
 
   // We matched a built-in operator; build it.
-  Base.release();
-  Idx.release();
-  return CreateBuiltinArraySubscriptExpr(Owned(Args[0]), LLoc,
-                                         Owned(Args[1]), RLoc);
+  return CreateBuiltinArraySubscriptExpr(Args[0], LLoc, Args[1], RLoc);
 }
 
 /// BuildCallToMemberFunction - Build a call to a member
@@ -7111,7 +7266,7 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 /// parameter). The caller needs to validate that the member
 /// expression refers to a member function or an overloaded member
 /// function.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
                                 SourceLocation LParenLoc, Expr **Args,
                                 unsigned NumArgs, SourceLocation *CommaLocs,
@@ -7174,7 +7329,8 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
     DeclarationName DeclName = UnresExpr->getMemberName();
 
     OverloadCandidateSet::iterator Best;
-    switch (BestViableFunction(CandidateSet, UnresExpr->getLocStart(), Best)) {
+    switch (CandidateSet.BestViableFunction(*this, UnresExpr->getLocStart(),
+                               Best)) {
     case OR_Success:
       Method = cast<CXXMethodDecl>(Best->Function);
       FoundDecl = Best->FoundDecl;
@@ -7186,14 +7342,14 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
       Diag(UnresExpr->getMemberLoc(),
            diag::err_ovl_no_viable_member_function_in_call)
         << DeclName << MemExprE->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
       // FIXME: Leaking incoming expressions!
       return ExprError();
 
     case OR_Ambiguous:
       Diag(UnresExpr->getMemberLoc(), diag::err_ovl_ambiguous_member_call)
         << DeclName << MemExprE->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
       // FIXME: Leaking incoming expressions!
       return ExprError();
 
@@ -7201,7 +7357,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
       Diag(UnresExpr->getMemberLoc(), diag::err_ovl_deleted_member_call)
         << Best->Function->isDeleted()
         << DeclName << MemExprE->getSourceRange();
-      PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+      CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
       // FIXME: Leaking incoming expressions!
       return ExprError();
     }
@@ -7219,15 +7375,14 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   }
 
   assert(Method && "Member call to something that isn't a method?");
-  ExprOwningPtr<CXXMemberCallExpr>
-    TheCall(this, new (Context) CXXMemberCallExpr(Context, MemExprE, Args,
-                                                  NumArgs,
-                                  Method->getCallResultType(),
-                                  RParenLoc));
+  CXXMemberCallExpr *TheCall = 
+    new (Context) CXXMemberCallExpr(Context, MemExprE, Args, NumArgs,
+                                    Method->getCallResultType(),
+                                    RParenLoc);
 
   // Check for a valid return type.
   if (CheckCallReturnType(Method->getResultType(), MemExpr->getMemberLoc(), 
-                          TheCall.get(), Method))
+                          TheCall, Method))
     return ExprError();
   
   // Convert the object argument (for a non-static member function call).
@@ -7242,21 +7397,21 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
 
   // Convert the rest of the arguments
   const FunctionProtoType *Proto = Method->getType()->getAs<FunctionProtoType>();
-  if (ConvertArgumentsForCall(&*TheCall, MemExpr, Method, Proto, Args, NumArgs,
+  if (ConvertArgumentsForCall(TheCall, MemExpr, Method, Proto, Args, NumArgs,
                               RParenLoc))
     return ExprError();
 
-  if (CheckFunctionCall(Method, TheCall.get()))
+  if (CheckFunctionCall(Method, TheCall))
     return ExprError();
 
-  return MaybeBindToTemporary(TheCall.release());
+  return MaybeBindToTemporary(TheCall);
 }
 
 /// BuildCallToObjectOfClassType - Build a call to an object of class
 /// type (C++ [over.call.object]), which can end up invoking an
 /// overloaded function call operator (@c operator()) or performing a
 /// user-defined conversion on the object argument.
-Sema::ExprResult
+ExprResult
 Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
                                    SourceLocation LParenLoc,
                                    Expr **Args, unsigned NumArgs,
@@ -7338,7 +7493,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
 
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, Object->getLocStart(), Best)) {
+  switch (CandidateSet.BestViableFunction(*this, Object->getLocStart(),
+                             Best)) {
   case OR_Success:
     // Overload resolution succeeded; we'll build the appropriate call
     // below.
@@ -7353,14 +7509,14 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
       Diag(Object->getSourceRange().getBegin(),
            diag::err_ovl_no_viable_object_call)
         << Object->getType() << Object->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
     break;
 
   case OR_Ambiguous:
     Diag(Object->getSourceRange().getBegin(),
          diag::err_ovl_ambiguous_object_call)
       << Object->getType() << Object->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_ViableCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, Args, NumArgs);
     break;
 
   case OR_Deleted:
@@ -7368,18 +7524,12 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
          diag::err_ovl_deleted_object_call)
       << Best->Function->isDeleted()
       << Object->getType() << Object->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, Args, NumArgs);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args, NumArgs);
     break;
   }
 
-  if (Best == CandidateSet.end()) {
-    // We had an error; delete all of the subexpressions and return
-    // the error.
-    Object->Destroy(Context);
-    for (unsigned ArgIdx = 0; ArgIdx < NumArgs; ++ArgIdx)
-      Args[ArgIdx]->Destroy(Context);
+  if (Best == CandidateSet.end())
     return true;
-  }
 
   if (Best->Function == 0) {
     // Since there is no function declaration, this is one of the
@@ -7400,9 +7550,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
     CXXMemberCallExpr *CE = BuildCXXMemberCallExpr(Object, Best->FoundDecl,
                                                    Conv);
       
-    return ActOnCallExpr(S, ExprArg(*this, CE), LParenLoc,
-                         MultiExprArg(*this, (ExprTy**)Args, NumArgs),
-                         CommaLocs, RParenLoc).result();
+    return ActOnCallExpr(S, CE, LParenLoc, MultiExprArg(Args, NumArgs),
+                         CommaLocs, RParenLoc);
   }
 
   CheckMemberOperatorAccess(LParenLoc, Object, 0, Best->FoundDecl);
@@ -7438,13 +7587,13 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
   // Once we've built TheCall, all of the expressions are properly
   // owned.
   QualType ResultTy = Method->getCallResultType();
-  ExprOwningPtr<CXXOperatorCallExpr>
-    TheCall(this, new (Context) CXXOperatorCallExpr(Context, OO_Call, NewFn,
-                                                    MethodArgs, NumArgs + 1,
-                                                    ResultTy, RParenLoc));
+  CXXOperatorCallExpr *TheCall =
+    new (Context) CXXOperatorCallExpr(Context, OO_Call, NewFn,
+                                      MethodArgs, NumArgs + 1,
+                                      ResultTy, RParenLoc);
   delete [] MethodArgs;
 
-  if (CheckCallReturnType(Method->getResultType(), LParenLoc, TheCall.get(), 
+  if (CheckCallReturnType(Method->getResultType(), LParenLoc, TheCall, 
                           Method))
     return true;
   
@@ -7471,15 +7620,15 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
 
       // Pass the argument.
 
-      OwningExprResult InputInit
+      ExprResult InputInit
         = PerformCopyInitialization(InitializedEntity::InitializeParameter(
                                                     Method->getParamDecl(i)),
-                                    SourceLocation(), Owned(Arg));
+                                    SourceLocation(), Arg);
       
       IsError |= InputInit.isInvalid();
       Arg = InputInit.takeAs<Expr>();
     } else {
-      OwningExprResult DefArg
+      ExprResult DefArg
         = BuildCXXDefaultArgExpr(LParenLoc, Method, Method->getParamDecl(i));
       if (DefArg.isInvalid()) {
         IsError = true;
@@ -7504,18 +7653,17 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Object,
 
   if (IsError) return true;
 
-  if (CheckFunctionCall(Method, TheCall.get()))
+  if (CheckFunctionCall(Method, TheCall))
     return true;
 
-  return MaybeBindToTemporary(TheCall.release()).result();
+  return MaybeBindToTemporary(TheCall);
 }
 
 /// BuildOverloadedArrowExpr - Build a call to an overloaded @c operator->
 ///  (if one exists), where @c Base is an expression of class type and
 /// @c Member is the name of the member we're trying to find.
-Sema::OwningExprResult
-Sema::BuildOverloadedArrowExpr(Scope *S, ExprArg BaseIn, SourceLocation OpLoc) {
-  Expr *Base = static_cast<Expr *>(BaseIn.get());
+ExprResult
+Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
   assert(Base->getType()->isRecordType() && "left-hand side must have class type");
 
   SourceLocation Loc = Base->getExprLoc();
@@ -7547,7 +7695,7 @@ Sema::BuildOverloadedArrowExpr(Scope *S, ExprArg BaseIn, SourceLocation OpLoc) {
 
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
-  switch (BestViableFunction(CandidateSet, OpLoc, Best)) {
+  switch (CandidateSet.BestViableFunction(*this, OpLoc, Best)) {
   case OR_Success:
     // Overload resolution succeeded; we'll build the call below.
     break;
@@ -7559,20 +7707,20 @@ Sema::BuildOverloadedArrowExpr(Scope *S, ExprArg BaseIn, SourceLocation OpLoc) {
     else
       Diag(OpLoc, diag::err_ovl_no_viable_oper)
         << "operator->" << Base->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, &Base, 1);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, &Base, 1);
     return ExprError();
 
   case OR_Ambiguous:
     Diag(OpLoc,  diag::err_ovl_ambiguous_oper)
       << "->" << Base->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_ViableCandidates, &Base, 1);
+    CandidateSet.NoteCandidates(*this, OCD_ViableCandidates, &Base, 1);
     return ExprError();
 
   case OR_Deleted:
     Diag(OpLoc,  diag::err_ovl_deleted_oper)
       << Best->Function->isDeleted()
       << "->" << Base->getSourceRange();
-    PrintOverloadCandidates(CandidateSet, OCD_AllCandidates, &Base, 1);
+    CandidateSet.NoteCandidates(*this, OCD_AllCandidates, &Base, 1);
     return ExprError();
   }
 
@@ -7585,23 +7733,20 @@ Sema::BuildOverloadedArrowExpr(Scope *S, ExprArg BaseIn, SourceLocation OpLoc) {
                                           Best->FoundDecl, Method))
     return ExprError();
 
-  // No concerns about early exits now.
-  BaseIn.release();
-
   // Build the operator call.
   Expr *FnExpr = new (Context) DeclRefExpr(Method, Method->getType(),
                                            SourceLocation());
   UsualUnaryConversions(FnExpr);
   
   QualType ResultTy = Method->getCallResultType();
-  ExprOwningPtr<CXXOperatorCallExpr> 
-    TheCall(this, new (Context) CXXOperatorCallExpr(Context, OO_Arrow, FnExpr, 
-                                                    &Base, 1, ResultTy, OpLoc));
+  CXXOperatorCallExpr *TheCall =
+    new (Context) CXXOperatorCallExpr(Context, OO_Arrow, FnExpr, 
+                                      &Base, 1, ResultTy, OpLoc);
 
-  if (CheckCallReturnType(Method->getResultType(), OpLoc, TheCall.get(), 
+  if (CheckCallReturnType(Method->getResultType(), OpLoc, TheCall, 
                           Method))
           return ExprError();
-  return move(TheCall);
+  return Owned(TheCall);
 }
 
 /// FixOverloadedFunctionReference - E is an expression that refers to
@@ -7626,17 +7771,18 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
     assert(Context.hasSameType(ICE->getSubExpr()->getType(), 
                                SubExpr->getType()) &&
            "Implicit cast type cannot be determined from overload");
+    assert(ICE->path_empty() && "fixing up hierarchy conversion?");
     if (SubExpr == ICE->getSubExpr())
       return ICE->Retain();
     
-    return new (Context) ImplicitCastExpr(ICE->getType(), 
-                                          ICE->getCastKind(),
-                                          SubExpr, CXXBaseSpecifierArray(),
-                                          ICE->isLvalueCast());
+    return ImplicitCastExpr::Create(Context, ICE->getType(), 
+                                    ICE->getCastKind(),
+                                    SubExpr, 0,
+                                    ICE->getValueKind());
   } 
   
   if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(E)) {
-    assert(UnOp->getOpcode() == UnaryOperator::AddrOf &&
+    assert(UnOp->getOpcode() == UO_AddrOf &&
            "Can only take the address of an overloaded function");
     if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Fn)) {
       if (Method->isStatic()) {
@@ -7664,7 +7810,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
         QualType MemPtrType
           = Context.getMemberPointerType(Fn->getType(), ClassType.getTypePtr());
 
-        return new (Context) UnaryOperator(SubExpr, UnaryOperator::AddrOf,
+        return new (Context) UnaryOperator(SubExpr, UO_AddrOf,
                                            MemPtrType, UnOp->getOperatorLoc());
       }
     }
@@ -7673,7 +7819,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
     if (SubExpr == UnOp->getSubExpr())
       return UnOp->Retain();
     
-    return new (Context) UnaryOperator(SubExpr, UnaryOperator::AddrOf,
+    return new (Context) UnaryOperator(SubExpr, UO_AddrOf,
                                      Context.getPointerType(SubExpr->getType()),
                                        UnOp->getOperatorLoc());
   } 
@@ -7732,7 +7878,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
                               MemExpr->getQualifierRange(),
                               Fn, 
                               Found,
-                              MemExpr->getMemberLoc(),
+                              MemExpr->getMemberNameInfo(),
                               TemplateArgs,
                               Fn->getType());
   }
@@ -7741,9 +7887,9 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
   return E->Retain();
 }
 
-Sema::OwningExprResult Sema::FixOverloadedFunctionReference(OwningExprResult E, 
-                                                          DeclAccessPair Found,
-                                                            FunctionDecl *Fn) {
+ExprResult Sema::FixOverloadedFunctionReference(ExprResult E, 
+                                                DeclAccessPair Found,
+                                                FunctionDecl *Fn) {
   return Owned(FixOverloadedFunctionReference((Expr *)E.get(), Found, Fn));
 }
 
diff --git a/lib/Sema/SemaOverload.h b/lib/Sema/SemaOverload.h
deleted file mode 100644
index eb4fc6581796..000000000000
--- a/lib/Sema/SemaOverload.h
+++ /dev/null
@@ -1,617 +0,0 @@
-//===--- Overload.h - C++ Overloading ---------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the data structures and types used in C++
-// overload resolution.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_SEMA_OVERLOAD_H
-#define LLVM_CLANG_SEMA_OVERLOAD_H
-
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclTemplate.h"
-#include "clang/AST/Expr.h"
-#include "clang/AST/TemplateBase.h"
-#include "clang/AST/Type.h"
-#include "clang/AST/UnresolvedSet.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallVector.h"
-
-namespace clang {
-  class ASTContext;
-  class CXXConstructorDecl;
-  class CXXConversionDecl;
-  class FunctionDecl;
-
-  /// OverloadingResult - Capture the result of performing overload
-  /// resolution.
-  enum OverloadingResult {
-    OR_Success,             ///< Overload resolution succeeded.
-    OR_No_Viable_Function,  ///< No viable function found.
-    OR_Ambiguous,           ///< Ambiguous candidates found.
-    OR_Deleted              ///< Succeeded, but refers to a deleted function.
-  };
-    
-  /// ImplicitConversionKind - The kind of implicit conversion used to
-  /// convert an argument to a parameter's type. The enumerator values
-  /// match with Table 9 of (C++ 13.3.3.1.1) and are listed such that
-  /// better conversion kinds have smaller values.
-  enum ImplicitConversionKind {
-    ICK_Identity = 0,          ///< Identity conversion (no conversion)
-    ICK_Lvalue_To_Rvalue,      ///< Lvalue-to-rvalue conversion (C++ 4.1)
-    ICK_Array_To_Pointer,      ///< Array-to-pointer conversion (C++ 4.2)
-    ICK_Function_To_Pointer,   ///< Function-to-pointer (C++ 4.3)
-    ICK_NoReturn_Adjustment,   ///< Removal of noreturn from a type (Clang)
-    ICK_Qualification,         ///< Qualification conversions (C++ 4.4)
-    ICK_Integral_Promotion,    ///< Integral promotions (C++ 4.5)
-    ICK_Floating_Promotion,    ///< Floating point promotions (C++ 4.6)
-    ICK_Complex_Promotion,     ///< Complex promotions (Clang extension)
-    ICK_Integral_Conversion,   ///< Integral conversions (C++ 4.7)
-    ICK_Floating_Conversion,   ///< Floating point conversions (C++ 4.8)
-    ICK_Complex_Conversion,    ///< Complex conversions (C99 6.3.1.6)
-    ICK_Floating_Integral,     ///< Floating-integral conversions (C++ 4.9)
-    ICK_Pointer_Conversion,    ///< Pointer conversions (C++ 4.10)
-    ICK_Pointer_Member,        ///< Pointer-to-member conversions (C++ 4.11)
-    ICK_Boolean_Conversion,    ///< Boolean conversions (C++ 4.12)
-    ICK_Compatible_Conversion, ///< Conversions between compatible types in C99
-    ICK_Derived_To_Base,       ///< Derived-to-base (C++ [over.best.ics])
-    ICK_Vector_Conversion,     ///< Vector conversions
-    ICK_Vector_Splat,          ///< A vector splat from an arithmetic type
-    ICK_Complex_Real,          ///< Complex-real conversions (C99 6.3.1.7)
-    ICK_Num_Conversion_Kinds   ///< The number of conversion kinds
-  };
-
-  /// ImplicitConversionCategory - The category of an implicit
-  /// conversion kind. The enumerator values match with Table 9 of
-  /// (C++ 13.3.3.1.1) and are listed such that better conversion
-  /// categories have smaller values.
-  enum ImplicitConversionCategory {
-    ICC_Identity = 0,              ///< Identity
-    ICC_Lvalue_Transformation,     ///< Lvalue transformation
-    ICC_Qualification_Adjustment,  ///< Qualification adjustment
-    ICC_Promotion,                 ///< Promotion
-    ICC_Conversion                 ///< Conversion
-  };
-
-  ImplicitConversionCategory
-  GetConversionCategory(ImplicitConversionKind Kind);
-
-  /// ImplicitConversionRank - The rank of an implicit conversion
-  /// kind. The enumerator values match with Table 9 of (C++
-  /// 13.3.3.1.1) and are listed such that better conversion ranks
-  /// have smaller values.
-  enum ImplicitConversionRank {
-    ICR_Exact_Match = 0,        ///< Exact Match
-    ICR_Promotion,              ///< Promotion
-    ICR_Conversion,             ///< Conversion
-    ICR_Complex_Real_Conversion ///< Complex <-> Real conversion
-  };
-
-  ImplicitConversionRank GetConversionRank(ImplicitConversionKind Kind);
-
-  /// StandardConversionSequence - represents a standard conversion
-  /// sequence (C++ 13.3.3.1.1). A standard conversion sequence
-  /// contains between zero and three conversions. If a particular
-  /// conversion is not needed, it will be set to the identity conversion
-  /// (ICK_Identity). Note that the three conversions are
-  /// specified as separate members (rather than in an array) so that
-  /// we can keep the size of a standard conversion sequence to a
-  /// single word.
-  struct StandardConversionSequence {
-    /// First -- The first conversion can be an lvalue-to-rvalue
-    /// conversion, array-to-pointer conversion, or
-    /// function-to-pointer conversion.
-    ImplicitConversionKind First : 8;
-
-    /// Second - The second conversion can be an integral promotion,
-    /// floating point promotion, integral conversion, floating point
-    /// conversion, floating-integral conversion, pointer conversion,
-    /// pointer-to-member conversion, or boolean conversion.
-    ImplicitConversionKind Second : 8;
-
-    /// Third - The third conversion can be a qualification conversion.
-    ImplicitConversionKind Third : 8;
-
-    /// Deprecated - Whether this the deprecated conversion of a
-    /// string literal to a pointer to non-const character data
-    /// (C++ 4.2p2).
-    bool DeprecatedStringLiteralToCharPtr : 1;
-
-    /// IncompatibleObjC - Whether this is an Objective-C conversion
-    /// that we should warn about (if we actually use it).
-    bool IncompatibleObjC : 1;
-
-    /// ReferenceBinding - True when this is a reference binding
-    /// (C++ [over.ics.ref]).
-    bool ReferenceBinding : 1;
-
-    /// DirectBinding - True when this is a reference binding that is a
-    /// direct binding (C++ [dcl.init.ref]).
-    bool DirectBinding : 1;
-
-    /// RRefBinding - True when this is a reference binding of an rvalue
-    /// reference to an rvalue (C++0x [over.ics.rank]p3b4).
-    bool RRefBinding : 1;
-
-    /// FromType - The type that this conversion is converting
-    /// from. This is an opaque pointer that can be translated into a
-    /// QualType.
-    void *FromTypePtr;
-
-    /// ToType - The types that this conversion is converting to in
-    /// each step. This is an opaque pointer that can be translated
-    /// into a QualType.
-    void *ToTypePtrs[3];
-
-    /// CopyConstructor - The copy constructor that is used to perform
-    /// this conversion, when the conversion is actually just the
-    /// initialization of an object via copy constructor. Such
-    /// conversions are either identity conversions or derived-to-base
-    /// conversions.
-    CXXConstructorDecl *CopyConstructor;
-
-    void setFromType(QualType T) { FromTypePtr = T.getAsOpaquePtr(); }
-    void setToType(unsigned Idx, QualType T) { 
-      assert(Idx < 3 && "To type index is out of range");
-      ToTypePtrs[Idx] = T.getAsOpaquePtr(); 
-    }
-    void setAllToTypes(QualType T) {
-      ToTypePtrs[0] = T.getAsOpaquePtr(); 
-      ToTypePtrs[1] = ToTypePtrs[0];
-      ToTypePtrs[2] = ToTypePtrs[0];
-    }
-
-    QualType getFromType() const {
-      return QualType::getFromOpaquePtr(FromTypePtr);
-    }
-    QualType getToType(unsigned Idx) const {
-      assert(Idx < 3 && "To type index is out of range");
-      return QualType::getFromOpaquePtr(ToTypePtrs[Idx]);
-    }
-
-    void setAsIdentityConversion();
-    
-    bool isIdentityConversion() const {
-      return First == ICK_Identity && Second == ICK_Identity && 
-             Third == ICK_Identity;
-    }
-    
-    ImplicitConversionRank getRank() const;
-    bool isPointerConversionToBool() const;
-    bool isPointerConversionToVoidPointer(ASTContext& Context) const;
-    void DebugPrint() const;
-  };
-
-  /// UserDefinedConversionSequence - Represents a user-defined
-  /// conversion sequence (C++ 13.3.3.1.2).
-  struct UserDefinedConversionSequence {
-    /// Before - Represents the standard conversion that occurs before
-    /// the actual user-defined conversion. (C++ 13.3.3.1.2p1):
-    ///
-    ///   If the user-defined conversion is specified by a constructor
-    ///   (12.3.1), the initial standard conversion sequence converts
-    ///   the source type to the type required by the argument of the
-    ///   constructor. If the user-defined conversion is specified by
-    ///   a conversion function (12.3.2), the initial standard
-    ///   conversion sequence converts the source type to the implicit
-    ///   object parameter of the conversion function.
-    StandardConversionSequence Before;
-
-    /// EllipsisConversion - When this is true, it means user-defined
-    /// conversion sequence starts with a ... (elipsis) conversion, instead of 
-    /// a standard conversion. In this case, 'Before' field must be ignored.
-    // FIXME. I much rather put this as the first field. But there seems to be
-    // a gcc code gen. bug which causes a crash in a test. Putting it here seems
-    // to work around the crash.
-    bool EllipsisConversion : 1;
-    
-    /// After - Represents the standard conversion that occurs after
-    /// the actual user-defined conversion.
-    StandardConversionSequence After;
-
-    /// ConversionFunction - The function that will perform the
-    /// user-defined conversion.
-    FunctionDecl* ConversionFunction;
-
-    void DebugPrint() const;
-  };
-
-  /// Represents an ambiguous user-defined conversion sequence.
-  struct AmbiguousConversionSequence {
-    typedef llvm::SmallVector<FunctionDecl*, 4> ConversionSet;
-
-    void *FromTypePtr;
-    void *ToTypePtr;
-    char Buffer[sizeof(ConversionSet)];
-
-    QualType getFromType() const {
-      return QualType::getFromOpaquePtr(FromTypePtr);
-    }
-    QualType getToType() const {
-      return QualType::getFromOpaquePtr(ToTypePtr);
-    }
-    void setFromType(QualType T) { FromTypePtr = T.getAsOpaquePtr(); }
-    void setToType(QualType T) { ToTypePtr = T.getAsOpaquePtr(); }
-
-    ConversionSet &conversions() {
-      return *reinterpret_cast<ConversionSet*>(Buffer);
-    }
-
-    const ConversionSet &conversions() const {
-      return *reinterpret_cast<const ConversionSet*>(Buffer);
-    }
-
-    void addConversion(FunctionDecl *D) {
-      conversions().push_back(D);
-    }
-
-    typedef ConversionSet::iterator iterator;
-    iterator begin() { return conversions().begin(); }
-    iterator end() { return conversions().end(); }
-
-    typedef ConversionSet::const_iterator const_iterator;
-    const_iterator begin() const { return conversions().begin(); }
-    const_iterator end() const { return conversions().end(); }
-
-    void construct();
-    void destruct();
-    void copyFrom(const AmbiguousConversionSequence &);
-  };
-
-  /// BadConversionSequence - Records information about an invalid
-  /// conversion sequence.
-  struct BadConversionSequence {
-    enum FailureKind {
-      no_conversion,
-      unrelated_class,
-      suppressed_user,
-      bad_qualifiers
-    };
-
-    // This can be null, e.g. for implicit object arguments.
-    Expr *FromExpr;
-
-    FailureKind Kind;
-
-  private:
-    // The type we're converting from (an opaque QualType).
-    void *FromTy;
-
-    // The type we're converting to (an opaque QualType).
-    void *ToTy;
-
-  public:
-    void init(FailureKind K, Expr *From, QualType To) {
-      init(K, From->getType(), To);
-      FromExpr = From;
-    }
-    void init(FailureKind K, QualType From, QualType To) {
-      Kind = K;
-      FromExpr = 0;
-      setFromType(From);
-      setToType(To);
-    }
-
-    QualType getFromType() const { return QualType::getFromOpaquePtr(FromTy); }
-    QualType getToType() const { return QualType::getFromOpaquePtr(ToTy); }
-
-    void setFromExpr(Expr *E) {
-      FromExpr = E;
-      setFromType(E->getType());
-    }
-    void setFromType(QualType T) { FromTy = T.getAsOpaquePtr(); }
-    void setToType(QualType T) { ToTy = T.getAsOpaquePtr(); }
-  };
-
-  /// ImplicitConversionSequence - Represents an implicit conversion
-  /// sequence, which may be a standard conversion sequence
-  /// (C++ 13.3.3.1.1), user-defined conversion sequence (C++ 13.3.3.1.2),
-  /// or an ellipsis conversion sequence (C++ 13.3.3.1.3).
-  struct ImplicitConversionSequence {
-    /// Kind - The kind of implicit conversion sequence. BadConversion
-    /// specifies that there is no conversion from the source type to
-    /// the target type.  AmbiguousConversion represents the unique
-    /// ambiguous conversion (C++0x [over.best.ics]p10).
-    enum Kind {
-      StandardConversion = 0,
-      UserDefinedConversion,
-      AmbiguousConversion,
-      EllipsisConversion,
-      BadConversion
-    };
-
-  private:
-    enum {
-      Uninitialized = BadConversion + 1
-    };
-
-    /// ConversionKind - The kind of implicit conversion sequence.
-    unsigned ConversionKind;
-
-    void setKind(Kind K) {
-      destruct();
-      ConversionKind = K;
-    }
-
-    void destruct() {
-      if (ConversionKind == AmbiguousConversion) Ambiguous.destruct();
-    }
-
-  public:
-    union {
-      /// When ConversionKind == StandardConversion, provides the
-      /// details of the standard conversion sequence.
-      StandardConversionSequence Standard;
-
-      /// When ConversionKind == UserDefinedConversion, provides the
-      /// details of the user-defined conversion sequence.
-      UserDefinedConversionSequence UserDefined;
-
-      /// When ConversionKind == AmbiguousConversion, provides the
-      /// details of the ambiguous conversion.
-      AmbiguousConversionSequence Ambiguous;
-
-      /// When ConversionKind == BadConversion, provides the details
-      /// of the bad conversion.
-      BadConversionSequence Bad;
-    };
-
-    ImplicitConversionSequence() : ConversionKind(Uninitialized) {}
-    ~ImplicitConversionSequence() {
-      destruct();
-    }
-    ImplicitConversionSequence(const ImplicitConversionSequence &Other)
-      : ConversionKind(Other.ConversionKind)
-    {
-      switch (ConversionKind) {
-      case Uninitialized: break;
-      case StandardConversion: Standard = Other.Standard; break;
-      case UserDefinedConversion: UserDefined = Other.UserDefined; break;
-      case AmbiguousConversion: Ambiguous.copyFrom(Other.Ambiguous); break;
-      case EllipsisConversion: break;
-      case BadConversion: Bad = Other.Bad; break;
-      }
-    }
-
-    ImplicitConversionSequence &
-        operator=(const ImplicitConversionSequence &Other) {
-      destruct();
-      new (this) ImplicitConversionSequence(Other);
-      return *this;
-    }
-    
-    Kind getKind() const {
-      assert(isInitialized() && "querying uninitialized conversion");
-      return Kind(ConversionKind);
-    }
-    
-    /// \brief Return a ranking of the implicit conversion sequence
-    /// kind, where smaller ranks represent better conversion
-    /// sequences.
-    ///
-    /// In particular, this routine gives user-defined conversion
-    /// sequences and ambiguous conversion sequences the same rank,
-    /// per C++ [over.best.ics]p10.
-    unsigned getKindRank() const {
-      switch (getKind()) {
-      case StandardConversion: 
-        return 0;
-
-      case UserDefinedConversion:
-      case AmbiguousConversion: 
-        return 1;
-
-      case EllipsisConversion:
-        return 2;
-
-      case BadConversion:
-        return 3;
-      }
-
-      return 3;
-    }
-
-    bool isBad() const { return getKind() == BadConversion; }
-    bool isStandard() const { return getKind() == StandardConversion; }
-    bool isEllipsis() const { return getKind() == EllipsisConversion; }
-    bool isAmbiguous() const { return getKind() == AmbiguousConversion; }
-    bool isUserDefined() const { return getKind() == UserDefinedConversion; }
-
-    /// Determines whether this conversion sequence has been
-    /// initialized.  Most operations should never need to query
-    /// uninitialized conversions and should assert as above.
-    bool isInitialized() const { return ConversionKind != Uninitialized; }
-
-    /// Sets this sequence as a bad conversion for an explicit argument.
-    void setBad(BadConversionSequence::FailureKind Failure,
-                Expr *FromExpr, QualType ToType) {
-      setKind(BadConversion);
-      Bad.init(Failure, FromExpr, ToType);
-    }
-
-    /// Sets this sequence as a bad conversion for an implicit argument.
-    void setBad(BadConversionSequence::FailureKind Failure,
-                QualType FromType, QualType ToType) {
-      setKind(BadConversion);
-      Bad.init(Failure, FromType, ToType);
-    }
-
-    void setStandard() { setKind(StandardConversion); }
-    void setEllipsis() { setKind(EllipsisConversion); }
-    void setUserDefined() { setKind(UserDefinedConversion); }
-    void setAmbiguous() {
-      if (ConversionKind == AmbiguousConversion) return;
-      ConversionKind = AmbiguousConversion;
-      Ambiguous.construct();
-    }
-
-    // The result of a comparison between implicit conversion
-    // sequences. Use Sema::CompareImplicitConversionSequences to
-    // actually perform the comparison.
-    enum CompareKind {
-      Better = -1,
-      Indistinguishable = 0,
-      Worse = 1
-    };
-
-    void DebugPrint() const;
-  };
-
-  enum OverloadFailureKind {
-    ovl_fail_too_many_arguments,
-    ovl_fail_too_few_arguments,
-    ovl_fail_bad_conversion,
-    ovl_fail_bad_deduction,
-
-    /// This conversion candidate was not considered because it
-    /// duplicates the work of a trivial or derived-to-base
-    /// conversion.
-    ovl_fail_trivial_conversion,
-
-    /// This conversion candidate is not viable because its result
-    /// type is not implicitly convertible to the desired type.
-    ovl_fail_bad_final_conversion,
-    
-    /// This conversion function template specialization candidate is not 
-    /// viable because the final conversion was not an exact match.
-    ovl_fail_final_conversion_not_exact
-  };
-
-  /// OverloadCandidate - A single candidate in an overload set (C++ 13.3).
-  struct OverloadCandidate {
-    /// Function - The actual function that this candidate
-    /// represents. When NULL, this is a built-in candidate
-    /// (C++ [over.oper]) or a surrogate for a conversion to a
-    /// function pointer or reference (C++ [over.call.object]).
-    FunctionDecl *Function;
-
-    /// FoundDecl - The original declaration that was looked up /
-    /// invented / otherwise found, together with its access.
-    /// Might be a UsingShadowDecl or a FunctionTemplateDecl.
-    DeclAccessPair FoundDecl;
-
-    // BuiltinTypes - Provides the return and parameter types of a
-    // built-in overload candidate. Only valid when Function is NULL.
-    struct {
-      QualType ResultTy;
-      QualType ParamTypes[3];
-    } BuiltinTypes;
-
-    /// Surrogate - The conversion function for which this candidate
-    /// is a surrogate, but only if IsSurrogate is true.
-    CXXConversionDecl *Surrogate;
-
-    /// Conversions - The conversion sequences used to convert the
-    /// function arguments to the function parameters.
-    llvm::SmallVector<ImplicitConversionSequence, 4> Conversions;
-
-    /// Viable - True to indicate that this overload candidate is viable.
-    bool Viable;
-
-    /// IsSurrogate - True to indicate that this candidate is a
-    /// surrogate for a conversion to a function pointer or reference
-    /// (C++ [over.call.object]).
-    bool IsSurrogate;
-
-    /// IgnoreObjectArgument - True to indicate that the first
-    /// argument's conversion, which for this function represents the
-    /// implicit object argument, should be ignored. This will be true
-    /// when the candidate is a static member function (where the
-    /// implicit object argument is just a placeholder) or a
-    /// non-static member function when the call doesn't have an
-    /// object argument.
-    bool IgnoreObjectArgument;
-
-    /// FailureKind - The reason why this candidate is not viable.
-    /// Actually an OverloadFailureKind.
-    unsigned char FailureKind;
-
-    /// A structure used to record information about a failed
-    /// template argument deduction.
-    struct DeductionFailureInfo {
-      // A Sema::TemplateDeductionResult.
-      unsigned Result;
-
-      /// \brief Opaque pointer containing additional data about
-      /// this deduction failure.
-      void *Data;
-      
-      /// \brief Retrieve the template parameter this deduction failure
-      /// refers to, if any.
-      TemplateParameter getTemplateParameter();
-      
-      /// \brief Retrieve the template argument list associated with this
-      /// deduction failure, if any.
-      TemplateArgumentList *getTemplateArgumentList();
-      
-      /// \brief Return the first template argument this deduction failure
-      /// refers to, if any.
-      const TemplateArgument *getFirstArg();
-
-      /// \brief Return the second template argument this deduction failure
-      /// refers to, if any.
-      const TemplateArgument *getSecondArg();
-      
-      /// \brief Free any memory associated with this deduction failure.
-      void Destroy();
-    };
-
-    union {
-      DeductionFailureInfo DeductionFailure;
-      
-      /// FinalConversion - For a conversion function (where Function is
-      /// a CXXConversionDecl), the standard conversion that occurs
-      /// after the call to the overload candidate to convert the result
-      /// of calling the conversion function to the required type.
-      StandardConversionSequence FinalConversion;
-    };
-
-    /// hasAmbiguousConversion - Returns whether this overload
-    /// candidate requires an ambiguous conversion or not.
-    bool hasAmbiguousConversion() const {
-      for (llvm::SmallVectorImpl<ImplicitConversionSequence>::const_iterator
-             I = Conversions.begin(), E = Conversions.end(); I != E; ++I) {
-        if (!I->isInitialized()) return false;
-        if (I->isAmbiguous()) return true;
-      }
-      return false;
-    }
-  };
-
-  /// OverloadCandidateSet - A set of overload candidates, used in C++
-  /// overload resolution (C++ 13.3).
-  class OverloadCandidateSet : public llvm::SmallVector<OverloadCandidate, 16> {
-    typedef llvm::SmallVector<OverloadCandidate, 16> inherited;
-    llvm::SmallPtrSet<Decl *, 16> Functions;
-
-    SourceLocation Loc;    
-    
-    OverloadCandidateSet(const OverloadCandidateSet &);
-    OverloadCandidateSet &operator=(const OverloadCandidateSet &);
-    
-  public:
-    OverloadCandidateSet(SourceLocation Loc) : Loc(Loc) {}
-
-    SourceLocation getLocation() const { return Loc; }
-
-    /// \brief Determine when this overload candidate will be new to the
-    /// overload set.
-    bool isNewCandidate(Decl *F) { 
-      return Functions.insert(F->getCanonicalDecl()); 
-    }
-
-    /// \brief Clear out all of the candidates.
-    void clear();
-    
-    ~OverloadCandidateSet() { clear(); }
-  };
-} // end namespace clang
-
-#endif // LLVM_CLANG_SEMA_OVERLOAD_H
diff --git a/lib/Sema/SemaStmt.cpp b/lib/Sema/SemaStmt.cpp
index 9c8f48bfea17..083e4dba85e3 100644
--- a/lib/Sema/SemaStmt.cpp
+++ b/lib/Sema/SemaStmt.cpp
@@ -11,8 +11,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
-#include "SemaInit.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Initialization.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclObjC.h"
@@ -26,19 +28,11 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 using namespace clang;
+using namespace sema;
 
-Sema::OwningStmtResult Sema::ActOnExprStmt(FullExprArg expr) {
-  Expr *E = expr->takeAs<Expr>();
+StmtResult Sema::ActOnExprStmt(FullExprArg expr) {
+  Expr *E = expr.get();
   assert(E && "ActOnExprStmt(): missing expression");
-  if (E->getType()->isObjCObjectType()) {
-    if (LangOpts.ObjCNonFragileABI)
-      Diag(E->getLocEnd(), diag::err_indirection_requires_nonfragile_object)
-             << E->getType();
-    else
-      Diag(E->getLocEnd(), diag::err_direct_interface_unsupported)
-             << E->getType();
-    return StmtError();
-  }
   // C99 6.8.3p2: The expression in an expression statement is evaluated as a
   // void expression for its side effects.  Conversion to void allows any
   // operand, even incomplete types.
@@ -48,11 +42,11 @@ Sema::OwningStmtResult Sema::ActOnExprStmt(FullExprArg expr) {
 }
 
 
-Sema::OwningStmtResult Sema::ActOnNullStmt(SourceLocation SemiLoc) {
+StmtResult Sema::ActOnNullStmt(SourceLocation SemiLoc) {
   return Owned(new (Context) NullStmt(SemiLoc));
 }
 
-Sema::OwningStmtResult Sema::ActOnDeclStmt(DeclGroupPtrTy dg,
+StmtResult Sema::ActOnDeclStmt(DeclGroupPtrTy dg,
                                            SourceLocation StartLoc,
                                            SourceLocation EndLoc) {
   DeclGroupRef DG = dg.getAsVal<DeclGroupRef>();
@@ -141,10 +135,10 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
     }
   }
 
-  Diag(Loc, DiagID) << R1 << R2;
+  DiagRuntimeBehavior(Loc, PDiag(DiagID) << R1 << R2);
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnCompoundStmt(SourceLocation L, SourceLocation R,
                         MultiStmtArg elts, bool isStmtExpr) {
   unsigned NumElts = elts.size();
@@ -179,70 +173,60 @@ Sema::ActOnCompoundStmt(SourceLocation L, SourceLocation R,
   return Owned(new (Context) CompoundStmt(Context, Elts, NumElts, L, R));
 }
 
-Action::OwningStmtResult
-Sema::ActOnCaseStmt(SourceLocation CaseLoc, ExprArg lhsval,
-                    SourceLocation DotDotDotLoc, ExprArg rhsval,
+StmtResult
+Sema::ActOnCaseStmt(SourceLocation CaseLoc, Expr *LHSVal,
+                    SourceLocation DotDotDotLoc, Expr *RHSVal,
                     SourceLocation ColonLoc) {
-  assert((lhsval.get() != 0) && "missing expression in case statement");
+  assert((LHSVal != 0) && "missing expression in case statement");
 
   // C99 6.8.4.2p3: The expression shall be an integer constant.
   // However, GCC allows any evaluatable integer expression.
-  Expr *LHSVal = static_cast<Expr*>(lhsval.get());
   if (!LHSVal->isTypeDependent() && !LHSVal->isValueDependent() &&
       VerifyIntegerConstantExpression(LHSVal))
     return StmtError();
 
   // GCC extension: The expression shall be an integer constant.
 
-  Expr *RHSVal = static_cast<Expr*>(rhsval.get());
   if (RHSVal && !RHSVal->isTypeDependent() && !RHSVal->isValueDependent() &&
       VerifyIntegerConstantExpression(RHSVal)) {
     RHSVal = 0;  // Recover by just forgetting about it.
-    rhsval = 0;
   }
 
-  if (getSwitchStack().empty()) {
+  if (getCurFunction()->SwitchStack.empty()) {
     Diag(CaseLoc, diag::err_case_not_in_switch);
     return StmtError();
   }
 
-  // Only now release the smart pointers.
-  lhsval.release();
-  rhsval.release();
   CaseStmt *CS = new (Context) CaseStmt(LHSVal, RHSVal, CaseLoc, DotDotDotLoc,
                                         ColonLoc);
-  getSwitchStack().back()->addSwitchCase(CS);
+  getCurFunction()->SwitchStack.back()->addSwitchCase(CS);
   return Owned(CS);
 }
 
 /// ActOnCaseStmtBody - This installs a statement as the body of a case.
-void Sema::ActOnCaseStmtBody(StmtTy *caseStmt, StmtArg subStmt) {
+void Sema::ActOnCaseStmtBody(Stmt *caseStmt, Stmt *SubStmt) {
   CaseStmt *CS = static_cast<CaseStmt*>(caseStmt);
-  Stmt *SubStmt = subStmt.takeAs<Stmt>();
   CS->setSubStmt(SubStmt);
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnDefaultStmt(SourceLocation DefaultLoc, SourceLocation ColonLoc,
-                       StmtArg subStmt, Scope *CurScope) {
-  Stmt *SubStmt = subStmt.takeAs<Stmt>();
-
-  if (getSwitchStack().empty()) {
+                       Stmt *SubStmt, Scope *CurScope) {
+  if (getCurFunction()->SwitchStack.empty()) {
     Diag(DefaultLoc, diag::err_default_not_in_switch);
     return Owned(SubStmt);
   }
 
   DefaultStmt *DS = new (Context) DefaultStmt(DefaultLoc, ColonLoc, SubStmt);
-  getSwitchStack().back()->addSwitchCase(DS);
+  getCurFunction()->SwitchStack.back()->addSwitchCase(DS);
   return Owned(DS);
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnLabelStmt(SourceLocation IdentLoc, IdentifierInfo *II,
-                     SourceLocation ColonLoc, StmtArg subStmt) {
-  Stmt *SubStmt = subStmt.takeAs<Stmt>();
+                     SourceLocation ColonLoc, Stmt *SubStmt) {
   // Look up the record for this label identifier.
-  LabelStmt *&LabelDecl = getLabelMap()[II];
+  LabelStmt *&LabelDecl = getCurFunction()->LabelMap[II];
 
   // If not forward referenced or defined already, just create a new LabelStmt.
   if (LabelDecl == 0)
@@ -265,15 +249,15 @@ Sema::ActOnLabelStmt(SourceLocation IdentLoc, IdentifierInfo *II,
   return Owned(LabelDecl);
 }
 
-Action::OwningStmtResult
-Sema::ActOnIfStmt(SourceLocation IfLoc, FullExprArg CondVal, DeclPtrTy CondVar,
-                  StmtArg ThenVal, SourceLocation ElseLoc,
-                  StmtArg ElseVal) {
-  OwningExprResult CondResult(CondVal.release());
+StmtResult
+Sema::ActOnIfStmt(SourceLocation IfLoc, FullExprArg CondVal, Decl *CondVar,
+                  Stmt *thenStmt, SourceLocation ElseLoc,
+                  Stmt *elseStmt) {
+  ExprResult CondResult(CondVal.release());
 
   VarDecl *ConditionVar = 0;
-  if (CondVar.get()) {
-    ConditionVar = CondVar.getAs<VarDecl>();
+  if (CondVar) {
+    ConditionVar = cast<VarDecl>(CondVar);
     CondResult = CheckConditionVariable(ConditionVar, IfLoc, true);
     if (CondResult.isInvalid())
       return StmtError();
@@ -282,22 +266,19 @@ Sema::ActOnIfStmt(SourceLocation IfLoc, FullExprArg CondVal, DeclPtrTy CondVar,
   if (!ConditionExpr)
     return StmtError();
   
-  Stmt *thenStmt = ThenVal.takeAs<Stmt>();
   DiagnoseUnusedExprResult(thenStmt);
 
   // Warn if the if block has a null body without an else value.
   // this helps prevent bugs due to typos, such as
   // if (condition);
   //   do_stuff();
-  if (!ElseVal.get()) {
+  if (!elseStmt) {
     if (NullStmt* stmt = dyn_cast<NullStmt>(thenStmt))
       Diag(stmt->getSemiLoc(), diag::warn_empty_if_body);
   }
 
-  Stmt *elseStmt = ElseVal.takeAs<Stmt>();
   DiagnoseUnusedExprResult(elseStmt);
 
-  CondResult.release();
   return Owned(new (Context) IfStmt(Context, IfLoc, ConditionVar, ConditionExpr, 
                                     thenStmt, ElseLoc, elseStmt));
 }
@@ -404,64 +385,63 @@ static QualType GetTypeBeforeIntegralPromotion(const Expr* expr) {
   return expr->getType();
 }
 
-Action::OwningStmtResult
-Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, ExprArg Cond, 
-                             DeclPtrTy CondVar) {
+StmtResult
+Sema::ActOnStartOfSwitchStmt(SourceLocation SwitchLoc, Expr *Cond, 
+                             Decl *CondVar) {
+  ExprResult CondResult;
+
   VarDecl *ConditionVar = 0;
-  if (CondVar.get()) {
-    ConditionVar = CondVar.getAs<VarDecl>();
-    OwningExprResult CondE = CheckConditionVariable(ConditionVar, SourceLocation(), false);
-    if (CondE.isInvalid())
+  if (CondVar) {
+    ConditionVar = cast<VarDecl>(CondVar);
+    CondResult = CheckConditionVariable(ConditionVar, SourceLocation(), false);
+    if (CondResult.isInvalid())
       return StmtError();
     
-    Cond = move(CondE);
+    Cond = CondResult.release();
   }
   
-  if (!Cond.get())
+  if (!Cond)
     return StmtError();
   
-  Expr *CondExpr = static_cast<Expr *>(Cond.get());
-  OwningExprResult ConvertedCond 
-    = ConvertToIntegralOrEnumerationType(SwitchLoc, move(Cond), 
+  CondResult
+    = ConvertToIntegralOrEnumerationType(SwitchLoc, Cond, 
                           PDiag(diag::err_typecheck_statement_requires_integer),
                                    PDiag(diag::err_switch_incomplete_class_type)
-                                     << CondExpr->getSourceRange(),
+                                     << Cond->getSourceRange(),
                                    PDiag(diag::err_switch_explicit_conversion),
                                          PDiag(diag::note_switch_conversion),
                                    PDiag(diag::err_switch_multiple_conversions),
                                          PDiag(diag::note_switch_conversion),
                                          PDiag(0));
-  if (ConvertedCond.isInvalid())
-    return StmtError();
+  if (CondResult.isInvalid()) return StmtError();
+  Cond = CondResult.take();
   
-  CondExpr = ConvertedCond.takeAs<Expr>();
-  
-  if (!CondVar.get()) {
-    CondExpr = MaybeCreateCXXExprWithTemporaries(CondExpr);
-    if (!CondExpr)
+  if (!CondVar) {
+    CondResult = MaybeCreateCXXExprWithTemporaries(Cond);
+    if (CondResult.isInvalid())
       return StmtError();
+    Cond = CondResult.take();
   }
+
+  getCurFunction()->setHasBranchIntoScope();
     
-  SwitchStmt *SS = new (Context) SwitchStmt(Context, ConditionVar, CondExpr);
-  getSwitchStack().push_back(SS);
+  SwitchStmt *SS = new (Context) SwitchStmt(Context, ConditionVar, Cond);
+  getCurFunction()->SwitchStack.push_back(SS);
   return Owned(SS);
 }
 
-Action::OwningStmtResult
-Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, StmtArg Switch,
-                            StmtArg Body) {
-  Stmt *BodyStmt = Body.takeAs<Stmt>();
-
-  SwitchStmt *SS = getSwitchStack().back();
-  assert(SS == (SwitchStmt*)Switch.get() && "switch stack missing push/pop!");
+StmtResult
+Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
+                            Stmt *BodyStmt) {
+  SwitchStmt *SS = cast<SwitchStmt>(Switch);
+  assert(SS == getCurFunction()->SwitchStack.back() &&
+         "switch stack missing push/pop!");
 
   SS->setBody(BodyStmt, SwitchLoc);
-  getSwitchStack().pop_back();
+  getCurFunction()->SwitchStack.pop_back();
 
-  if (SS->getCond() == 0) {
-    SS->Destroy(Context);
+  if (SS->getCond() == 0)
     return StmtError();
-  }
     
   Expr *CondExpr = SS->getCond();
   Expr *CondExprBeforePromotion = CondExpr;
@@ -556,7 +536,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, StmtArg Switch,
 
       // If the LHS is not the same type as the condition, insert an implicit
       // cast.
-      ImpCastExprToType(Lo, CondType, CastExpr::CK_IntegralCast);
+      ImpCastExprToType(Lo, CondType, CK_IntegralCast);
       CS->setLHS(Lo);
 
       // If this is a case range, remember it in CaseRanges, otherwise CaseVals.
@@ -635,7 +615,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, StmtArg Switch,
 
         // If the LHS is not the same type as the condition, insert an implicit
         // cast.
-        ImpCastExprToType(Hi, CondType, CastExpr::CK_IntegralCast);
+        ImpCastExprToType(Hi, CondType, CK_IntegralCast);
         CR->setRHS(Hi);
 
         // If the low value is bigger than the high value, the case is empty.
@@ -804,65 +784,55 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, StmtArg Switch,
   if (CaseListIsErroneous)
     return StmtError();
 
-  Switch.release();
   return Owned(SS);
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnWhileStmt(SourceLocation WhileLoc, FullExprArg Cond, 
-                     DeclPtrTy CondVar, StmtArg Body) {
-  OwningExprResult CondResult(Cond.release());
+                     Decl *CondVar, Stmt *Body) {
+  ExprResult CondResult(Cond.release());
   
   VarDecl *ConditionVar = 0;
-  if (CondVar.get()) {
-    ConditionVar = CondVar.getAs<VarDecl>();
+  if (CondVar) {
+    ConditionVar = cast<VarDecl>(CondVar);
     CondResult = CheckConditionVariable(ConditionVar, WhileLoc, true);
     if (CondResult.isInvalid())
       return StmtError();
   }
-  Expr *ConditionExpr = CondResult.takeAs<Expr>();
+  Expr *ConditionExpr = CondResult.take();
   if (!ConditionExpr)
     return StmtError();
   
-  Stmt *bodyStmt = Body.takeAs<Stmt>();
-  DiagnoseUnusedExprResult(bodyStmt);
+  DiagnoseUnusedExprResult(Body);
 
-  CondResult.release();
   return Owned(new (Context) WhileStmt(Context, ConditionVar, ConditionExpr,
-                                       bodyStmt, WhileLoc));
+                                       Body, WhileLoc));
 }
 
-Action::OwningStmtResult
-Sema::ActOnDoStmt(SourceLocation DoLoc, StmtArg Body,
+StmtResult
+Sema::ActOnDoStmt(SourceLocation DoLoc, Stmt *Body,
                   SourceLocation WhileLoc, SourceLocation CondLParen,
-                  ExprArg Cond, SourceLocation CondRParen) {
-  Expr *condExpr = Cond.takeAs<Expr>();
-  assert(condExpr && "ActOnDoStmt(): missing expression");
+                  Expr *Cond, SourceLocation CondRParen) {
+  assert(Cond && "ActOnDoStmt(): missing expression");
 
-  if (CheckBooleanCondition(condExpr, DoLoc)) {
-    Cond = condExpr;
+  if (CheckBooleanCondition(Cond, DoLoc))
     return StmtError();
-  }
 
-  condExpr = MaybeCreateCXXExprWithTemporaries(condExpr);
-  if (!condExpr)
+  ExprResult CondResult = MaybeCreateCXXExprWithTemporaries(Cond);
+  if (CondResult.isInvalid())
     return StmtError();
+  Cond = CondResult.take();
   
-  Stmt *bodyStmt = Body.takeAs<Stmt>();
-  DiagnoseUnusedExprResult(bodyStmt);
+  DiagnoseUnusedExprResult(Body);
 
-  Cond.release();
-  return Owned(new (Context) DoStmt(bodyStmt, condExpr, DoLoc,
-                                    WhileLoc, CondRParen));
+  return Owned(new (Context) DoStmt(Body, Cond, DoLoc, WhileLoc, CondRParen));
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnForStmt(SourceLocation ForLoc, SourceLocation LParenLoc,
-                   StmtArg first, FullExprArg second, DeclPtrTy secondVar,
+                   Stmt *First, FullExprArg second, Decl *secondVar,
                    FullExprArg third,
-                   SourceLocation RParenLoc, StmtArg body) {
-  Stmt *First  = static_cast<Stmt*>(first.get());
-
+                   SourceLocation RParenLoc, Stmt *Body) {
   if (!getLangOptions().CPlusPlus) {
     if (DeclStmt *DS = dyn_cast_or_null<DeclStmt>(First)) {
       // C99 6.8.5p3: The declaration part of a 'for' statement shall only
@@ -880,38 +850,32 @@ Sema::ActOnForStmt(SourceLocation ForLoc, SourceLocation LParenLoc,
     }
   }
 
-  OwningExprResult SecondResult(second.release());
+  ExprResult SecondResult(second.release());
   VarDecl *ConditionVar = 0;
-  if (secondVar.get()) {
-    ConditionVar = secondVar.getAs<VarDecl>();
+  if (secondVar) {
+    ConditionVar = cast<VarDecl>(secondVar);
     SecondResult = CheckConditionVariable(ConditionVar, ForLoc, true);
     if (SecondResult.isInvalid())
       return StmtError();
   }
   
   Expr *Third  = third.release().takeAs<Expr>();
-  Stmt *Body  = static_cast<Stmt*>(body.get());
   
   DiagnoseUnusedExprResult(First);
   DiagnoseUnusedExprResult(Third);
   DiagnoseUnusedExprResult(Body);
 
-  first.release();
-  body.release();
   return Owned(new (Context) ForStmt(Context, First, 
-                                     SecondResult.takeAs<Expr>(), ConditionVar, 
+                                     SecondResult.take(), ConditionVar, 
                                      Third, Body, ForLoc, LParenLoc, 
                                      RParenLoc));
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
                                  SourceLocation LParenLoc,
-                                 StmtArg first, ExprArg second,
-                                 SourceLocation RParenLoc, StmtArg body) {
-  Stmt *First  = static_cast<Stmt*>(first.get());
-  Expr *Second = static_cast<Expr*>(second.get());
-  Stmt *Body  = static_cast<Stmt*>(body.get());
+                                 Stmt *First, Expr *Second,
+                                 SourceLocation RParenLoc, Stmt *Body) {
   if (First) {
     QualType FirstType;
     if (DeclStmt *DS = dyn_cast<DeclStmt>(First)) {
@@ -950,19 +914,39 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
     if (!SecondType->isObjCObjectPointerType())
       Diag(ForLoc, diag::err_collection_expr_type)
         << SecondType << Second->getSourceRange();
+    else if (const ObjCObjectPointerType *OPT =
+             SecondType->getAsObjCInterfacePointerType()) {
+      llvm::SmallVector<IdentifierInfo *, 4> KeyIdents;
+      IdentifierInfo* selIdent = 
+        &Context.Idents.get("countByEnumeratingWithState");
+      KeyIdents.push_back(selIdent);
+      selIdent = &Context.Idents.get("objects");
+      KeyIdents.push_back(selIdent);
+      selIdent = &Context.Idents.get("count");
+      KeyIdents.push_back(selIdent);
+      Selector CSelector = Context.Selectors.getSelector(3, &KeyIdents[0]);
+      if (ObjCInterfaceDecl *IDecl = OPT->getInterfaceDecl()) {
+        if (!IDecl->isForwardDecl() && 
+            !IDecl->lookupInstanceMethod(CSelector)) {
+          // Must further look into private implementation methods.
+          if (!LookupPrivateInstanceMethod(CSelector, IDecl))
+            Diag(ForLoc, diag::warn_collection_expr_type)
+              << SecondType << CSelector << Second->getSourceRange();
+        }
+      }
+    }
   }
-  first.release();
-  second.release();
-  body.release();
   return Owned(new (Context) ObjCForCollectionStmt(First, Second, Body,
                                                    ForLoc, RParenLoc));
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnGotoStmt(SourceLocation GotoLoc, SourceLocation LabelLoc,
                     IdentifierInfo *LabelII) {
   // Look up the record for this label identifier.
-  LabelStmt *&LabelDecl = getLabelMap()[LabelII];
+  LabelStmt *&LabelDecl = getCurFunction()->LabelMap[LabelII];
+
+  getCurFunction()->setHasBranchIntoScope();
 
   // If we haven't seen this label yet, create a forward reference.
   if (LabelDecl == 0)
@@ -971,11 +955,10 @@ Sema::ActOnGotoStmt(SourceLocation GotoLoc, SourceLocation LabelLoc,
   return Owned(new (Context) GotoStmt(LabelDecl, GotoLoc, LabelLoc));
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnIndirectGotoStmt(SourceLocation GotoLoc, SourceLocation StarLoc,
-                            ExprArg DestExp) {
+                            Expr *E) {
   // Convert operand to void*
-  Expr* E = DestExp.takeAs<Expr>();
   if (!E->isTypeDependent()) {
     QualType ETy = E->getType();
     QualType DestTy = Context.getPointerType(Context.VoidTy.withConst());
@@ -984,10 +967,13 @@ Sema::ActOnIndirectGotoStmt(SourceLocation GotoLoc, SourceLocation StarLoc,
     if (DiagnoseAssignmentResult(ConvTy, StarLoc, DestTy, ETy, E, AA_Passing))
       return StmtError();
   }
+
+  getCurFunction()->setHasIndirectGoto();
+
   return Owned(new (Context) IndirectGotoStmt(GotoLoc, StarLoc, E));
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnContinueStmt(SourceLocation ContinueLoc, Scope *CurScope) {
   Scope *S = CurScope->getContinueParent();
   if (!S) {
@@ -998,7 +984,7 @@ Sema::ActOnContinueStmt(SourceLocation ContinueLoc, Scope *CurScope) {
   return Owned(new (Context) ContinueStmt(ContinueLoc));
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnBreakStmt(SourceLocation BreakLoc, Scope *CurScope) {
   Scope *S = CurScope->getBreakParent();
   if (!S) {
@@ -1050,7 +1036,7 @@ static const VarDecl *getNRVOCandidate(ASTContext &Ctx, QualType RetType,
 
 /// ActOnBlockReturnStmt - Utility routine to figure out block's return type.
 ///
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   // If this is the first return we've seen in the block, infer the type of
   // the block from it.
@@ -1086,7 +1072,6 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   if (CurBlock->ReturnType->isVoidType()) {
     if (RetValExp) {
       Diag(ReturnLoc, diag::err_return_block_has_expr);
-      RetValExp->Destroy(Context);
       RetValExp = 0;
     }
     Result = new (Context) ReturnStmt(ReturnLoc, RetValExp, 0);
@@ -1105,7 +1090,7 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
       // In C++ the return statement is handled via a copy initialization.
       // the C version of which boils down to CheckSingleAssignmentConstraints.
       NRVOCandidate = getNRVOCandidate(Context, FnRetType, RetValExp);
-      OwningExprResult Res = PerformCopyInitialization(
+      ExprResult Res = PerformCopyInitialization(
                                InitializedEntity::InitializeResult(ReturnLoc, 
                                                                    FnRetType,
                                                             NRVOCandidate != 0),
@@ -1136,9 +1121,8 @@ Sema::ActOnBlockReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   return Owned(Result);
 }
 
-Action::OwningStmtResult
-Sema::ActOnReturnStmt(SourceLocation ReturnLoc, ExprArg rex) {
-  Expr *RetValExp = rex.takeAs<Expr>();
+StmtResult
+Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   if (getCurBlock())
     return ActOnBlockReturnStmt(ReturnLoc, RetValExp);
 
@@ -1197,7 +1181,7 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, ExprArg rex) {
       // In C++ the return statement is handled via a copy initialization.
       // the C version of which boils down to CheckSingleAssignmentConstraints.
       NRVOCandidate = getNRVOCandidate(Context, FnRetType, RetValExp);
-      OwningExprResult Res = PerformCopyInitialization(
+      ExprResult Res = PerformCopyInitialization(
                                InitializedEntity::InitializeResult(ReturnLoc, 
                                                                    FnRetType,
                                                             NRVOCandidate != 0),
@@ -1261,7 +1245,7 @@ static bool CheckAsmLValue(const Expr *E, Sema &S) {
 }
 
 
-Sema::OwningStmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
+StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
                                           bool IsSimple,
                                           bool IsVolatile,
                                           unsigned NumOutputs,
@@ -1269,15 +1253,15 @@ Sema::OwningStmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
                                           IdentifierInfo **Names,
                                           MultiExprArg constraints,
                                           MultiExprArg exprs,
-                                          ExprArg asmString,
+                                          Expr *asmString,
                                           MultiExprArg clobbers,
                                           SourceLocation RParenLoc,
                                           bool MSAsm) {
   unsigned NumClobbers = clobbers.size();
   StringLiteral **Constraints =
     reinterpret_cast<StringLiteral**>(constraints.get());
-  Expr **Exprs = reinterpret_cast<Expr **>(exprs.get());
-  StringLiteral *AsmString = cast<StringLiteral>((Expr *)asmString.get());
+  Expr **Exprs = exprs.get();
+  StringLiteral *AsmString = cast<StringLiteral>(asmString);
   StringLiteral **Clobbers = reinterpret_cast<StringLiteral**>(clobbers.get());
 
   llvm::SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
@@ -1373,10 +1357,6 @@ Sema::OwningStmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
                   diag::err_asm_unknown_register_name) << Clobber);
   }
 
-  constraints.release();
-  exprs.release();
-  asmString.release();
-  clobbers.release();
   AsmStmt *NS =
     new (Context) AsmStmt(Context, AsmLoc, IsSimple, IsVolatile, MSAsm, 
                           NumOutputs, NumInputs, Names, Constraints, Exprs, 
@@ -1388,7 +1368,6 @@ Sema::OwningStmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
   if (unsigned DiagID = NS->AnalyzeAsmString(Pieces, Context, DiagOffs)) {
     Diag(getLocationOfStringLiteralByte(AsmString, DiagOffs), DiagID)
            << AsmString->getSourceRange();
-    DeleteStmt(NS);
     return StmtError();
   }
 
@@ -1479,45 +1458,41 @@ Sema::OwningStmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc,
          diag::err_asm_tying_incompatible_types)
       << InTy << OutTy << OutputExpr->getSourceRange()
       << InputExpr->getSourceRange();
-    DeleteStmt(NS);
     return StmtError();
   }
 
   return Owned(NS);
 }
 
-Action::OwningStmtResult
+StmtResult
 Sema::ActOnObjCAtCatchStmt(SourceLocation AtLoc,
-                           SourceLocation RParen, DeclPtrTy Parm,
-                           StmtArg Body) {
-  VarDecl *Var = cast_or_null<VarDecl>(Parm.getAs<Decl>());
+                           SourceLocation RParen, Decl *Parm,
+                           Stmt *Body) {
+  VarDecl *Var = cast_or_null<VarDecl>(Parm);
   if (Var && Var->isInvalidDecl())
     return StmtError();
   
-  return Owned(new (Context) ObjCAtCatchStmt(AtLoc, RParen, Var, 
-                                             Body.takeAs<Stmt>()));
+  return Owned(new (Context) ObjCAtCatchStmt(AtLoc, RParen, Var, Body));
 }
 
-Action::OwningStmtResult
-Sema::ActOnObjCAtFinallyStmt(SourceLocation AtLoc, StmtArg Body) {
-  return Owned(new (Context) ObjCAtFinallyStmt(AtLoc,
-                                           static_cast<Stmt*>(Body.release())));
+StmtResult
+Sema::ActOnObjCAtFinallyStmt(SourceLocation AtLoc, Stmt *Body) {
+  return Owned(new (Context) ObjCAtFinallyStmt(AtLoc, Body));
 }
 
-Action::OwningStmtResult
-Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, StmtArg Try, 
-                         MultiStmtArg CatchStmts, StmtArg Finally) {
-  FunctionNeedsScopeChecking() = true;
+StmtResult
+Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try, 
+                         MultiStmtArg CatchStmts, Stmt *Finally) {
+  getCurFunction()->setHasBranchProtectedScope();
   unsigned NumCatchStmts = CatchStmts.size();
-  return Owned(ObjCAtTryStmt::Create(Context, AtLoc, Try.takeAs<Stmt>(),
-                                     (Stmt **)CatchStmts.release(),
+  return Owned(ObjCAtTryStmt::Create(Context, AtLoc, Try,
+                                     CatchStmts.release(),
                                      NumCatchStmts,
-                                     Finally.takeAs<Stmt>()));
+                                     Finally));
 }
 
-Sema::OwningStmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc,
-                                                  ExprArg ThrowE) {
-  Expr *Throw = static_cast<Expr *>(ThrowE.get());
+StmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc,
+                                                  Expr *Throw) {
   if (Throw) {
     QualType ThrowType = Throw->getType();
     // Make sure the expression type is an ObjC pointer or "void *".
@@ -1530,13 +1505,13 @@ Sema::OwningStmtResult Sema::BuildObjCAtThrowStmt(SourceLocation AtLoc,
     }
   }
   
-  return Owned(new (Context) ObjCAtThrowStmt(AtLoc, ThrowE.takeAs<Expr>()));
+  return Owned(new (Context) ObjCAtThrowStmt(AtLoc, Throw));
 }
 
-Action::OwningStmtResult
-Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, ExprArg Throw, 
+StmtResult
+Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, Expr *Throw, 
                            Scope *CurScope) {
-  if (!Throw.get()) {
+  if (!Throw) {
     // @throw without an expression designates a rethrow (which much occur
     // in the context of an @catch clause).
     Scope *AtCatchParent = CurScope;
@@ -1546,16 +1521,15 @@ Sema::ActOnObjCAtThrowStmt(SourceLocation AtLoc, ExprArg Throw,
       return StmtError(Diag(AtLoc, diag::error_rethrow_used_outside_catch));
   } 
   
-  return BuildObjCAtThrowStmt(AtLoc, move(Throw));
+  return BuildObjCAtThrowStmt(AtLoc, Throw);
 }
 
-Action::OwningStmtResult
-Sema::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, ExprArg SynchExpr,
-                                  StmtArg SynchBody) {
-  FunctionNeedsScopeChecking() = true;
+StmtResult
+Sema::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, Expr *SyncExpr,
+                                  Stmt *SyncBody) {
+  getCurFunction()->setHasBranchProtectedScope();
 
   // Make sure the expression type is an ObjC pointer or "void *".
-  Expr *SyncExpr = static_cast<Expr*>(SynchExpr.get());
   if (!SyncExpr->getType()->isDependentType() &&
       !SyncExpr->getType()->isObjCObjectPointerType()) {
     const PointerType *PT = SyncExpr->getType()->getAs<PointerType>();
@@ -1564,22 +1538,22 @@ Sema::ActOnObjCAtSynchronizedStmt(SourceLocation AtLoc, ExprArg SynchExpr,
                        << SyncExpr->getType() << SyncExpr->getSourceRange());
   }
 
-  return Owned(new (Context) ObjCAtSynchronizedStmt(AtLoc,
-                                                    SynchExpr.takeAs<Stmt>(),
-                                                    SynchBody.takeAs<Stmt>()));
+  return Owned(new (Context) ObjCAtSynchronizedStmt(AtLoc, SyncExpr, SyncBody));
 }
 
 /// ActOnCXXCatchBlock - Takes an exception declaration and a handler block
 /// and creates a proper catch handler from them.
-Action::OwningStmtResult
-Sema::ActOnCXXCatchBlock(SourceLocation CatchLoc, DeclPtrTy ExDecl,
-                         StmtArg HandlerBlock) {
+StmtResult
+Sema::ActOnCXXCatchBlock(SourceLocation CatchLoc, Decl *ExDecl,
+                         Stmt *HandlerBlock) {
   // There's nothing to test that ActOnExceptionDecl didn't already test.
   return Owned(new (Context) CXXCatchStmt(CatchLoc,
-                                  cast_or_null<VarDecl>(ExDecl.getAs<Decl>()),
-                                          HandlerBlock.takeAs<Stmt>()));
+                                          cast_or_null<VarDecl>(ExDecl),
+                                          HandlerBlock));
 }
 
+namespace {
+
 class TypeWithHandler {
   QualType t;
   CXXCatchStmt *stmt;
@@ -1602,22 +1576,23 @@ public:
     return t == other.t;
   }
 
-  QualType getQualType() const { return t; }
   CXXCatchStmt *getCatchStmt() const { return stmt; }
   SourceLocation getTypeSpecStartLoc() const {
     return stmt->getExceptionDecl()->getTypeSpecStartLoc();
   }
 };
 
+}
+
 /// ActOnCXXTryBlock - Takes a try compound-statement and a number of
 /// handlers and creates a try statement from them.
-Action::OwningStmtResult
-Sema::ActOnCXXTryBlock(SourceLocation TryLoc, StmtArg TryBlock,
+StmtResult
+Sema::ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
                        MultiStmtArg RawHandlers) {
   unsigned NumHandlers = RawHandlers.size();
   assert(NumHandlers > 0 &&
          "The parser shouldn't call this if there are no handlers.");
-  Stmt **Handlers = reinterpret_cast<Stmt**>(RawHandlers.get());
+  Stmt **Handlers = RawHandlers.get();
 
   llvm::SmallVector<TypeWithHandler, 8> TypesWithHandlers;
 
@@ -1657,15 +1632,14 @@ Sema::ActOnCXXTryBlock(SourceLocation TryLoc, StmtArg TryBlock,
     }
   }
 
+  getCurFunction()->setHasBranchProtectedScope();
+
   // FIXME: We should detect handlers that cannot catch anything because an
   // earlier handler catches a superclass. Need to find a method that is not
   // quadratic for this.
   // Neither of these are explicitly forbidden, but every compiler detects them
   // and warns.
 
-  FunctionNeedsScopeChecking() = true;
-  RawHandlers.release();
-  return Owned(CXXTryStmt::Create(Context, TryLoc,
-                                  static_cast<Stmt*>(TryBlock.release()),
+  return Owned(CXXTryStmt::Create(Context, TryLoc, TryBlock,
                                   Handlers, NumHandlers));
 }
diff --git a/lib/Sema/SemaTemplate.cpp b/lib/Sema/SemaTemplate.cpp
index f121954eed6e..0fc83927b79c 100644
--- a/lib/Sema/SemaTemplate.cpp
+++ b/lib/Sema/SemaTemplate.cpp
@@ -9,20 +9,24 @@
 //  This file implements semantic analysis for C++ templates.
 //===----------------------------------------------------------------------===/
 
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/Template.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "TreeTransform.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclTemplate.h"
-#include "clang/Parse/DeclSpec.h"
-#include "clang/Parse/Template.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ParsedTemplate.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "llvm/ADT/StringExtras.h"
 using namespace clang;
+using namespace sema;
 
 /// \brief Determine whether the declaration found is acceptable as the name
 /// of a template and, if so, return that template declaration. Otherwise,
@@ -88,8 +92,13 @@ static void FilterAcceptableTemplateNames(ASTContext &C, LookupResult &R) {
           filter.erase();
           continue;
         }
-          
-      filter.replace(Repl);
+
+      // FIXME: we promote access to public here as a workaround to
+      // the fact that LookupResult doesn't let us remember that we
+      // found this template through a particular injected class name,
+      // which means we end up doing nasty things to the invariants.
+      // Pretending that access is public is *much* safer.
+      filter.replace(Repl, AS_public);
     }
   }
   filter.done();
@@ -97,8 +106,9 @@ static void FilterAcceptableTemplateNames(ASTContext &C, LookupResult &R) {
 
 TemplateNameKind Sema::isTemplateName(Scope *S,
                                       CXXScopeSpec &SS,
+                                      bool hasTemplateKeyword,
                                       UnqualifiedId &Name,
-                                      TypeTy *ObjectTypePtr,
+                                      ParsedType ObjectTypePtr,
                                       bool EnteringContext,
                                       TemplateTy &TemplateResult,
                                       bool &MemberOfUnknownSpecialization) {
@@ -125,15 +135,21 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
     return TNK_Non_template;
   }
 
-  QualType ObjectType = QualType::getFromOpaquePtr(ObjectTypePtr);
+  QualType ObjectType = ObjectTypePtr.get();
 
   LookupResult R(*this, TName, Name.getSourceRange().getBegin(), 
                  LookupOrdinaryName);
-  R.suppressDiagnostics();
   LookupTemplateName(R, S, SS, ObjectType, EnteringContext,
                      MemberOfUnknownSpecialization);
-  if (R.empty() || R.isAmbiguous())
+  if (R.empty()) return TNK_Non_template;
+  if (R.isAmbiguous()) {
+    // Suppress diagnostics;  we'll redo this lookup later.
+    R.suppressDiagnostics();
+
+    // FIXME: we might have ambiguous templates, in which case we
+    // should at least parse them properly!
     return TNK_Non_template;
+  }
 
   TemplateName Template;
   TemplateNameKind TemplateKind;
@@ -144,20 +160,27 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
     // template name in other ways.
     Template = Context.getOverloadedTemplateName(R.begin(), R.end());
     TemplateKind = TNK_Function_template;
+
+    // We'll do this lookup again later.
+    R.suppressDiagnostics();
   } else {
     TemplateDecl *TD = cast<TemplateDecl>((*R.begin())->getUnderlyingDecl());
 
     if (SS.isSet() && !SS.isInvalid()) {
       NestedNameSpecifier *Qualifier
         = static_cast<NestedNameSpecifier *>(SS.getScopeRep());
-      Template = Context.getQualifiedTemplateName(Qualifier, false, TD);
+      Template = Context.getQualifiedTemplateName(Qualifier,
+                                                  hasTemplateKeyword, TD);
     } else {
       Template = TemplateName(TD);
     }
 
-    if (isa<FunctionTemplateDecl>(TD))
+    if (isa<FunctionTemplateDecl>(TD)) {
       TemplateKind = TNK_Function_template;
-    else {
+
+      // We'll do this lookup again later.
+      R.suppressDiagnostics();
+    } else {
       assert(isa<ClassTemplateDecl>(TD) || isa<TemplateTemplateParmDecl>(TD));
       TemplateKind = TNK_Type_template;
     }
@@ -238,13 +261,10 @@ void Sema::LookupTemplateName(LookupResult &Found,
       //   expression. If the identifier is not found, it is then looked up in
       //   the context of the entire postfix-expression and shall name a class
       //   or function template.
-      //
-      // FIXME: When we're instantiating a template, do we actually have to
-      // look in the scope of the template? Seems fishy...
       if (S) LookupName(Found, S);
       ObjectTypeSearchedInScope = true;
     }
-  } else if (isDependent) {
+  } else if (isDependent && (!S || ObjectType.isNull())) {
     // We cannot look into a dependent object type or nested nme
     // specifier.
     MemberOfUnknownSpecialization = true;
@@ -282,8 +302,11 @@ void Sema::LookupTemplateName(LookupResult &Found,
   }
 
   FilterAcceptableTemplateNames(Context, Found);
-  if (Found.empty())
+  if (Found.empty()) {
+    if (isDependent)
+      MemberOfUnknownSpecialization = true;
     return;
+  }
 
   if (S && !ObjectType.isNull() && !ObjectTypeSearchedInScope) {
     // C++ [basic.lookup.classref]p1:
@@ -330,10 +353,9 @@ void Sema::LookupTemplateName(LookupResult &Found,
 /// ActOnDependentIdExpression - Handle a dependent id-expression that
 /// was just parsed.  This is only possible with an explicit scope
 /// specifier naming a dependent type.
-Sema::OwningExprResult
+ExprResult
 Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
-                                 DeclarationName Name,
-                                 SourceLocation NameLoc,
+                                 const DeclarationNameInfo &NameInfo,
                                  bool isAddressOfOperand,
                            const TemplateArgumentListInfo *TemplateArgs) {
   NestedNameSpecifier *Qualifier
@@ -356,22 +378,21 @@ Sema::ActOnDependentIdExpression(const CXXScopeSpec &SS,
                                                      /*Op*/ SourceLocation(),
                                                      Qualifier, SS.getRange(),
                                                      FirstQualifierInScope,
-                                                     Name, NameLoc,
+                                                     NameInfo,
                                                      TemplateArgs));
   }
 
-  return BuildDependentDeclRefExpr(SS, Name, NameLoc, TemplateArgs);
+  return BuildDependentDeclRefExpr(SS, NameInfo, TemplateArgs);
 }
 
-Sema::OwningExprResult
+ExprResult
 Sema::BuildDependentDeclRefExpr(const CXXScopeSpec &SS,
-                                DeclarationName Name,
-                                SourceLocation NameLoc,
+                                const DeclarationNameInfo &NameInfo,
                                 const TemplateArgumentListInfo *TemplateArgs) {
   return Owned(DependentScopeDeclRefExpr::Create(Context,
                static_cast<NestedNameSpecifier*>(SS.getScopeRep()),
                                                  SS.getRange(),
-                                                 Name, NameLoc,
+                                                 NameInfo,
                                                  TemplateArgs));
 }
 
@@ -398,9 +419,9 @@ bool Sema::DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl) {
 /// AdjustDeclIfTemplate - If the given decl happens to be a template, reset
 /// the parameter D to reference the templated declaration and return a pointer
 /// to the template declaration. Otherwise, do nothing to D and return null.
-TemplateDecl *Sema::AdjustDeclIfTemplate(DeclPtrTy &D) {
-  if (TemplateDecl *Temp = dyn_cast_or_null<TemplateDecl>(D.getAs<Decl>())) {
-    D = DeclPtrTy::make(Temp->getTemplatedDecl());
+TemplateDecl *Sema::AdjustDeclIfTemplate(Decl *&D) {
+  if (TemplateDecl *Temp = dyn_cast_or_null<TemplateDecl>(D)) {
+    D = Temp->getTemplatedDecl();
     return Temp;
   }
   return 0;
@@ -424,8 +445,7 @@ static TemplateArgumentLoc translateTemplateArgument(Sema &SemaRef,
   }
     
   case ParsedTemplateArgument::Template: {
-    TemplateName Template
-      = TemplateName::getFromVoidPointer(Arg.getAsTemplate().get());
+    TemplateName Template = Arg.getAsTemplate().get();
     return TemplateArgumentLoc(TemplateArgument(Template),
                                Arg.getScopeSpec().getRange(),
                                Arg.getLocation());
@@ -454,14 +474,14 @@ void Sema::translateTemplateArguments(const ASTTemplateArgsPtr &TemplateArgsIn,
 /// ParamName is the location of the parameter name (if any).
 /// If the type parameter has a default argument, it will be added
 /// later via ActOnTypeParameterDefault.
-Sema::DeclPtrTy Sema::ActOnTypeParameter(Scope *S, bool Typename, bool Ellipsis,
-                                         SourceLocation EllipsisLoc,
-                                         SourceLocation KeyLoc,
-                                         IdentifierInfo *ParamName,
-                                         SourceLocation ParamNameLoc,
-                                         unsigned Depth, unsigned Position,
-                                         SourceLocation EqualLoc,
-                                         TypeTy *DefaultArg) {
+Decl *Sema::ActOnTypeParameter(Scope *S, bool Typename, bool Ellipsis,
+                               SourceLocation EllipsisLoc,
+                               SourceLocation KeyLoc,
+                               IdentifierInfo *ParamName,
+                               SourceLocation ParamNameLoc,
+                               unsigned Depth, unsigned Position,
+                               SourceLocation EqualLoc,
+                               ParsedType DefaultArg) {
   assert(S->isTemplateParamScope() &&
          "Template type parameter not in template parameter scope!");
   bool Invalid = false;
@@ -488,7 +508,7 @@ Sema::DeclPtrTy Sema::ActOnTypeParameter(Scope *S, bool Typename, bool Ellipsis,
 
   if (ParamName) {
     // Add the template parameter into the current scope.
-    S->AddDecl(DeclPtrTy::make(Param));
+    S->AddDecl(Param);
     IdResolver.AddDecl(Param);
   }
 
@@ -504,19 +524,19 @@ Sema::DeclPtrTy Sema::ActOnTypeParameter(Scope *S, bool Typename, bool Ellipsis,
     // template-parameter that is not a template parameter pack.
     if (Ellipsis) {
       Diag(EqualLoc, diag::err_template_param_pack_default_arg);
-      return DeclPtrTy::make(Param);
+      return Param;
     }
     
     // Check the template argument itself.
     if (CheckTemplateArgument(Param, DefaultTInfo)) {
       Param->setInvalidDecl();
-      return DeclPtrTy::make(Param);;
+      return Param;
     }
     
     Param->setDefaultArgument(DefaultTInfo, false);
   }
   
-  return DeclPtrTy::make(Param);
+  return Param;
 }
 
 /// \brief Check that the type of a non-type template parameter is
@@ -542,9 +562,7 @@ Sema::CheckNonTypeTemplateParameterType(QualType T, SourceLocation Loc) {
   //       -- integral or enumeration type,
   if (T->isIntegralOrEnumerationType() ||
       //   -- pointer to object or pointer to function,
-      (T->isPointerType() &&
-       (T->getAs<PointerType>()->getPointeeType()->isObjectType() ||
-        T->getAs<PointerType>()->getPointeeType()->isFunctionType())) ||
+      T->isPointerType() ||
       //   -- reference to object or reference to function,
       T->isReferenceType() ||
       //   -- pointer to member.
@@ -571,11 +589,11 @@ Sema::CheckNonTypeTemplateParameterType(QualType T, SourceLocation Loc) {
   return QualType();
 }
 
-Sema::DeclPtrTy Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
-                                                    unsigned Depth,
-                                                    unsigned Position,
-                                                    SourceLocation EqualLoc,
-                                                    ExprArg DefaultArg) {
+Decl *Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
+                                          unsigned Depth,
+                                          unsigned Position,
+                                          SourceLocation EqualLoc,
+                                          Expr *Default) {
   TypeSourceInfo *TInfo = GetTypeForDeclarator(D, S);
   QualType T = TInfo->getType();
 
@@ -608,35 +626,35 @@ Sema::DeclPtrTy Sema::ActOnNonTypeTemplateParameter(Scope *S, Declarator &D,
 
   if (D.getIdentifier()) {
     // Add the template parameter into the current scope.
-    S->AddDecl(DeclPtrTy::make(Param));
+    S->AddDecl(Param);
     IdResolver.AddDecl(Param);
   }
   
   // Check the well-formedness of the default template argument, if provided.
-  if (Expr *Default = static_cast<Expr *>(DefaultArg.get())) {  
+  if (Default) {  
     TemplateArgument Converted;
     if (CheckTemplateArgument(Param, Param->getType(), Default, Converted)) {
       Param->setInvalidDecl();
-      return DeclPtrTy::make(Param);;
+      return Param;
     }
   
-    Param->setDefaultArgument(DefaultArg.takeAs<Expr>(), false);
+    Param->setDefaultArgument(Default, false);
   }
   
-  return DeclPtrTy::make(Param);
+  return Param;
 }
 
 /// ActOnTemplateTemplateParameter - Called when a C++ template template
 /// parameter (e.g. T in template <template <typename> class T> class array)
 /// has been parsed. S is the current scope.
-Sema::DeclPtrTy Sema::ActOnTemplateTemplateParameter(Scope* S,
-                                                     SourceLocation TmpLoc,
-                                                     TemplateParamsTy *Params,
-                                                     IdentifierInfo *Name,
-                                                     SourceLocation NameLoc,
-                                                     unsigned Depth,
-                                                     unsigned Position,
-                                                     SourceLocation EqualLoc,
+Decl *Sema::ActOnTemplateTemplateParameter(Scope* S,
+                                           SourceLocation TmpLoc,
+                                           TemplateParamsTy *Params,
+                                           IdentifierInfo *Name,
+                                           SourceLocation NameLoc,
+                                           unsigned Depth,
+                                           unsigned Position,
+                                           SourceLocation EqualLoc,
                                        const ParsedTemplateArgument &Default) {
   assert(S->isTemplateParamScope() &&
          "Template template parameter not in template parameter scope!");
@@ -644,13 +662,14 @@ Sema::DeclPtrTy Sema::ActOnTemplateTemplateParameter(Scope* S,
   // Construct the parameter object.
   TemplateTemplateParmDecl *Param =
     TemplateTemplateParmDecl::Create(Context, Context.getTranslationUnitDecl(),
-                                     TmpLoc, Depth, Position, Name,
+                                     NameLoc.isInvalid()? TmpLoc : NameLoc, 
+                                     Depth, Position, Name,
                                      (TemplateParameterList*)Params);
 
   // If the template template parameter has a name, then link the identifier 
   // into the scope and lookup mechanisms.
   if (Name) {
-    S->AddDecl(DeclPtrTy::make(Param));
+    S->AddDecl(Param);
     IdResolver.AddDecl(Param);
   }
 
@@ -667,13 +686,13 @@ Sema::DeclPtrTy Sema::ActOnTemplateTemplateParameter(Scope* S,
     if (DefaultArg.getArgument().getAsTemplate().isNull()) {
       Diag(DefaultArg.getLocation(), diag::err_template_arg_not_class_template)
         << DefaultArg.getSourceRange();
-      return DeclPtrTy::make(Param);
+      return Param;
     }
     
     Param->setDefaultArgument(DefaultArg, false);
   }
   
-  return DeclPtrTy::make(Param);
+  return Param;
 }
 
 /// ActOnTemplateParameterList - Builds a TemplateParameterList that
@@ -683,7 +702,7 @@ Sema::ActOnTemplateParameterList(unsigned Depth,
                                  SourceLocation ExportLoc,
                                  SourceLocation TemplateLoc,
                                  SourceLocation LAngleLoc,
-                                 DeclPtrTy *Params, unsigned NumParams,
+                                 Decl **Params, unsigned NumParams,
                                  SourceLocation RAngleLoc) {
   if (ExportLoc.isValid())
     Diag(ExportLoc, diag::warn_template_export_unsupported);
@@ -699,7 +718,7 @@ static void SetNestedNameSpecifier(TagDecl *T, const CXXScopeSpec &SS) {
                         SS.getRange());
 }
 
-Sema::DeclResult
+DeclResult
 Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
                          SourceLocation KWLoc, CXXScopeSpec &SS,
                          IdentifierInfo *Name, SourceLocation NameLoc,
@@ -922,7 +941,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
     
     // Friend templates are visible in fairly strange ways.
     if (!CurContext->isDependentContext()) {
-      DeclContext *DC = SemanticContext->getLookupContext();
+      DeclContext *DC = SemanticContext->getRedeclContext();
       DC->makeDeclVisibleInContext(NewTemplate, /* Recoverable = */ false);
       if (Scope *EnclosingScope = getScopeForDeclContext(S, DC))
         PushOnScopeChains(NewTemplate, EnclosingScope,
@@ -941,7 +960,7 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
     NewTemplate->setInvalidDecl();
     NewClass->setInvalidDecl();
   }
-  return DeclPtrTy::make(NewTemplate);
+  return NewTemplate;
 }
 
 /// \brief Diagnose the presence of a default template argument on a
@@ -1102,7 +1121,6 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
           DiagnoseDefaultTemplateArgument(*this, TPC, 
                                           NewNonTypeParm->getLocation(), 
                     NewNonTypeParm->getDefaultArgument()->getSourceRange())) {
-        NewNonTypeParm->getDefaultArgument()->Destroy(Context);
         NewNonTypeParm->removeDefaultArgument();
       }
 
@@ -1477,14 +1495,10 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
                = dyn_cast<ClassTemplateDecl>(Template)) {
     // Find the class template specialization declaration that
     // corresponds to these arguments.
-    llvm::FoldingSetNodeID ID;
-    ClassTemplateSpecializationDecl::Profile(ID,
-                                             Converted.getFlatArguments(),
-                                             Converted.flatSize(),
-                                             Context);
     void *InsertPos = 0;
     ClassTemplateSpecializationDecl *Decl
-      = ClassTemplate->getSpecializations().FindNodeOrInsertPos(ID, InsertPos);
+      = ClassTemplate->findSpecialization(Converted.getFlatArguments(),
+                                          Converted.flatSize(), InsertPos);
     if (!Decl) {
       // This is the first time we have referenced this class template
       // specialization. Create the canonical declaration and add it to
@@ -1495,7 +1509,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
                                                 ClassTemplate->getLocation(),
                                                 ClassTemplate,
                                                 Converted, 0);
-      ClassTemplate->getSpecializations().InsertNode(Decl, InsertPos);
+      ClassTemplate->AddSpecialization(Decl, InsertPos);
       Decl->setLexicalDeclContext(CurContext);
     }
 
@@ -1510,7 +1524,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
   return Context.getTemplateSpecializationType(Name, TemplateArgs, CanonType);
 }
 
-Action::TypeResult
+TypeResult
 Sema::ActOnTemplateIdType(TemplateTy TemplateD, SourceLocation TemplateLoc,
                           SourceLocation LAngleLoc,
                           ASTTemplateArgsPtr TemplateArgsIn,
@@ -1536,15 +1550,15 @@ Sema::ActOnTemplateIdType(TemplateTy TemplateD, SourceLocation TemplateLoc,
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     TL.setArgLocInfo(i, TemplateArgs[i].getLocInfo());
 
-  return CreateLocInfoType(Result, DI).getAsOpaquePtr();
+  return CreateParsedType(Result, DI);
 }
 
-Sema::TypeResult Sema::ActOnTagTemplateIdType(TypeResult TypeResult,
-                                              TagUseKind TUK,
-                                              DeclSpec::TST TagSpec,
-                                              SourceLocation TagLoc) {
+TypeResult Sema::ActOnTagTemplateIdType(TypeResult TypeResult,
+                                        TagUseKind TUK,
+                                        TypeSpecifierType TagSpec,
+                                        SourceLocation TagLoc) {
   if (TypeResult.isInvalid())
-    return Sema::TypeResult();
+    return ::TypeResult();
 
   // FIXME: preserve source info, ideally without copying the DI.
   TypeSourceInfo *DI;
@@ -1571,10 +1585,10 @@ Sema::TypeResult Sema::ActOnTagTemplateIdType(TypeResult TypeResult,
     = TypeWithKeyword::getKeywordForTagTypeKind(TagKind);
   QualType ElabType = Context.getElaboratedType(Keyword, /*NNS=*/0, Type);
 
-  return ElabType.getAsOpaquePtr();
+  return ParsedType::make(ElabType);
 }
 
-Sema::OwningExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
+ExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
                                                  LookupResult &R,
                                                  bool RequiresADL,
                                  const TemplateArgumentListInfo &TemplateArgs) {
@@ -1603,7 +1617,7 @@ Sema::OwningExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
   UnresolvedLookupExpr *ULE
     = UnresolvedLookupExpr::Create(Context, Dependent, R.getNamingClass(),
                                    Qualifier, QualifierRange,
-                                   R.getLookupName(), R.getNameLoc(),
+                                   R.getLookupNameInfo(),
                                    RequiresADL, TemplateArgs, 
                                    R.begin(), R.end());
 
@@ -1611,19 +1625,18 @@ Sema::OwningExprResult Sema::BuildTemplateIdExpr(const CXXScopeSpec &SS,
 }
 
 // We actually only call this from template instantiation.
-Sema::OwningExprResult
+ExprResult
 Sema::BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS,
-                                   DeclarationName Name,
-                                   SourceLocation NameLoc,
+                                   const DeclarationNameInfo &NameInfo,
                              const TemplateArgumentListInfo &TemplateArgs) {
   DeclContext *DC;
   if (!(DC = computeDeclContext(SS, false)) ||
       DC->isDependentContext() ||
       RequireCompleteDeclContext(SS, DC))
-    return BuildDependentDeclRefExpr(SS, Name, NameLoc, &TemplateArgs);
+    return BuildDependentDeclRefExpr(SS, NameInfo, &TemplateArgs);
 
   bool MemberOfUnknownSpecialization;
-  LookupResult R(*this, Name, NameLoc, LookupOrdinaryName);
+  LookupResult R(*this, NameInfo, LookupOrdinaryName);
   LookupTemplateName(R, (Scope*) 0, SS, QualType(), /*Entering*/ false,
                      MemberOfUnknownSpecialization);
 
@@ -1631,14 +1644,15 @@ Sema::BuildQualifiedTemplateIdExpr(CXXScopeSpec &SS,
     return ExprError();
   
   if (R.empty()) {
-    Diag(NameLoc, diag::err_template_kw_refers_to_non_template)
-      << Name << SS.getRange();
+    Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_non_template)
+      << NameInfo.getName() << SS.getRange();
     return ExprError();
   }
 
   if (ClassTemplateDecl *Temp = R.getAsSingle<ClassTemplateDecl>()) {
-    Diag(NameLoc, diag::err_template_kw_refers_to_class_template)
-      << (NestedNameSpecifier*) SS.getScopeRep() << Name << SS.getRange();
+    Diag(NameInfo.getLoc(), diag::err_template_kw_refers_to_class_template)
+      << (NestedNameSpecifier*) SS.getScopeRep()
+      << NameInfo.getName() << SS.getRange();
     Diag(Temp->getLocation(), diag::note_referenced_class_template);
     return ExprError();
   }
@@ -1657,7 +1671,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
                                                   SourceLocation TemplateKWLoc,
                                                   CXXScopeSpec &SS,
                                                   UnqualifiedId &Name,
-                                                  TypeTy *ObjectType,
+                                                  ParsedType ObjectType,
                                                   bool EnteringContext,
                                                   TemplateTy &Result) {
   if (TemplateKWLoc.isValid() && S && !S->getTemplateParamParent() &&
@@ -1669,7 +1683,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
   if (SS.isSet())
     LookupCtx = computeDeclContext(SS, EnteringContext);
   if (!LookupCtx && ObjectType)
-    LookupCtx = computeDeclContext(QualType::getFromOpaquePtr(ObjectType));
+    LookupCtx = computeDeclContext(ObjectType.get());
   if (LookupCtx) {
     // C++0x [temp.names]p5:
     //   If a name prefixed by the keyword template is not the name of
@@ -1688,8 +1702,8 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
     // "template" keyword is now permitted). We follow the C++0x
     // rules, even in C++03 mode with a warning, retroactively applying the DR.
     bool MemberOfUnknownSpecialization;
-    TemplateNameKind TNK = isTemplateName(0, SS, Name, ObjectType,
-                                          EnteringContext, Result,
+    TemplateNameKind TNK = isTemplateName(0, SS, TemplateKWLoc.isValid(), Name,
+                                          ObjectType, EnteringContext, Result,
                                           MemberOfUnknownSpecialization);
     if (TNK == TNK_Non_template && LookupCtx->isDependentContext() &&
         isa<CXXRecordDecl>(LookupCtx) &&
@@ -1698,7 +1712,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
     } else if (TNK == TNK_Non_template) {
       Diag(Name.getSourceRange().getBegin(), 
            diag::err_template_kw_refers_to_non_template)
-        << GetNameFromUnqualifiedId(Name)
+        << GetNameFromUnqualifiedId(Name).getName()
         << Name.getSourceRange()
         << TemplateKWLoc;
       return TNK_Non_template;
@@ -1731,7 +1745,7 @@ TemplateNameKind Sema::ActOnDependentTemplateName(Scope *S,
   
   Diag(Name.getSourceRange().getBegin(), 
        diag::err_template_kw_refers_to_non_template)
-    << GetNameFromUnqualifiedId(Name)
+    << GetNameFromUnqualifiedId(Name).getName()
     << Name.getSourceRange()
     << TemplateKWLoc;
   return TNK_Non_template;
@@ -1856,7 +1870,7 @@ SubstDefaultTemplateArgument(Sema &SemaRef,
 /// parameters that precede \p Param in the template parameter list.
 ///
 /// \returns the substituted template argument, or NULL if an error occurred.
-static Sema::OwningExprResult
+static ExprResult
 SubstDefaultTemplateArgument(Sema &SemaRef,
                              TemplateDecl *Template,
                              SourceLocation TemplateLoc,
@@ -1952,7 +1966,7 @@ Sema::SubstDefaultTemplateArgumentIfAvailable(TemplateDecl *Template,
     if (!NonTypeParm->hasDefaultArgument())
       return TemplateArgumentLoc();
 
-    OwningExprResult Arg = SubstDefaultTemplateArgument(*this, Template,
+    ExprResult Arg = SubstDefaultTemplateArgument(*this, Template,
                                                         TemplateLoc,
                                                         RAngleLoc,
                                                         NonTypeParm,
@@ -2052,12 +2066,15 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
         // We have a template argument such as \c T::template X, which we
         // parsed as a template template argument. However, since we now
         // know that we need a non-type template argument, convert this
-        // template name into an expression.          
+        // template name into an expression.
+
+        DeclarationNameInfo NameInfo(DTN->getIdentifier(),
+                                     Arg.getTemplateNameLoc());
+
         Expr *E = DependentScopeDeclRefExpr::Create(Context,
                                                     DTN->getQualifier(),
                                                Arg.getTemplateQualifierRange(),
-                                                    DTN->getIdentifier(),
-                                                    Arg.getTemplateNameLoc());
+                                                    NameInfo);
         
         TemplateArgument Result;
         if (CheckTemplateArgument(NTTP, NTTPType, E, Result))
@@ -2272,7 +2289,7 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
         break;
       }
 
-      Sema::OwningExprResult E = SubstDefaultTemplateArgument(*this, Template,
+      ExprResult E = SubstDefaultTemplateArgument(*this, Template,
                                                               TemplateLoc, 
                                                               RAngleLoc, 
                                                               NTTP, 
@@ -2330,31 +2347,33 @@ bool Sema::CheckTemplateArgument(TemplateTypeParmDecl *Param,
   assert(ArgInfo && "invalid TypeSourceInfo");
   QualType Arg = ArgInfo->getType();
 
-  // C++ [temp.arg.type]p2:
+  // C++03 [temp.arg.type]p2:
   //   A local type, a type with no linkage, an unnamed type or a type
   //   compounded from any of these types shall not be used as a
   //   template-argument for a template type-parameter.
-  //
-  // FIXME: Perform the unnamed type check.
+  // C++0x allows these, and even in C++03 we allow them as an extension with
+  // a warning.
   SourceRange SR = ArgInfo->getTypeLoc().getSourceRange();
-  const TagType *Tag = 0;
-  if (const EnumType *EnumT = Arg->getAs<EnumType>())
-    Tag = EnumT;
-  else if (const RecordType *RecordT = Arg->getAs<RecordType>())
-    Tag = RecordT;
-  if (Tag && Tag->getDecl()->getDeclContext()->isFunctionOrMethod()) {
-    SourceRange SR = ArgInfo->getTypeLoc().getSourceRange();
-    return Diag(SR.getBegin(), diag::err_template_arg_local_type)
-      << QualType(Tag, 0) << SR;
-  } else if (Tag && !Tag->getDecl()->getDeclName() &&
-           !Tag->getDecl()->getTypedefForAnonDecl()) {
-    Diag(SR.getBegin(), diag::err_template_arg_unnamed_type) << SR;
-    Diag(Tag->getDecl()->getLocation(), diag::note_template_unnamed_type_here);
-    return true;
-  } else if (Arg->isVariablyModifiedType()) {
-    Diag(SR.getBegin(), diag::err_variably_modified_template_arg)
-      << Arg;
-    return true;
+  if (!LangOpts.CPlusPlus0x) {
+    const TagType *Tag = 0;
+    if (const EnumType *EnumT = Arg->getAs<EnumType>())
+      Tag = EnumT;
+    else if (const RecordType *RecordT = Arg->getAs<RecordType>())
+      Tag = RecordT;
+    if (Tag && Tag->getDecl()->getDeclContext()->isFunctionOrMethod()) {
+      SourceRange SR = ArgInfo->getTypeLoc().getSourceRange();
+      Diag(SR.getBegin(), diag::ext_template_arg_local_type)
+        << QualType(Tag, 0) << SR;
+    } else if (Tag && !Tag->getDecl()->getDeclName() &&
+               !Tag->getDecl()->getTypedefForAnonDecl()) {
+      Diag(SR.getBegin(), diag::ext_template_arg_unnamed_type) << SR;
+      Diag(Tag->getDecl()->getLocation(),
+           diag::note_template_unnamed_type_here);
+    }
+  }
+
+  if (Arg->isVariablyModifiedType()) {
+    return Diag(SR.getBegin(), diag::err_variably_modified_template_arg) << Arg;
   } else if (Context.hasSameUnqualifiedType(Arg, Context.OverloadTy)) {
     return Diag(SR.getBegin(), diag::err_template_arg_overload_type) << SR;
   }
@@ -2406,7 +2425,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
   bool AddressTaken = false;
   SourceLocation AddrOpLoc;
   if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(Arg)) {
-    if (UnOp->getOpcode() == UnaryOperator::AddrOf) {
+    if (UnOp->getOpcode() == UO_AddrOf) {
       DRE = dyn_cast<DeclRefExpr>(UnOp->getSubExpr());
       AddressTaken = true;
       AddrOpLoc = UnOp->getOperatorLoc();
@@ -2653,7 +2672,7 @@ bool Sema::CheckTemplateArgumentPointerToMember(Expr *Arg,
 
   // A pointer-to-member constant written &Class::member.
   if (UnaryOperator *UnOp = dyn_cast<UnaryOperator>(Arg)) {
-    if (UnOp->getOpcode() == UnaryOperator::AddrOf) {
+    if (UnOp->getOpcode() == UO_AddrOf) {
       DRE = dyn_cast<DeclRefExpr>(UnOp->getSubExpr());
       if (DRE && !DRE->getQualifier())
         DRE = 0;
@@ -2788,7 +2807,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     } else if (IsIntegralPromotion(Arg, ArgType, ParamType) ||
                !ParamType->isEnumeralType()) {
       // This is an integral promotion or conversion.
-      ImpCastExprToType(Arg, ParamType, CastExpr::CK_IntegralCast);
+      ImpCastExprToType(Arg, ParamType, CK_IntegralCast);
     } else {
       // We can't perform this conversion.
       Diag(Arg->getSourceRange().getBegin(),
@@ -2909,8 +2928,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
                                                             Arg, Converted);
 
     if (IsQualificationConversion(ArgType, ParamType.getNonReferenceType())) {
-      ImpCastExprToType(Arg, ParamType, CastExpr::CK_NoOp,
-                        Arg->isLvalue(Context) == Expr::LV_Valid);
+      ImpCastExprToType(Arg, ParamType, CK_NoOp, CastCategory(Arg));
     } else if (!Context.hasSameUnqualifiedType(ArgType,
                                            ParamType.getNonReferenceType())) {
       // We can't perform this conversion.
@@ -2929,7 +2947,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     //      object, qualification conversions (4.4) and the
     //      array-to-pointer conversion (4.2) are applied.
     // C++0x also allows a value of std::nullptr_t.
-    assert(ParamType->getAs<PointerType>()->getPointeeType()->isObjectType() &&
+    assert(ParamType->getPointeeType()->isIncompleteOrObjectType() &&
            "Only object pointers allowed here");
 
     return CheckTemplateArgumentAddressOfObjectOrFunction(*this, Param, 
@@ -2944,7 +2962,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
     //      identical) type of the template-argument. The
     //      template-parameter is bound directly to the
     //      template-argument, which must be an lvalue.
-    assert(ParamRefType->getPointeeType()->isObjectType() &&
+    assert(ParamRefType->getPointeeType()->isIncompleteOrObjectType() &&
            "Only object references allowed here");
 
     if (Arg->getType() == Context.OverloadTy) {
@@ -2973,8 +2991,7 @@ bool Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
   if (Context.hasSameUnqualifiedType(ParamType, ArgType)) {
     // Types match exactly: nothing more to do here.
   } else if (IsQualificationConversion(ArgType, ParamType)) {
-    ImpCastExprToType(Arg, ParamType, CastExpr::CK_NoOp,
-                      Arg->isLvalue(Context) == Expr::LV_Valid);
+    ImpCastExprToType(Arg, ParamType, CK_NoOp, CastCategory(Arg));
   } else {
     // We can't perform this conversion.
     Diag(Arg->getSourceRange().getBegin(),
@@ -3033,7 +3050,7 @@ bool Sema::CheckTemplateArgument(TemplateTemplateParmDecl *Param,
 /// declaration and the type of its corresponding non-type template
 /// parameter, produce an expression that properly refers to that
 /// declaration.
-Sema::OwningExprResult 
+ExprResult 
 Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
                                               QualType ParamType,
                                               SourceLocation Loc) {
@@ -3052,17 +3069,18 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
       QualType ClassType
         = Context.getTypeDeclType(cast<RecordDecl>(VD->getDeclContext()));
       NestedNameSpecifier *Qualifier
-        = NestedNameSpecifier::Create(Context, 0, false, ClassType.getTypePtr());
+        = NestedNameSpecifier::Create(Context, 0, false,
+                                      ClassType.getTypePtr());
       CXXScopeSpec SS;
       SS.setScopeRep(Qualifier);
-      OwningExprResult RefExpr = BuildDeclRefExpr(VD, 
+      ExprResult RefExpr = BuildDeclRefExpr(VD, 
                                            VD->getType().getNonReferenceType(), 
                                                   Loc,
                                                   &SS);
       if (RefExpr.isInvalid())
         return ExprError();
       
-      RefExpr = CreateBuiltinUnaryOp(Loc, UnaryOperator::AddrOf, move(RefExpr));
+      RefExpr = CreateBuiltinUnaryOp(Loc, UO_AddrOf, RefExpr.get());
       
       // We might need to perform a trailing qualification conversion, since
       // the element type on the parameter could be more qualified than the
@@ -3070,8 +3088,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
       if (IsQualificationConversion(((Expr*) RefExpr.get())->getType(),
                                     ParamType.getUnqualifiedType())) {
         Expr *RefE = RefExpr.takeAs<Expr>();
-        ImpCastExprToType(RefE, ParamType.getUnqualifiedType(),
-                          CastExpr::CK_NoOp);
+        ImpCastExprToType(RefE, ParamType.getUnqualifiedType(), CK_NoOp);
         RefExpr = Owned(RefE);
       }
       
@@ -3086,7 +3103,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
   if (ParamType->isPointerType()) {
     // When the non-type template parameter is a pointer, take the
     // address of the declaration.
-    OwningExprResult RefExpr = BuildDeclRefExpr(VD, T, Loc);
+    ExprResult RefExpr = BuildDeclRefExpr(VD, T, Loc);
     if (RefExpr.isInvalid())
       return ExprError();
 
@@ -3103,7 +3120,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
     }
     
     // Take the address of everything else
-    return CreateBuiltinUnaryOp(Loc, UnaryOperator::AddrOf, move(RefExpr));
+    return CreateBuiltinUnaryOp(Loc, UO_AddrOf, RefExpr.get());
   }
 
   // If the non-type template parameter has reference type, qualify the
@@ -3122,7 +3139,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
 /// This routine takes care of the mapping from an integral template
 /// argument (which may have any integral type) to the appropriate
 /// literal value.
-Sema::OwningExprResult 
+ExprResult 
 Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
                                                   SourceLocation Loc) {
   assert(Arg.getKind() == TemplateArgument::Integral &&
@@ -3140,7 +3157,7 @@ Sema::BuildExpressionFromIntegralTemplateArgument(const TemplateArgument &Arg,
                                             T,
                                             Loc));
 
-  return Owned(new (Context) IntegerLiteral(*Arg.getAsIntegral(), T, Loc));
+  return Owned(IntegerLiteral::Create(Context, *Arg.getAsIntegral(), T, Loc));
 }
 
 
@@ -3401,7 +3418,7 @@ static bool CheckTemplateSpecializationScope(Sema &S,
   //   the explicit specialization was declared, or in a namespace
   //   that encloses the one in which the explicit specialization was
   //   declared.
-  if (S.CurContext->getLookupContext()->isFunctionOrMethod()) {
+  if (S.CurContext->getRedeclContext()->isFunctionOrMethod()) {
     S.Diag(Loc, diag::err_template_spec_decl_function_scope)
       << Specialized;
     return true;
@@ -3426,8 +3443,8 @@ static bool CheckTemplateSpecializationScope(Sema &S,
        getTemplateSpecializationKind(PrevDecl) == TSK_ImplicitInstantiation)){
     // There is no prior declaration of this entity, so this
     // specialization must be in the same context as the template
-    // itself.
-    if (!DC->Equals(SpecializedContext)) {
+    // itself, or in the enclosing namespace set.
+    if (!DC->InEnclosingNamespaceSetOf(SpecializedContext)) {
       if (isa<TranslationUnitDecl>(SpecializedContext))
         S.Diag(Loc, diag::err_template_spec_decl_out_of_scope_global)
         << EntityKind << Specialized;
@@ -3597,7 +3614,7 @@ static NamedDecl *getPreviousDecl(NamedDecl *ND) {
   return 0;
 }
 
-Sema::DeclResult
+DeclResult
 Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                        TagUseKind TUK,
                                        SourceLocation KWLoc,
@@ -3666,7 +3683,6 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                diag::err_default_arg_in_partial_spec)
             << DefArg->getSourceRange();
           NTTP->removeDefaultArgument();
-          DefArg->Destroy(Context);
         }
       } else {
         TemplateTemplateParmDecl *TTP = cast<TemplateTemplateParmDecl>(Param);
@@ -3729,7 +3745,6 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
 
   // Find the class template (partial) specialization declaration that
   // corresponds to these arguments.
-  llvm::FoldingSetNodeID ID;
   if (isPartialSpecialization) {
     bool MirrorsPrimaryTemplate;
     if (CheckClassTemplatePartialSpecializationArgs(
@@ -3762,30 +3777,22 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
       Diag(TemplateNameLoc, diag::err_partial_spec_fully_specialized)
         << ClassTemplate->getDeclName();
       isPartialSpecialization = false;
-    } else {
-      // FIXME: Template parameter list matters, too
-      ClassTemplatePartialSpecializationDecl::Profile(ID,
-                                                  Converted.getFlatArguments(),
-                                                      Converted.flatSize(),
-                                                      Context);
     }
   }
-  
-  if (!isPartialSpecialization)
-    ClassTemplateSpecializationDecl::Profile(ID,
-                                             Converted.getFlatArguments(),
-                                             Converted.flatSize(),
-                                             Context);
+
   void *InsertPos = 0;
   ClassTemplateSpecializationDecl *PrevDecl = 0;
 
   if (isPartialSpecialization)
+    // FIXME: Template parameter list matters, too
     PrevDecl
-      = ClassTemplate->getPartialSpecializations().FindNodeOrInsertPos(ID,
-                                                                    InsertPos);
+      = ClassTemplate->findPartialSpecialization(Converted.getFlatArguments(),
+                                                 Converted.flatSize(),
+                                                 InsertPos);
   else
     PrevDecl
-      = ClassTemplate->getSpecializations().FindNodeOrInsertPos(ID, InsertPos);
+      = ClassTemplate->findSpecialization(Converted.getFlatArguments(),
+                                          Converted.flatSize(), InsertPos);
 
   ClassTemplateSpecializationDecl *Specialization = 0;
 
@@ -3823,7 +3830,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     ClassTemplatePartialSpecializationDecl *PrevPartial
       = cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl);
     unsigned SequenceNumber = PrevPartial? PrevPartial->getSequenceNumber()
-                            : ClassTemplate->getPartialSpecializations().size();
+                            : ClassTemplate->getNextPartialSpecSequenceNumber();
     ClassTemplatePartialSpecializationDecl *Partial
       = ClassTemplatePartialSpecializationDecl::Create(Context, Kind,
                                              ClassTemplate->getDeclContext(),
@@ -3836,18 +3843,14 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                                        PrevPartial,
                                                        SequenceNumber);
     SetNestedNameSpecifier(Partial, SS);
-    if (NumMatchedTemplateParamLists > 0) {
+    if (NumMatchedTemplateParamLists > 0 && SS.isSet()) {
       Partial->setTemplateParameterListsInfo(Context,
                                              NumMatchedTemplateParamLists,
                     (TemplateParameterList**) TemplateParameterLists.release());
     }
 
-    if (PrevPartial) {
-      ClassTemplate->getPartialSpecializations().RemoveNode(PrevPartial);
-      ClassTemplate->getPartialSpecializations().GetOrInsertNode(Partial);
-    } else {
-      ClassTemplate->getPartialSpecializations().InsertNode(Partial, InsertPos);
-    }
+    if (!PrevPartial)
+      ClassTemplate->AddPartialSpecialization(Partial, InsertPos);
     Specialization = Partial;
 
     // If we are providing an explicit specialization of a member class 
@@ -3883,7 +3886,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
           else
             Diag(Param->getLocation(),
                  diag::note_partial_spec_unused_parameter)
-              << std::string("<anonymous>");
+              << "<anonymous>";
         }
       }
     }
@@ -3898,19 +3901,14 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                                 Converted,
                                                 PrevDecl);
     SetNestedNameSpecifier(Specialization, SS);
-    if (NumMatchedTemplateParamLists > 0) {
+    if (NumMatchedTemplateParamLists > 0 && SS.isSet()) {
       Specialization->setTemplateParameterListsInfo(Context,
                                                   NumMatchedTemplateParamLists,
                     (TemplateParameterList**) TemplateParameterLists.release());
     }
 
-    if (PrevDecl) {
-      ClassTemplate->getSpecializations().RemoveNode(PrevDecl);
-      ClassTemplate->getSpecializations().GetOrInsertNode(Specialization);
-    } else {
-      ClassTemplate->getSpecializations().InsertNode(Specialization,
-                                                     InsertPos);
-    }
+    if (!PrevDecl)
+      ClassTemplate->AddSpecialization(Specialization, InsertPos);
 
     CanonType = Context.getTypeDeclType(Specialization);
   }
@@ -4004,20 +4002,18 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     // context. However, specializations are not found by name lookup.
     CurContext->addDecl(Specialization);
   }
-  return DeclPtrTy::make(Specialization);
+  return Specialization;
 }
 
-Sema::DeclPtrTy
-Sema::ActOnTemplateDeclarator(Scope *S,
+Decl *Sema::ActOnTemplateDeclarator(Scope *S,
                               MultiTemplateParamsArg TemplateParameterLists,
-                              Declarator &D) {
+                                    Declarator &D) {
   return HandleDeclarator(S, D, move(TemplateParameterLists), false);
 }
 
-Sema::DeclPtrTy
-Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
+Decl *Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
                                MultiTemplateParamsArg TemplateParameterLists,
-                                      Declarator &D) {
+                                            Declarator &D) {
   assert(getCurFunctionDecl() == 0 && "Function parsing confused");
   assert(D.getTypeObject(0).Kind == DeclaratorChunk::Function &&
          "Not a function declarator!");
@@ -4029,22 +4025,22 @@ Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
 
   Scope *ParentScope = FnBodyScope->getParent();
 
-  DeclPtrTy DP = HandleDeclarator(ParentScope, D,
-                                  move(TemplateParameterLists),
-                                  /*IsFunctionDefinition=*/true);
+  Decl *DP = HandleDeclarator(ParentScope, D,
+                              move(TemplateParameterLists),
+                              /*IsFunctionDefinition=*/true);
   if (FunctionTemplateDecl *FunctionTemplate
-        = dyn_cast_or_null<FunctionTemplateDecl>(DP.getAs<Decl>()))
+        = dyn_cast_or_null<FunctionTemplateDecl>(DP))
     return ActOnStartOfFunctionDef(FnBodyScope,
-                      DeclPtrTy::make(FunctionTemplate->getTemplatedDecl()));
-  if (FunctionDecl *Function = dyn_cast_or_null<FunctionDecl>(DP.getAs<Decl>()))
-    return ActOnStartOfFunctionDef(FnBodyScope, DeclPtrTy::make(Function));
-  return DeclPtrTy();
+                                   FunctionTemplate->getTemplatedDecl());
+  if (FunctionDecl *Function = dyn_cast_or_null<FunctionDecl>(DP))
+    return ActOnStartOfFunctionDef(FnBodyScope, Function);
+  return 0;
 }
 
 /// \brief Strips various properties off an implicit instantiation
 /// that has just been explicitly specialized.
 static void StripImplicitInstantiation(NamedDecl *D) {
-  D->invalidateAttrs();
+  D->dropAttrs();
 
   if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     FD->setInlineSpecified(false);
@@ -4241,12 +4237,13 @@ Sema::CheckDependentFunctionTemplateSpecialization(FunctionDecl *FD,
                                                    LookupResult &Previous) {
   // Remove anything from Previous that isn't a function template in
   // the correct context.
-  DeclContext *FDLookupContext = FD->getDeclContext()->getLookupContext();
+  DeclContext *FDLookupContext = FD->getDeclContext()->getRedeclContext();
   LookupResult::Filter F = Previous.makeFilter();
   while (F.hasNext()) {
     NamedDecl *D = F.next()->getUnderlyingDecl();
     if (!isa<FunctionTemplateDecl>(D) ||
-        !FDLookupContext->Equals(D->getDeclContext()->getLookupContext()))
+        !FDLookupContext->InEnclosingNamespaceSetOf(
+                              D->getDeclContext()->getRedeclContext()))
       F.erase();
   }
   F.done();
@@ -4285,14 +4282,15 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
   // explicit function template specialization.
   UnresolvedSet<8> Candidates;
   
-  DeclContext *FDLookupContext = FD->getDeclContext()->getLookupContext();
+  DeclContext *FDLookupContext = FD->getDeclContext()->getRedeclContext();
   for (LookupResult::iterator I = Previous.begin(), E = Previous.end();
          I != E; ++I) {
     NamedDecl *Ovl = (*I)->getUnderlyingDecl();
     if (FunctionTemplateDecl *FunTmpl = dyn_cast<FunctionTemplateDecl>(Ovl)) {
       // Only consider templates found within the same semantic lookup scope as 
       // FD.
-      if (!FDLookupContext->Equals(Ovl->getDeclContext()->getLookupContext()))
+      if (!FDLookupContext->InEnclosingNamespaceSetOf(
+                                Ovl->getDeclContext()->getRedeclContext()))
         continue;
       
       // C++ [temp.expl.spec]p11:
@@ -4373,8 +4371,10 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
   
   // Mark the prior declaration as an explicit specialization, so that later
   // clients know that this is an explicit specialization.
-  if (!isFriend)
+  if (!isFriend) {
     SpecInfo->setTemplateSpecializationKind(TSK_ExplicitSpecialization);
+    MarkUnusedFileScopedDecl(Specialization);
+  }
   
   // Turn the given function declaration into a function template
   // specialization, with the template arguments from the previous
@@ -4527,6 +4527,7 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
     cast<FunctionDecl>(Member)->setInstantiationOfMemberFunction(
                                         cast<CXXMethodDecl>(InstantiatedFrom),
                                                   TSK_ExplicitSpecialization);
+    MarkUnusedFileScopedDecl(InstantiationFunction);
   } else if (isa<VarDecl>(Member)) {
     VarDecl *InstantiationVar = cast<VarDecl>(Instantiation);
     if (InstantiationVar->getTemplateSpecializationKind() ==
@@ -4539,6 +4540,7 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
     Context.setInstantiatedFromStaticDataMember(cast<VarDecl>(Member),
                                                 cast<VarDecl>(InstantiatedFrom),
                                                 TSK_ExplicitSpecialization);
+    MarkUnusedFileScopedDecl(InstantiationVar);
   } else {
     assert(isa<CXXRecordDecl>(Member) && "Only member classes remain");
     CXXRecordDecl *InstantiationClass = cast<CXXRecordDecl>(Instantiation);
@@ -4567,9 +4569,8 @@ Sema::CheckMemberSpecialization(NamedDecl *Member, LookupResult &Previous) {
 static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
                                             SourceLocation InstLoc,
                                             bool WasQualifiedName) {
-  DeclContext *ExpectedContext
-    = D->getDeclContext()->getEnclosingNamespaceContext()->getLookupContext();
-  DeclContext *CurContext = S.CurContext->getLookupContext();
+  DeclContext *OrigContext= D->getDeclContext()->getEnclosingNamespaceContext();
+  DeclContext *CurContext = S.CurContext->getRedeclContext();
   
   if (CurContext->isRecord()) {
     S.Diag(InstLoc, diag::err_explicit_instantiation_in_class)
@@ -4583,8 +4584,8 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
   //
   // This is DR275, which we do not retroactively apply to C++98/03.
   if (S.getLangOptions().CPlusPlus0x && 
-      !CurContext->Encloses(ExpectedContext)) {
-    if (NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ExpectedContext))
+      !CurContext->Encloses(OrigContext)) {
+    if (NamespaceDecl *NS = dyn_cast<NamespaceDecl>(OrigContext))
       S.Diag(InstLoc, 
              S.getLangOptions().CPlusPlus0x? 
                  diag::err_explicit_instantiation_out_of_scope
@@ -4599,7 +4600,7 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
     S.Diag(D->getLocation(), diag::note_explicit_instantiation_here);
     return false;
   }
-  
+
   // C++0x [temp.explicit]p2:
   //   If the name declared in the explicit instantiation is an unqualified 
   //   name, the explicit instantiation shall appear in the namespace where 
@@ -4607,15 +4608,15 @@ static bool CheckExplicitInstantiationScope(Sema &S, NamedDecl *D,
   //   namespace from its enclosing namespace set.
   if (WasQualifiedName)
     return false;
-  
-  if (CurContext->Equals(ExpectedContext))
+
+  if (CurContext->InEnclosingNamespaceSetOf(OrigContext))
     return false;
-  
+
   S.Diag(InstLoc, 
          S.getLangOptions().CPlusPlus0x?
              diag::err_explicit_instantiation_unqualified_wrong_namespace
            : diag::warn_explicit_instantiation_unqualified_wrong_namespace_0x)
-    << D << ExpectedContext;
+    << D << OrigContext;
   S.Diag(D->getLocation(), diag::note_explicit_instantiation_here);
   return false;
 }
@@ -4642,7 +4643,7 @@ static bool ScopeSpecifierHasTemplateId(const CXXScopeSpec &SS) {
 }
 
 // Explicit instantiation of a class template specialization
-Sema::DeclResult
+DeclResult
 Sema::ActOnExplicitInstantiation(Scope *S,
                                  SourceLocation ExternLoc,
                                  SourceLocation TemplateLoc,
@@ -4703,14 +4704,10 @@ Sema::ActOnExplicitInstantiation(Scope *S,
 
   // Find the class template specialization declaration that
   // corresponds to these arguments.
-  llvm::FoldingSetNodeID ID;
-  ClassTemplateSpecializationDecl::Profile(ID,
-                                           Converted.getFlatArguments(),
-                                           Converted.flatSize(),
-                                           Context);
   void *InsertPos = 0;
   ClassTemplateSpecializationDecl *PrevDecl
-    = ClassTemplate->getSpecializations().FindNodeOrInsertPos(ID, InsertPos);
+    = ClassTemplate->findSpecialization(Converted.getFlatArguments(),
+                                        Converted.flatSize(), InsertPos);
 
   TemplateSpecializationKind PrevDecl_TSK
     = PrevDecl ? PrevDecl->getTemplateSpecializationKind() : TSK_Undeclared;
@@ -4733,7 +4730,7 @@ Sema::ActOnExplicitInstantiation(Scope *S,
                                                PrevDecl, PrevDecl_TSK,
                                             PrevDecl->getPointOfInstantiation(),
                                                HasNoEffect))
-      return DeclPtrTy::make(PrevDecl);
+      return PrevDecl;
 
     // Even though HasNoEffect == true means that this explicit instantiation
     // has no effect on semantics, we go on to put its syntax in the AST.
@@ -4763,15 +4760,9 @@ Sema::ActOnExplicitInstantiation(Scope *S,
                                                 Converted, PrevDecl);
     SetNestedNameSpecifier(Specialization, SS);
 
-    if (!HasNoEffect) {
-      if (PrevDecl) {
-        // Remove the previous declaration from the folding set, since we want
-        // to introduce a new declaration.
-        ClassTemplate->getSpecializations().RemoveNode(PrevDecl);
-        ClassTemplate->getSpecializations().FindNodeOrInsertPos(ID, InsertPos);
-      }
+    if (!HasNoEffect && !PrevDecl) {
       // Insert the new specialization.
-      ClassTemplate->getSpecializations().InsertNode(Specialization, InsertPos);
+      ClassTemplate->AddSpecialization(Specialization, InsertPos);
     }
   }
 
@@ -4803,7 +4794,7 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   if (HasNoEffect) {
     // Set the template specialization kind.
     Specialization->setTemplateSpecializationKind(TSK);
-    return DeclPtrTy::make(Specialization);
+    return Specialization;
   }
 
   // C++ [temp.explicit]p3:
@@ -4840,11 +4831,11 @@ Sema::ActOnExplicitInstantiation(Scope *S,
 
   // Set the template specialization kind.
   Specialization->setTemplateSpecializationKind(TSK);
-  return DeclPtrTy::make(Specialization);
+  return Specialization;
 }
 
 // Explicit instantiation of a member class of a class template.
-Sema::DeclResult
+DeclResult
 Sema::ActOnExplicitInstantiation(Scope *S,
                                  SourceLocation ExternLoc,
                                  SourceLocation TemplateLoc,
@@ -4857,16 +4848,16 @@ Sema::ActOnExplicitInstantiation(Scope *S,
 
   bool Owned = false;
   bool IsDependent = false;
-  DeclPtrTy TagD = ActOnTag(S, TagSpec, Action::TUK_Reference,
-                            KWLoc, SS, Name, NameLoc, Attr, AS_none,
-                            MultiTemplateParamsArg(*this, 0, 0),
-                            Owned, IsDependent);
+  Decl *TagD = ActOnTag(S, TagSpec, Sema::TUK_Reference,
+                        KWLoc, SS, Name, NameLoc, Attr, AS_none,
+                        MultiTemplateParamsArg(*this, 0, 0),
+                        Owned, IsDependent);
   assert(!IsDependent && "explicit instantiation of dependent name not yet handled");
 
   if (!TagD)
     return true;
 
-  TagDecl *Tag = cast<TagDecl>(TagD.getAs<Decl>());
+  TagDecl *Tag = cast<TagDecl>(TagD);
   if (Tag->isEnum()) {
     Diag(TemplateLoc, diag::err_explicit_instantiation_enum)
       << Context.getTypeDeclType(Tag);
@@ -4969,12 +4960,14 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   return TagD;
 }
 
-Sema::DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
-                                                  SourceLocation ExternLoc,
-                                                  SourceLocation TemplateLoc,
-                                                  Declarator &D) {
+DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
+                                            SourceLocation ExternLoc,
+                                            SourceLocation TemplateLoc,
+                                            Declarator &D) {
   // Explicit instantiations always require a name.
-  DeclarationName Name = GetNameForDeclarator(D);
+  // TODO: check if/when DNInfo should replace Name.
+  DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
+  DeclarationName Name = NameInfo.getName();
   if (!Name) {
     if (!D.isInvalidType())
       Diag(D.getDeclSpec().getSourceRange().getBegin(),
@@ -5024,7 +5017,7 @@ Sema::DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     = ExternLoc.isInvalid()? TSK_ExplicitInstantiationDefinition
                            : TSK_ExplicitInstantiationDeclaration;
     
-  LookupResult Previous(*this, Name, D.getIdentifierLoc(), LookupOrdinaryName);
+  LookupResult Previous(*this, NameInfo, LookupOrdinaryName);
   LookupParsedName(Previous, S, &D.getCXXScopeSpec());
 
   if (!R->isFunctionType()) {
@@ -5081,16 +5074,15 @@ Sema::DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
                                                HasNoEffect))
       return true;
     if (HasNoEffect)
-      return DeclPtrTy();
+      return (Decl*) 0;
     
     // Instantiate static data member.
     Prev->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
     if (TSK == TSK_ExplicitInstantiationDefinition)
-      InstantiateStaticDataMemberDefinition(D.getIdentifierLoc(), Prev, false,
-                                            /*DefinitionRequired=*/true);
+      InstantiateStaticDataMemberDefinition(D.getIdentifierLoc(), Prev);
     
     // FIXME: Create an ExplicitInstantiation node?
-    return DeclPtrTy();
+    return (Decl*) 0;
   }
   
   // If the declarator is a template-id, translate the parser's template 
@@ -5188,14 +5180,13 @@ Sema::DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     // FIXME: We may still want to build some representation of this
     // explicit specialization.
     if (HasNoEffect)
-      return DeclPtrTy();
+      return (Decl*) 0;
   }
 
   Specialization->setTemplateSpecializationKind(TSK, D.getIdentifierLoc());
   
   if (TSK == TSK_ExplicitInstantiationDefinition)
-    InstantiateFunctionDefinition(D.getIdentifierLoc(), Specialization, 
-                                  false, /*DefinitionRequired=*/true);
+    InstantiateFunctionDefinition(D.getIdentifierLoc(), Specialization);
  
   // C++0x [temp.explicit]p2:
   //   If the explicit instantiation is for a member function, a member class 
@@ -5219,10 +5210,10 @@ Sema::DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
                                   D.getCXXScopeSpec().isSet());
   
   // FIXME: Create some kind of ExplicitInstantiationDecl here.
-  return DeclPtrTy();
+  return (Decl*) 0;
 }
 
-Sema::TypeResult
+TypeResult
 Sema::ActOnDependentTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
                         const CXXScopeSpec &SS, IdentifierInfo *Name,
                         SourceLocation TagLoc, SourceLocation NameLoc) {
@@ -5243,10 +5234,10 @@ Sema::ActOnDependentTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   }
 
   ElaboratedTypeKeyword Kwd = TypeWithKeyword::getKeywordForTagTypeKind(Kind);
-  return Context.getDependentNameType(Kwd, NNS, Name).getAsOpaquePtr();
+  return ParsedType::make(Context.getDependentNameType(Kwd, NNS, Name));
 }
 
-Sema::TypeResult
+TypeResult
 Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, 
                         const CXXScopeSpec &SS, const IdentifierInfo &II, 
                         SourceLocation IdLoc) {
@@ -5278,13 +5269,13 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     cast<TypeSpecTypeLoc>(TL.getNamedTypeLoc()).setNameLoc(IdLoc);
   }
   
-  return CreateLocInfoType(T, TSI).getAsOpaquePtr();
+  return CreateParsedType(T, TSI);
 }
 
-Sema::TypeResult
+TypeResult
 Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc, 
                         const CXXScopeSpec &SS, SourceLocation TemplateLoc, 
-                        TypeTy *Ty) {
+                        ParsedType Ty) {
   if (TypenameLoc.isValid() && S && !S->getTemplateParamParent() &&
       !getLangOptions().CPlusPlus0x)
     Diag(TypenameLoc, diag::ext_typename_outside_of_template)
@@ -5292,8 +5283,6 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   
   TypeSourceInfo *InnerTSI = 0;
   QualType T = GetTypeFromParser(Ty, &InnerTSI);
-  NestedNameSpecifier *NNS
-    = static_cast<NestedNameSpecifier *>(SS.getScopeRep());
 
   assert(isa<TemplateSpecializationType>(T) && 
          "Expected a template specialization type");
@@ -5310,13 +5299,14 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     else
       Builder.push<TemplateSpecializationTypeLoc>(T).initialize(TemplateLoc);
 
-    T = Context.getElaboratedType(ETK_Typename, NNS, T);
+    /* Note: NNS already embedded in template specialization type T. */
+    T = Context.getElaboratedType(ETK_Typename, /*NNS=*/0, T);
     ElaboratedTypeLoc TL = Builder.push<ElaboratedTypeLoc>(T);
     TL.setKeywordLoc(TypenameLoc);
     TL.setQualifierRange(SS.getRange());
 
     TypeSourceInfo *TSI = Builder.getTypeSourceInfo(Context, T);
-    return CreateLocInfoType(T, TSI).getAsOpaquePtr();
+    return CreateParsedType(T, TSI);
   }
 
   // TODO: it's really silly that we make a template specialization
@@ -5325,7 +5315,10 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   DependentTemplateName *DTN =
     TST->getTemplateName().getAsDependentTemplateName();
   assert(DTN && "dependent template has non-dependent name?");
-  T = Context.getDependentTemplateSpecializationType(ETK_Typename, NNS,
+  assert(DTN->getQualifier()
+         == static_cast<NestedNameSpecifier*>(SS.getScopeRep()));
+  T = Context.getDependentTemplateSpecializationType(ETK_Typename,
+                                                     DTN->getQualifier(),
                                                      DTN->getIdentifier(),
                                                      TST->getNumArgs(),
                                                      TST->getArgs());
@@ -5344,7 +5337,7 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   }
   TL.setKeywordLoc(TypenameLoc);
   TL.setQualifierRange(SS.getRange());
-  return CreateLocInfoType(T, TSI).getAsOpaquePtr();
+  return CreateParsedType(T, TSI);
 }
 
 /// \brief Build the type that describes a C++ typename specifier,
@@ -5463,15 +5456,6 @@ namespace {
       this->Loc = Loc;
       this->Entity = Entity;
     }
-      
-    /// \brief Transforms an expression by returning the expression itself
-    /// (an identity function).
-    ///
-    /// FIXME: This is completely unsafe; we will need to actually clone the
-    /// expressions.
-    Sema::OwningExprResult TransformExpr(Expr *E) {
-      return getSema().Owned(E->Retain());
-    }
   };
 }
 
@@ -5511,6 +5495,12 @@ TypeSourceInfo *Sema::RebuildTypeInCurrentInstantiation(TypeSourceInfo *T,
   return Rebuilder.TransformType(T);
 }
 
+ExprResult Sema::RebuildExprInCurrentInstantiation(Expr *E) {
+  CurrentInstantiationRebuilder Rebuilder(*this, E->getExprLoc(),
+                                          DeclarationName());
+  return Rebuilder.TransformExpr(E);
+}
+
 bool Sema::RebuildNestedNameSpecifierInCurrentInstantiation(CXXScopeSpec &SS) {
   if (SS.isInvalid()) return true;
 
diff --git a/lib/Sema/SemaTemplate.h b/lib/Sema/SemaTemplate.h
deleted file mode 100644
index b3f46519ab7f..000000000000
--- a/lib/Sema/SemaTemplate.h
+++ /dev/null
@@ -1,151 +0,0 @@
-//===------- SemaTemplate.h - C++ Templates ---------------------*- C++ -*-===/
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//===----------------------------------------------------------------------===/
-//
-//  This file provides types used in the semantic analysis of C++ templates.
-//
-//===----------------------------------------------------------------------===/
-#ifndef LLVM_CLANG_SEMA_TEMPLATE_H
-#define LLVM_CLANG_SEMA_TEMPLATE_H
-
-#include "clang/AST/DeclTemplate.h"
-#include "llvm/ADT/SmallVector.h"
-#include <cassert>
-
-namespace clang {
-  /// \brief Data structure that captures multiple levels of template argument
-  /// lists for use in template instantiation.
-  ///
-  /// Multiple levels of template arguments occur when instantiating the 
-  /// definitions of member templates. For example:
-  ///
-  /// \code
-  /// template<typename T>
-  /// struct X {
-  ///   template<T Value>
-  ///   struct Y {
-  ///     void f();
-  ///   };
-  /// };
-  /// \endcode
-  ///
-  /// When instantiating X<int>::Y<17>::f, the multi-level template argument
-  /// list will contain a template argument list (int) at depth 0 and a
-  /// template argument list (17) at depth 1.
-  class MultiLevelTemplateArgumentList {
-  public:
-    typedef std::pair<const TemplateArgument *, unsigned> ArgList;
-    
-  private:
-    /// \brief The template argument lists, stored from the innermost template
-    /// argument list (first) to the outermost template argument list (last).
-    llvm::SmallVector<ArgList, 4> TemplateArgumentLists;
-    
-  public:
-    /// \brief Construct an empty set of template argument lists.
-    MultiLevelTemplateArgumentList() { }
-    
-    /// \brief Construct a single-level template argument list.
-    explicit 
-    MultiLevelTemplateArgumentList(const TemplateArgumentList &TemplateArgs) {
-      addOuterTemplateArguments(&TemplateArgs);
-    }
-    
-    /// \brief Determine the number of levels in this template argument
-    /// list.
-    unsigned getNumLevels() const { return TemplateArgumentLists.size(); }
-    
-    /// \brief Retrieve the template argument at a given depth and index.
-    const TemplateArgument &operator()(unsigned Depth, unsigned Index) const {
-      assert(Depth < TemplateArgumentLists.size());
-      assert(Index < TemplateArgumentLists[getNumLevels() - Depth - 1].second);
-      return TemplateArgumentLists[getNumLevels() - Depth - 1].first[Index];
-    }
-    
-    /// \brief Determine whether there is a non-NULL template argument at the
-    /// given depth and index.
-    ///
-    /// There must exist a template argument list at the given depth.
-    bool hasTemplateArgument(unsigned Depth, unsigned Index) const {
-      assert(Depth < TemplateArgumentLists.size());
-      
-      if (Index >= TemplateArgumentLists[getNumLevels() - Depth - 1].second)
-        return false;
-      
-      return !(*this)(Depth, Index).isNull();
-    }
-    
-    /// \brief Add a new outermost level to the multi-level template argument 
-    /// list.
-    void addOuterTemplateArguments(const TemplateArgumentList *TemplateArgs) {
-      TemplateArgumentLists.push_back(
-                                    ArgList(TemplateArgs->getFlatArgumentList(),
-                                            TemplateArgs->flat_size()));
-    }
-    
-    /// \brief Add a new outmost level to the multi-level template argument
-    /// list.
-    void addOuterTemplateArguments(const TemplateArgument *Args, 
-                                   unsigned NumArgs) {
-      TemplateArgumentLists.push_back(ArgList(Args, NumArgs));
-    }
-    
-    /// \brief Retrieve the innermost template argument list.
-    const ArgList &getInnermost() const { 
-      return TemplateArgumentLists.front(); 
-    }
-  };
-  
-  /// \brief The context in which partial ordering of function templates occurs.
-  enum TemplatePartialOrderingContext {
-    /// \brief Partial ordering of function templates for a function call.
-    TPOC_Call,
-    /// \brief Partial ordering of function templates for a call to a 
-    /// conversion function.
-    TPOC_Conversion,
-    /// \brief Partial ordering of function templates in other contexts, e.g.,
-    /// taking the address of a function template or matching a function 
-    /// template specialization to a function template.
-    TPOC_Other
-  };
-
-  /// \brief Captures a template argument whose value has been deduced
-  /// via c++ template argument deduction.
-  class DeducedTemplateArgument : public TemplateArgument {
-    /// \brief For a non-type template argument, whether the value was
-    /// deduced from an array bound.
-    bool DeducedFromArrayBound;
-
-  public:
-    DeducedTemplateArgument()
-      : TemplateArgument(), DeducedFromArrayBound(false) { }
-
-    DeducedTemplateArgument(const TemplateArgument &Arg,
-                            bool DeducedFromArrayBound = false)
-      : TemplateArgument(Arg), DeducedFromArrayBound(DeducedFromArrayBound) { }
-
-    /// \brief Construct an integral non-type template argument that
-    /// has been deduced, possible from an array bound.
-    DeducedTemplateArgument(const llvm::APSInt &Value,
-                            QualType ValueType,
-                            bool DeducedFromArrayBound)
-      : TemplateArgument(Value, ValueType), 
-        DeducedFromArrayBound(DeducedFromArrayBound) { }
-
-    /// \brief For a non-type template argument, determine whether the
-    /// template argument was deduced from an array bound.
-    bool wasDeducedFromArrayBound() const { return DeducedFromArrayBound; }
-
-    /// \brief Specify whether the given non-type template argument
-    /// was deduced from an array bound.
-    void setDeducedFromArrayBound(bool Deduced) {
-      DeducedFromArrayBound = Deduced;
-    }
-  };
-}
-
-#endif // LLVM_CLANG_SEMA_TEMPLATE_H
diff --git a/lib/Sema/SemaTemplateDeduction.cpp b/lib/Sema/SemaTemplateDeduction.cpp
index 403d5543a64f..5c77ed61060b 100644
--- a/lib/Sema/SemaTemplateDeduction.cpp
+++ b/lib/Sema/SemaTemplateDeduction.cpp
@@ -10,16 +10,21 @@
 //
 //===----------------------------------------------------------------------===/
 
-#include "Sema.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Template.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
-#include "clang/Parse/DeclSpec.h"
 #include <algorithm>
 
 namespace clang {
+  using namespace sema;
+
   /// \brief Various flags that control template argument deduction.
   ///
   /// These flags can be bitwise-OR'd together.
@@ -74,7 +79,7 @@ DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         const TemplateArgument &Param,
                         const TemplateArgument &Arg,
-                        Sema::TemplateDeductionInfo &Info,
+                        TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced);
 
 /// \brief If the given expression is of a form that permits the deduction
@@ -97,7 +102,7 @@ DeduceNonTypeTemplateArgument(Sema &S,
                               NonTypeTemplateParmDecl *NTTP,
                               llvm::APSInt Value, QualType ValueType,
                               bool DeducedFromArrayBound,
-                              Sema::TemplateDeductionInfo &Info,
+                              TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
@@ -138,7 +143,7 @@ static Sema::TemplateDeductionResult
 DeduceNonTypeTemplateArgument(Sema &S,
                               NonTypeTemplateParmDecl *NTTP,
                               Expr *Value,
-                              Sema::TemplateDeductionInfo &Info,
+                              TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
@@ -180,7 +185,7 @@ static Sema::TemplateDeductionResult
 DeduceNonTypeTemplateArgument(Sema &S,
                               NonTypeTemplateParmDecl *NTTP,
                               Decl *D,
-                              Sema::TemplateDeductionInfo &Info,
+                              TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
@@ -214,7 +219,7 @@ DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         TemplateName Param,
                         TemplateName Arg,
-                        Sema::TemplateDeductionInfo &Info,
+                        TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   TemplateDecl *ParamDecl = Param.getAsTemplateDecl();
   if (!ParamDecl) {
@@ -278,7 +283,7 @@ DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         const TemplateSpecializationType *Param,
                         QualType Arg,
-                        Sema::TemplateDeductionInfo &Info,
+                        TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(Arg.isCanonical() && "Argument type must be canonical");
 
@@ -345,6 +350,29 @@ DeduceTemplateArguments(Sema &S,
   return Sema::TDK_Success;
 }
 
+/// \brief Determines whether the given type is an opaque type that
+/// might be more qualified when instantiated.
+static bool IsPossiblyOpaquelyQualifiedType(QualType T) {
+  switch (T->getTypeClass()) {
+  case Type::TypeOfExpr:
+  case Type::TypeOf:
+  case Type::DependentName:
+  case Type::Decltype:
+  case Type::UnresolvedUsing:
+    return true;
+
+  case Type::ConstantArray:
+  case Type::IncompleteArray:
+  case Type::VariableArray:
+  case Type::DependentSizedArray:
+    return IsPossiblyOpaquelyQualifiedType(
+                                      cast<ArrayType>(T)->getElementType());
+
+  default:
+    return false;
+  }
+}
+
 /// \brief Deduce the template arguments by comparing the parameter type and
 /// the argument type (C++ [temp.deduct.type]).
 ///
@@ -370,7 +398,7 @@ static Sema::TemplateDeductionResult
 DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         QualType ParamIn, QualType ArgIn,
-                        Sema::TemplateDeductionInfo &Info,
+                        TemplateDeductionInfo &Info,
                      llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                         unsigned TDF) {
   // We only want to look at the canonical types, since typedefs and
@@ -428,9 +456,9 @@ DeduceTemplateArguments(Sema &S,
     // type.
     if (Param.isMoreQualifiedThan(Arg) && !(TDF & TDF_IgnoreQualifiers)) {
       Info.Param = cast<TemplateTypeParmDecl>(TemplateParams->getParam(Index));
-      Info.FirstArg = Deduced[Index];
+      Info.FirstArg = TemplateArgument(Param);
       Info.SecondArg = TemplateArgument(Arg);
-      return Sema::TDK_InconsistentQuals;
+      return Sema::TDK_Underqualified;
     }
 
     assert(TemplateTypeParm->getDepth() == 0 && "Can't deduce with depth > 0");
@@ -469,7 +497,7 @@ DeduceTemplateArguments(Sema &S,
     if (TDF & TDF_ParamWithReferenceType) {
       if (Param.isMoreQualifiedThan(Arg))
         return Sema::TDK_NonDeducedMismatch;
-    } else {
+    } else if (!IsPossiblyOpaquelyQualifiedType(Param)) {
       if (Param.getCVRQualifiers() != Arg.getCVRQualifiers())
         return Sema::TDK_NonDeducedMismatch;
     }
@@ -530,10 +558,11 @@ DeduceTemplateArguments(Sema &S,
       if (!IncompleteArrayArg)
         return Sema::TDK_NonDeducedMismatch;
 
+      unsigned SubTDF = TDF & TDF_IgnoreQualifiers;
       return DeduceTemplateArguments(S, TemplateParams,
                      S.Context.getAsIncompleteArrayType(Param)->getElementType(),
                                      IncompleteArrayArg->getElementType(),
-                                     Info, Deduced, 0);
+                                     Info, Deduced, SubTDF);
     }
 
     //     T [integer-constant]
@@ -548,10 +577,11 @@ DeduceTemplateArguments(Sema &S,
       if (ConstantArrayArg->getSize() != ConstantArrayParm->getSize())
         return Sema::TDK_NonDeducedMismatch;
 
+      unsigned SubTDF = TDF & TDF_IgnoreQualifiers;
       return DeduceTemplateArguments(S, TemplateParams,
                                      ConstantArrayParm->getElementType(),
                                      ConstantArrayArg->getElementType(),
-                                     Info, Deduced, 0);
+                                     Info, Deduced, SubTDF);
     }
 
     //     type [i]
@@ -560,6 +590,8 @@ DeduceTemplateArguments(Sema &S,
       if (!ArrayArg)
         return Sema::TDK_NonDeducedMismatch;
 
+      unsigned SubTDF = TDF & TDF_IgnoreQualifiers;
+
       // Check the element type of the arrays
       const DependentSizedArrayType *DependentArrayParm
         = S.Context.getAsDependentSizedArrayType(Param);
@@ -567,7 +599,7 @@ DeduceTemplateArguments(Sema &S,
             = DeduceTemplateArguments(S, TemplateParams,
                                       DependentArrayParm->getElementType(),
                                       ArrayArg->getElementType(),
-                                      Info, Deduced, 0))
+                                      Info, Deduced, SubTDF))
         return Result;
 
       // Determine the array bound is something we can deduce.
@@ -797,7 +829,7 @@ DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         const TemplateArgument &Param,
                         const TemplateArgument &Arg,
-                        Sema::TemplateDeductionInfo &Info,
+                        TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   switch (Param.getKind()) {
   case TemplateArgument::Null:
@@ -847,7 +879,6 @@ DeduceTemplateArguments(Sema &S,
       return Sema::TDK_NonDeducedMismatch;
     }
 
-    assert(false && "Type/value mismatch");
     Info.FirstArg = Param;
     Info.SecondArg = Arg;
     return Sema::TDK_NonDeducedMismatch;
@@ -868,7 +899,6 @@ DeduceTemplateArguments(Sema &S,
         return DeduceNonTypeTemplateArgument(S, NTTP, Arg.getAsDecl(),
                                              Info, Deduced);
       
-      assert(false && "Type/value mismatch");
       Info.FirstArg = Param;
       Info.SecondArg = Arg;
       return Sema::TDK_NonDeducedMismatch;
@@ -890,7 +920,7 @@ DeduceTemplateArguments(Sema &S,
                         TemplateParameterList *TemplateParams,
                         const TemplateArgumentList &ParamList,
                         const TemplateArgumentList &ArgList,
-                        Sema::TemplateDeductionInfo &Info,
+                        TemplateDeductionInfo &Info,
                     llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(ParamList.size() == ArgList.size());
   for (unsigned I = 0, N = ParamList.size(); I != N; ++I) {
@@ -974,7 +1004,7 @@ FinishTemplateArgumentDeduction(Sema &S,
                                 ClassTemplatePartialSpecializationDecl *Partial,
                                 const TemplateArgumentList &TemplateArgs,
                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
-                                Sema::TemplateDeductionInfo &Info) {
+                                TemplateDeductionInfo &Info) {
   // Trap errors.
   Sema::SFINAETrap Trap(S);
   
@@ -1010,7 +1040,7 @@ FinishTemplateArgumentDeduction(Sema &S,
   // to the class template.
   // FIXME: Do we have to correct the types of deduced non-type template 
   // arguments (in particular, integral non-type template arguments?).
-  Sema::LocalInstantiationScope InstScope(S);
+  LocalInstantiationScope InstScope(S);
   ClassTemplateDecl *ClassTemplate = Partial->getSpecializedTemplate();
   const TemplateArgumentLoc *PartialTemplateArgs
     = Partial->getTemplateArgsAsWritten();
@@ -1245,7 +1275,8 @@ Sema::SubstituteExplicitTemplateArguments(
                                       Proto->isVariadic(),
                                       Proto->getTypeQuals(),
                                       Function->getLocation(),
-                                      Function->getDeclName());
+                                      Function->getDeclName(),
+                                      Proto->getExtInfo());
     if (FunctionType->isNull() || Trap.hasErrorOccurred())
       return TDK_SubstitutionFailure;
   }
@@ -1486,14 +1517,22 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
   return TDK_Success;
 }
 
+/// Gets the type of a function for template-argument-deducton
+/// purposes when it's considered as part of an overload set.
 static QualType GetTypeOfFunction(ASTContext &Context,
-                                  bool isAddressOfOperand,
+                                  const OverloadExpr::FindResult &R,
                                   FunctionDecl *Fn) {
-  if (!isAddressOfOperand) return Fn->getType();
   if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Fn))
-    if (Method->isInstance())
+    if (Method->isInstance()) {
+      // An instance method that's referenced in a form that doesn't
+      // look like a member pointer is just invalid.
+      if (!R.HasFormOfMemberPointer) return QualType();
+
       return Context.getMemberPointerType(Fn->getType(),
                Context.getTypeDeclType(Method->getParent()).getTypePtr());
+    }
+
+  if (!R.IsAddressOfOperand) return Fn->getType();
   return Context.getPointerType(Fn->getType());
 }
 
@@ -1503,11 +1542,19 @@ static QualType GetTypeOfFunction(ASTContext &Context,
 /// undeduced context
 static QualType
 ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
-                            Expr *Arg, QualType ParamType) {
-  llvm::PointerIntPair<OverloadExpr*,1> R = OverloadExpr::find(Arg);
+                            Expr *Arg, QualType ParamType,
+                            bool ParamWasReference) {
+  
+  OverloadExpr::FindResult R = OverloadExpr::find(Arg);
 
-  bool isAddressOfOperand = bool(R.getInt());
-  OverloadExpr *Ovl = R.getPointer();
+  OverloadExpr *Ovl = R.Expression;
+
+  // C++0x [temp.deduct.call]p4
+  unsigned TDF = 0;
+  if (ParamWasReference)
+    TDF |= TDF_ParamWithReferenceType;
+  if (R.IsAddressOfOperand)
+    TDF |= TDF_IgnoreQualifiers;
 
   // If there were explicit template arguments, we can only find
   // something via C++ [temp.arg.explicit]p3, i.e. if the arguments
@@ -1516,7 +1563,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
     // But we can still look for an explicit specialization.
     if (FunctionDecl *ExplicitSpec
           = S.ResolveSingleFunctionTemplateSpecialization(Ovl))
-      return GetTypeOfFunction(S.Context, isAddressOfOperand, ExplicitSpec);
+      return GetTypeOfFunction(S.Context, R, ExplicitSpec);
     return QualType();
   }
 
@@ -1541,8 +1588,14 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
       return QualType();
 
     FunctionDecl *Fn = cast<FunctionDecl>(D);
-    QualType ArgType = GetTypeOfFunction(S.Context, isAddressOfOperand, Fn);
+    QualType ArgType = GetTypeOfFunction(S.Context, R, Fn);
+    if (ArgType.isNull()) continue;
 
+    // Function-to-pointer conversion.
+    if (!ParamWasReference && ParamType->isPointerType() && 
+        ArgType->isFunctionType())
+      ArgType = S.Context.getPointerType(ArgType);
+    
     //   - If the argument is an overload set (not containing function
     //     templates), trial argument deduction is attempted using each
     //     of the members of the set. If deduction succeeds for only one
@@ -1557,9 +1610,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
     // So we do not reject deductions which were made elsewhere.
     llvm::SmallVector<DeducedTemplateArgument, 8> 
       Deduced(TemplateParams->size());
-    Sema::TemplateDeductionInfo Info(S.Context, Ovl->getNameLoc());
-    unsigned TDF = 0;
-
+    TemplateDeductionInfo Info(S.Context, Ovl->getNameLoc());
     Sema::TemplateDeductionResult Result
       = DeduceTemplateArguments(S, TemplateParams,
                                 ParamType, ArgType,
@@ -1624,7 +1675,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
 
   // The types of the parameters from which we will perform template argument
   // deduction.
-  Sema::LocalInstantiationScope InstScope(*this);
+  LocalInstantiationScope InstScope(*this);
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
   llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
@@ -1654,20 +1705,40 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
     QualType ParamType = ParamTypes[I];
     QualType ArgType = Args[I]->getType();
 
+    // C++0x [temp.deduct.call]p3:
+    //   If P is a cv-qualified type, the top level cv-qualifiers of P’s type
+    //   are ignored for type deduction.
+    if (ParamType.getCVRQualifiers())
+      ParamType = ParamType.getLocalUnqualifiedType();
+    const ReferenceType *ParamRefType = ParamType->getAs<ReferenceType>();
+    if (ParamRefType) {
+      //   [...] If P is a reference type, the type referred to by P is used
+      //   for type deduction.
+      ParamType = ParamRefType->getPointeeType();
+    }
+    
     // Overload sets usually make this parameter an undeduced
     // context, but there are sometimes special circumstances.
     if (ArgType == Context.OverloadTy) {
       ArgType = ResolveOverloadForDeduction(*this, TemplateParams,
-                                            Args[I], ParamType);
+                                            Args[I], ParamType,
+                                            ParamRefType != 0);
       if (ArgType.isNull())
         continue;
     }
 
-    // C++ [temp.deduct.call]p2:
-    //   If P is not a reference type:
-    QualType CanonParamType = Context.getCanonicalType(ParamType);
-    bool ParamWasReference = isa<ReferenceType>(CanonParamType);
-    if (!ParamWasReference) {
+    if (ParamRefType) {
+      // C++0x [temp.deduct.call]p3:
+      //   [...] If P is of the form T&&, where T is a template parameter, and
+      //   the argument is an lvalue, the type A& is used in place of A for
+      //   type deduction.
+      if (ParamRefType->isRValueReferenceType() &&
+          ParamRefType->getAs<TemplateTypeParmType>() &&
+          Args[I]->isLvalue(Context) == Expr::LV_Valid)
+        ArgType = Context.getLValueReferenceType(ArgType);
+    } else {
+      // C++ [temp.deduct.call]p2:
+      //   If P is not a reference type:
       //   - If A is an array type, the pointer type produced by the
       //     array-to-pointer standard conversion (4.2) is used in place of
       //     A for type deduction; otherwise,
@@ -1682,30 +1753,11 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
         // - If A is a cv-qualified type, the top level cv-qualifiers of A’s
         //   type are ignored for type deduction.
         QualType CanonArgType = Context.getCanonicalType(ArgType);
-        if (CanonArgType.getLocalCVRQualifiers())
-          ArgType = CanonArgType.getLocalUnqualifiedType();
+        if (ArgType.getCVRQualifiers())
+          ArgType = ArgType.getUnqualifiedType();
       }
     }
 
-    // C++0x [temp.deduct.call]p3:
-    //   If P is a cv-qualified type, the top level cv-qualifiers of P’s type
-    //   are ignored for type deduction.
-    if (CanonParamType.getLocalCVRQualifiers())
-      ParamType = CanonParamType.getLocalUnqualifiedType();
-    if (const ReferenceType *ParamRefType = ParamType->getAs<ReferenceType>()) {
-      //   [...] If P is a reference type, the type referred to by P is used
-      //   for type deduction.
-      ParamType = ParamRefType->getPointeeType();
-
-      //   [...] If P is of the form T&&, where T is a template parameter, and
-      //   the argument is an lvalue, the type A& is used in place of A for
-      //   type deduction.
-      if (isa<RValueReferenceType>(ParamRefType) &&
-          ParamRefType->getAs<TemplateTypeParmType>() &&
-          Args[I]->isLvalue(Context) == Expr::LV_Valid)
-        ArgType = Context.getLValueReferenceType(ArgType);
-    }
-
     // C++0x [temp.deduct.call]p4:
     //   In general, the deduction process attempts to find template argument
     //   values that will make the deduced A identical to A (after the type A
@@ -1715,12 +1767,13 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
     //     - If the original P is a reference type, the deduced A (i.e., the
     //       type referred to by the reference) can be more cv-qualified than
     //       the transformed A.
-    if (ParamWasReference)
+    if (ParamRefType)
       TDF |= TDF_ParamWithReferenceType;
     //     - The transformed A can be another pointer or pointer to member
     //       type that can be converted to the deduced A via a qualification
     //       conversion (4.4).
-    if (ArgType->isPointerType() || ArgType->isMemberPointerType())
+    if (ArgType->isPointerType() || ArgType->isMemberPointerType() ||
+        ArgType->isObjCObjectPointerType())
       TDF |= TDF_IgnoreQualifiers;
     //     - If P is a class and P has the form simple-template-id, then the
     //       transformed A can be a derived class of the deduced A. Likewise,
@@ -1783,7 +1836,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   QualType FunctionType = Function->getType();
 
   // Substitute any explicit template arguments.
-  Sema::LocalInstantiationScope InstScope(*this);
+  LocalInstantiationScope InstScope(*this);
   llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
   unsigned NumExplicitlySpecified = 0;
   llvm::SmallVector<QualType, 4> ParamTypes;
@@ -1915,7 +1968,7 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
   // modulo the various allowed differences.
 
   // Finish template argument deduction.
-  Sema::LocalInstantiationScope InstScope(*this);
+  LocalInstantiationScope InstScope(*this);
   FunctionDecl *Spec = 0;
   TemplateDeductionResult Result
     = FinishTemplateArgumentDeduction(FunctionTemplate, Deduced, 0, Spec, 
@@ -1979,7 +2032,7 @@ static Sema::TemplateDeductionResult
 DeduceTemplateArgumentsDuringPartialOrdering(Sema &S,
                                         TemplateParameterList *TemplateParams,
                                              QualType ParamIn, QualType ArgIn,
-                                             Sema::TemplateDeductionInfo &Info,
+                                             TemplateDeductionInfo &Info,
                       llvm::SmallVectorImpl<DeducedTemplateArgument> &Deduced,
    llvm::SmallVectorImpl<DeductionQualifierComparison> *QualifierComparisons) {
   CanQualType Param = S.Context.getCanonicalType(ParamIn);
@@ -2061,7 +2114,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
   // C++0x [temp.deduct.partial]p3:
   //   The types used to determine the ordering depend on the context in which
   //   the partial ordering is done:
-  Sema::TemplateDeductionInfo Info(S.Context, Loc);
+  TemplateDeductionInfo Info(S.Context, Loc);
   switch (TPOC) {
   case TPOC_Call: {
     //   - In the context of a function call, the function parameter types are
@@ -2386,7 +2439,7 @@ Sema::getMoreSpecializedPartialSpecialization(
   // template partial specialization's template arguments, for
   // example.
   llvm::SmallVector<DeducedTemplateArgument, 4> Deduced;
-  Sema::TemplateDeductionInfo Info(Context, Loc);
+  TemplateDeductionInfo Info(Context, Loc);
 
   QualType PT1 = PS1->getInjectedSpecializationType();
   QualType PT2 = PS2->getInjectedSpecializationType();
diff --git a/lib/Sema/SemaTemplateInstantiate.cpp b/lib/Sema/SemaTemplateInstantiate.cpp
index 0cdc8a12ab73..4d4c18130b01 100644
--- a/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/lib/Sema/SemaTemplateInstantiate.cpp
@@ -10,17 +10,20 @@
 //
 //===----------------------------------------------------------------------===/
 
-#include "Sema.h"
+#include "clang/Sema/SemaInternal.h"
 #include "TreeTransform.h"
-#include "Lookup.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Template.h"
+#include "clang/Sema/TemplateDeduction.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/DeclTemplate.h"
-#include "clang/Parse/DeclSpec.h"
 #include "clang/Basic/LangOptions.h"
 
 using namespace clang;
+using namespace sema;
 
 //===----------------------------------------------------------------------===/
 // Template Instantiation Support
@@ -614,10 +617,10 @@ namespace {
     QualType RebuildElaboratedType(ElaboratedTypeKeyword Keyword,
                                    NestedNameSpecifier *NNS, QualType T);
 
-    Sema::OwningExprResult TransformPredefinedExpr(PredefinedExpr *E);
-    Sema::OwningExprResult TransformDeclRefExpr(DeclRefExpr *E);
-    Sema::OwningExprResult TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E);
-    Sema::OwningExprResult TransformTemplateParmRefExpr(DeclRefExpr *E,
+    ExprResult TransformPredefinedExpr(PredefinedExpr *E);
+    ExprResult TransformDeclRefExpr(DeclRefExpr *E);
+    ExprResult TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E);
+    ExprResult TransformTemplateParmRefExpr(DeclRefExpr *E,
                                                 NonTypeTemplateParmDecl *D);
 
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
@@ -631,9 +634,9 @@ namespace {
                                            TemplateTypeParmTypeLoc TL,
                                            QualType ObjectType);
 
-    Sema::OwningExprResult TransformCallExpr(CallExpr *CE) {
+    ExprResult TransformCallExpr(CallExpr *CE) {
       getSema().CallsUndergoingInstantiation.push_back(CE);
-      OwningExprResult Result =
+      ExprResult Result =
           TreeTransform<TemplateInstantiator>::TransformCallExpr(CE);
       getSema().CallsUndergoingInstantiation.pop_back();
       return move(Result);
@@ -768,7 +771,7 @@ TemplateInstantiator::RebuildElaboratedType(ElaboratedTypeKeyword Keyword,
                                                                     NNS, T);
 }
 
-Sema::OwningExprResult 
+ExprResult 
 TemplateInstantiator::TransformPredefinedExpr(PredefinedExpr *E) {
   if (!E->isTypeDependent())
     return SemaRef.Owned(E->Retain());
@@ -790,7 +793,7 @@ TemplateInstantiator::TransformPredefinedExpr(PredefinedExpr *E) {
   return getSema().Owned(PE);
 }
 
-Sema::OwningExprResult
+ExprResult
 TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
                                                NonTypeTemplateParmDecl *NTTP) {
   // If the corresponding template argument is NULL or non-existent, it's
@@ -818,7 +821,7 @@ TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
                             getSema().FindInstantiatedDecl(E->getLocation(),
                                                            VD, TemplateArgs));
     if (!VD)
-      return SemaRef.ExprError();
+      return ExprError();
 
     // Derive the type we want the substituted decl to have.  This had
     // better be non-dependent, or these checks will have serious problems.
@@ -837,7 +840,7 @@ TemplateInstantiator::TransformTemplateParmRefExpr(DeclRefExpr *E,
 }
                                                    
 
-Sema::OwningExprResult
+ExprResult
 TemplateInstantiator::TransformDeclRefExpr(DeclRefExpr *E) {
   NamedDecl *D = E->getDecl();
   if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(D)) {
@@ -851,7 +854,7 @@ TemplateInstantiator::TransformDeclRefExpr(DeclRefExpr *E) {
   return TreeTransform<TemplateInstantiator>::TransformDeclRefExpr(E);
 }
 
-Sema::OwningExprResult TemplateInstantiator::TransformCXXDefaultArgExpr(
+ExprResult TemplateInstantiator::TransformCXXDefaultArgExpr(
     CXXDefaultArgExpr *E) {
   assert(!cast<FunctionDecl>(E->getParam()->getDeclContext())->
              getDescribedFunctionTemplate() &&
@@ -865,7 +868,7 @@ QualType TemplateInstantiator::TransformFunctionProtoType(TypeLocBuilder &TLB,
                                                         FunctionProtoTypeLoc TL,
                                                           QualType ObjectType) {
   // We need a local instantiation scope for this function prototype.
-  Sema::LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true);
+  LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true);
   return inherited::TransformFunctionProtoType(TLB, TL, ObjectType);
 }
 
@@ -1067,7 +1070,8 @@ ParmVarDecl *Sema::SubstParmVarDecl(ParmVarDecl *OldParm,
   NewParm->setHasInheritedDefaultArg(OldParm->hasInheritedDefaultArg());
 
   CurrentInstantiationScope->InstantiatedLocal(OldParm, NewParm);
-  // Set DeclContext if inside a Block.
+  // FIXME: OldParm may come from a FunctionProtoType, in which case CurContext
+  // can be anything, is this right ?
   NewParm->setDeclContext(CurContext);
   
   return NewParm;  
@@ -1100,11 +1104,11 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
       continue;
     }
 
-    QualType BaseType = SubstType(Base->getType(),
-                                  TemplateArgs,
-                                  Base->getSourceRange().getBegin(),
-                                  DeclarationName());
-    if (BaseType.isNull()) {
+    TypeSourceInfo *BaseTypeLoc = SubstType(Base->getTypeSourceInfo(),
+                                            TemplateArgs,
+                                            Base->getSourceRange().getBegin(),
+                                            DeclarationName());
+    if (!BaseTypeLoc) {
       Invalid = true;
       continue;
     }
@@ -1114,9 +1118,7 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
                                Base->getSourceRange(),
                                Base->isVirtual(),
                                Base->getAccessSpecifierAsWritten(),
-                               BaseType,
-                               /*FIXME: Not totally accurate */
-                               Base->getSourceRange().getBegin()))
+                               BaseTypeLoc))
       InstantiatedBases.push_back(InstantiatedBase);
     else
       Invalid = true;
@@ -1199,13 +1201,16 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   // PushDeclContext because we don't have a scope.
   ContextRAII SavedContext(*this, Instantiation);
   EnterExpressionEvaluationContext EvalContext(*this, 
-                                               Action::PotentiallyEvaluated);
+                                               Sema::PotentiallyEvaluated);
 
   // If this is an instantiation of a local class, merge this local
   // instantiation scope with the enclosing scope. Otherwise, every
   // instantiation of a class has its own local instantiation scope.
   bool MergeWithParentScope = !Instantiation->isDefinedOutsideFunctionOrMethod();
-  Sema::LocalInstantiationScope Scope(*this, MergeWithParentScope);
+  LocalInstantiationScope Scope(*this, MergeWithParentScope);
+
+  // Pull attributes from the pattern onto the instantiation.
+  InstantiateAttrs(TemplateArgs, Pattern, Instantiation);
 
   // Start the definition of this instantiation.
   Instantiation->startDefinition();
@@ -1216,14 +1221,14 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   if (SubstBaseSpecifiers(Instantiation, Pattern, TemplateArgs))
     Invalid = true;
 
-  llvm::SmallVector<DeclPtrTy, 4> Fields;
+  llvm::SmallVector<Decl*, 4> Fields;
   for (RecordDecl::decl_iterator Member = Pattern->decls_begin(),
          MemberEnd = Pattern->decls_end();
        Member != MemberEnd; ++Member) {
     Decl *NewMember = SubstDecl(*Member, Instantiation, TemplateArgs);
     if (NewMember) {
       if (FieldDecl *Field = dyn_cast<FieldDecl>(NewMember))
-        Fields.push_back(DeclPtrTy::make(Field));
+        Fields.push_back(Field);
       else if (NewMember->isInvalidDecl())
         Invalid = true;
     } else {
@@ -1234,7 +1239,7 @@ Sema::InstantiateClass(SourceLocation PointOfInstantiation,
   }
 
   // Finish checking fields.
-  ActOnFields(0, Instantiation->getLocation(), DeclPtrTy::make(Instantiation),
+  ActOnFields(0, Instantiation->getLocation(), Instantiation,
               Fields.data(), Fields.size(), SourceLocation(), SourceLocation(),
               0);
   CheckCompletedCXXClass(Instantiation);
@@ -1416,12 +1421,6 @@ Sema::InstantiateClassTemplateSpecialization(
                                  TSK,
                                  Complain);
 
-  for (unsigned I = 0, N = Matched.size(); I != N; ++I) {
-    // FIXME: Implement TemplateArgumentList::Destroy!
-    //    if (Matched[I].first != Pattern)
-    //      Matched[I].second->Destroy(Context);
-  }
-
   return Result;
 }
 
@@ -1583,7 +1582,7 @@ Sema::InstantiateClassTemplateSpecializationMembers(
                           TSK);
 }
 
-Sema::OwningStmtResult
+StmtResult
 Sema::SubstStmt(Stmt *S, const MultiLevelTemplateArgumentList &TemplateArgs) {
   if (!S)
     return Owned(S);
@@ -1594,7 +1593,7 @@ Sema::SubstStmt(Stmt *S, const MultiLevelTemplateArgumentList &TemplateArgs) {
   return Instantiator.TransformStmt(S);
 }
 
-Sema::OwningExprResult
+ExprResult
 Sema::SubstExpr(Expr *E, const MultiLevelTemplateArgumentList &TemplateArgs) {
   if (!E)
     return Owned(E);
@@ -1615,6 +1614,15 @@ Sema::SubstNestedNameSpecifier(NestedNameSpecifier *NNS,
   return Instantiator.TransformNestedNameSpecifier(NNS, Range);
 }
 
+/// \brief Do template substitution on declaration name info.
+DeclarationNameInfo
+Sema::SubstDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
+                         const MultiLevelTemplateArgumentList &TemplateArgs) {
+  TemplateInstantiator Instantiator(*this, TemplateArgs, NameInfo.getLoc(),
+                                    NameInfo.getName());
+  return Instantiator.TransformDeclarationNameInfo(NameInfo);
+}
+
 TemplateName
 Sema::SubstTemplateName(TemplateName Name, SourceLocation Loc,
                         const MultiLevelTemplateArgumentList &TemplateArgs) {
@@ -1631,7 +1639,7 @@ bool Sema::Subst(const TemplateArgumentLoc &Input, TemplateArgumentLoc &Output,
   return Instantiator.TransformTemplateArgument(Input, Output);
 }
 
-Decl *Sema::LocalInstantiationScope::getInstantiationOf(const Decl *D) {
+Decl *LocalInstantiationScope::getInstantiationOf(const Decl *D) {
   for (LocalInstantiationScope *Current = this; Current; 
        Current = Current->Outer) {
     // Check if we found something within this scope.
@@ -1650,8 +1658,7 @@ Decl *Sema::LocalInstantiationScope::getInstantiationOf(const Decl *D) {
   return 0;
 }
 
-void Sema::LocalInstantiationScope::InstantiatedLocal(const Decl *D, 
-                                                      Decl *Inst) {
+void LocalInstantiationScope::InstantiatedLocal(const Decl *D, Decl *Inst) {
   Decl *&Stored = LocalDecls[D];
   assert((!Stored || Stored == Inst)&& "Already instantiated this local");
   Stored = Inst;
diff --git a/lib/Sema/SemaTemplateInstantiateDecl.cpp b/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 2fd35285324e..1c7869fecd86 100644
--- a/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -9,8 +9,10 @@
 //  This file implements C++ template instantiation for declarations.
 //
 //===----------------------------------------------------------------------===/
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/PrettyDeclStackTrace.h"
+#include "clang/Sema/Template.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclTemplate.h"
@@ -19,7 +21,6 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/TypeLoc.h"
-#include "clang/Basic/PrettyStackTrace.h"
 #include "clang/Lex/Preprocessor.h"
 
 using namespace clang;
@@ -31,11 +32,7 @@ namespace {
     DeclContext *Owner;
     const MultiLevelTemplateArgumentList &TemplateArgs;
 
-    void InstantiateAttrs(Decl *Tmpl, Decl *New);
-      
   public:
-    typedef Sema::OwningExprResult OwningExprResult;
-
     TemplateDeclInstantiator(Sema &SemaRef, DeclContext *Owner,
                              const MultiLevelTemplateArgumentList &TemplateArgs)
       : SemaRef(SemaRef), Owner(Owner), TemplateArgs(TemplateArgs) { }
@@ -87,10 +84,6 @@ namespace {
       return 0;
     }
 
-    const LangOptions &getLangOptions() {
-      return SemaRef.getLangOptions();
-    }
-
     // Helper functions for instantiating methods.
     TypeSourceInfo *SubstFunctionType(FunctionDecl *D,
                              llvm::SmallVectorImpl<ParmVarDecl *> &Params);
@@ -144,28 +137,38 @@ bool TemplateDeclInstantiator::SubstQualifier(const TagDecl *OldDecl,
 }
 
 // FIXME: Is this still too simple?
-void TemplateDeclInstantiator::InstantiateAttrs(Decl *Tmpl, Decl *New) {
-  for (const Attr *TmplAttr = Tmpl->getAttrs(); TmplAttr;
-       TmplAttr = TmplAttr->getNext()) {
+void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
+                            Decl *Tmpl, Decl *New) {
+  for (AttrVec::const_iterator i = Tmpl->attr_begin(), e = Tmpl->attr_end();
+       i != e; ++i) {
+    const Attr *TmplAttr = *i;
     // FIXME: This should be generalized to more than just the AlignedAttr.
     if (const AlignedAttr *Aligned = dyn_cast<AlignedAttr>(TmplAttr)) {
-      if (Aligned->isDependent()) {
+      if (Aligned->isAlignmentDependent()) {
         // The alignment expression is not potentially evaluated.
-        EnterExpressionEvaluationContext Unevaluated(SemaRef,
-                                                     Action::Unevaluated);
-
-        OwningExprResult Result = SemaRef.SubstExpr(Aligned->getAlignmentExpr(),
-                                                    TemplateArgs);
-        if (!Result.isInvalid())
-          // FIXME: Is this the correct source location?
-          SemaRef.AddAlignedAttr(Aligned->getAlignmentExpr()->getExprLoc(),
-                                 New, Result.takeAs<Expr>());
+        EnterExpressionEvaluationContext Unevaluated(*this,
+                                                     Sema::Unevaluated);
+
+        if (Aligned->isAlignmentExpr()) {
+          ExprResult Result = SubstExpr(Aligned->getAlignmentExpr(),
+                                              TemplateArgs);
+          if (!Result.isInvalid())
+            AddAlignedAttr(Aligned->getLocation(), New, Result.takeAs<Expr>());
+        }
+        else {
+          TypeSourceInfo *Result = SubstType(Aligned->getAlignmentType(),
+                                              TemplateArgs,
+                                              Aligned->getLocation(), 
+                                              DeclarationName());
+          if (Result)
+            AddAlignedAttr(Aligned->getLocation(), New, Result);
+        }
         continue;
       }
     }
 
     // FIXME: Is cloning correct for all attributes?
-    Attr *NewAttr = TmplAttr->clone(SemaRef.Context);
+    Attr *NewAttr = TmplAttr->clone(Context);
     New->addAttr(NewAttr);
   }
 }
@@ -234,7 +237,7 @@ Decl *TemplateDeclInstantiator::VisitTypedefDecl(TypedefDecl *D) {
     Typedef->setPreviousDeclaration(cast<TypedefDecl>(InstPrev));
   }
 
-  InstantiateAttrs(D, Typedef);
+  SemaRef.InstantiateAttrs(TemplateArgs, D, Typedef);
 
   Typedef->setAccess(D->getAccess());
   Owner->addDecl(Typedef);
@@ -249,14 +252,14 @@ static bool InstantiateInitializationArguments(Sema &SemaRef,
                                                Expr **Args, unsigned NumArgs,
                            const MultiLevelTemplateArgumentList &TemplateArgs,
                          llvm::SmallVectorImpl<SourceLocation> &FakeCommaLocs,
-                           ASTOwningVector<&ActionBase::DeleteExpr> &InitArgs) {
+                           ASTOwningVector<Expr*> &InitArgs) {
   for (unsigned I = 0; I != NumArgs; ++I) {
     // When we hit the first defaulted argument, break out of the loop:
     // we don't pass those default arguments on.
     if (Args[I]->isDefaultArgument())
       break;
   
-    Sema::OwningExprResult Arg = SemaRef.SubstExpr(Args[I], TemplateArgs);
+    ExprResult Arg = SemaRef.SubstExpr(Args[I], TemplateArgs);
     if (Arg.isInvalid())
       return true;
   
@@ -288,7 +291,7 @@ static bool InstantiateInitializer(Sema &S, Expr *Init,
                             const MultiLevelTemplateArgumentList &TemplateArgs,
                                    SourceLocation &LParenLoc,
                                llvm::SmallVector<SourceLocation, 4> &CommaLocs,
-                             ASTOwningVector<&ActionBase::DeleteExpr> &NewArgs,
+                             ASTOwningVector<Expr*> &NewArgs,
                                    SourceLocation &RParenLoc) {
   NewArgs.clear();
   LParenLoc = SourceLocation();
@@ -331,7 +334,7 @@ static bool InstantiateInitializer(Sema &S, Expr *Init,
     }
   }
  
-  Sema::OwningExprResult Result = S.SubstExpr(Init, TemplateArgs);
+  ExprResult Result = S.SubstExpr(Init, TemplateArgs);
   if (Result.isInvalid())
     return true;
 
@@ -363,7 +366,6 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
                                  D->getStorageClassAsWritten());
   Var->setThreadSpecified(D->isThreadSpecified());
   Var->setCXXDirectInitializer(D->hasCXXDirectInitializer());
-  Var->setDeclaredInCondition(D->isDeclaredInCondition());
 
   // Substitute the nested name specifier, if any.
   if (SubstQualifier(D, Var))
@@ -399,7 +401,7 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
     if (Owner->isFunctionOrMethod())
       SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Var);
   }
-  InstantiateAttrs(D, Var);
+  SemaRef.InstantiateAttrs(TemplateArgs, D, Var);
   
   // Link instantiations of static data members back to the template from
   // which they were instantiated.
@@ -418,25 +420,23 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
     // Instantiate the initializer.
     SourceLocation LParenLoc, RParenLoc;
     llvm::SmallVector<SourceLocation, 4> CommaLocs;
-    ASTOwningVector<&ActionBase::DeleteExpr> InitArgs(SemaRef);
+    ASTOwningVector<Expr*> InitArgs(SemaRef);
     if (!InstantiateInitializer(SemaRef, D->getInit(), TemplateArgs, LParenLoc,
                                 CommaLocs, InitArgs, RParenLoc)) {
       // Attach the initializer to the declaration.
       if (D->hasCXXDirectInitializer()) {
         // Add the direct initializer to the declaration.
-        SemaRef.AddCXXDirectInitializerToDecl(Sema::DeclPtrTy::make(Var),
+        SemaRef.AddCXXDirectInitializerToDecl(Var,
                                               LParenLoc,
                                               move_arg(InitArgs),
                                               CommaLocs.data(),
                                               RParenLoc);
       } else if (InitArgs.size() == 1) {
-        Expr *Init = (Expr*)(InitArgs.take()[0]);
-        SemaRef.AddInitializerToDecl(Sema::DeclPtrTy::make(Var), 
-                                     SemaRef.Owned(Init),
-                                     false);        
+        Expr *Init = InitArgs.take()[0];
+        SemaRef.AddInitializerToDecl(Var, Init, false);
       } else {
         assert(InitArgs.size() == 0);
-        SemaRef.ActOnUninitializedDecl(Sema::DeclPtrTy::make(Var), false);    
+        SemaRef.ActOnUninitializedDecl(Var, false);    
       }
     } else {
       // FIXME: Not too happy about invalidating the declaration
@@ -446,12 +446,12 @@ Decl *TemplateDeclInstantiator::VisitVarDecl(VarDecl *D) {
     
     SemaRef.PopExpressionEvaluationContext();
   } else if (!Var->isStaticDataMember() || Var->isOutOfLine())
-    SemaRef.ActOnUninitializedDecl(Sema::DeclPtrTy::make(Var), false);
+    SemaRef.ActOnUninitializedDecl(Var, false);
 
   // Diagnose unused local variables.
   if (!Var->isInvalidDecl() && Owner->isFunctionOrMethod() && !Var->isUsed())
     SemaRef.DiagnoseUnusedDecl(Var);
-  
+
   return Var;
 }
 
@@ -493,9 +493,9 @@ Decl *TemplateDeclInstantiator::VisitFieldDecl(FieldDecl *D) {
     BitWidth = 0;
   else if (BitWidth) {
     // The bit-width expression is not potentially evaluated.
-    EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+    EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-    OwningExprResult InstantiatedBitWidth
+    ExprResult InstantiatedBitWidth
       = SemaRef.SubstExpr(BitWidth, TemplateArgs);
     if (InstantiatedBitWidth.isInvalid()) {
       Invalid = true;
@@ -518,7 +518,7 @@ Decl *TemplateDeclInstantiator::VisitFieldDecl(FieldDecl *D) {
     return 0;
   }
 
-  InstantiateAttrs(D, Field);
+  SemaRef.InstantiateAttrs(TemplateArgs, D, Field);
   
   if (Invalid)
     Field->setInvalidDecl();
@@ -529,7 +529,7 @@ Decl *TemplateDeclInstantiator::VisitFieldDecl(FieldDecl *D) {
   } 
   if (CXXRecordDecl *Parent= dyn_cast<CXXRecordDecl>(Field->getDeclContext())) {
     if (Parent->isAnonymousStructOrUnion() &&
-        Parent->getLookupContext()->isFunctionOrMethod())
+        Parent->getRedeclContext()->isFunctionOrMethod())
       SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Field);
   }
 
@@ -581,20 +581,18 @@ Decl *TemplateDeclInstantiator::VisitStaticAssertDecl(StaticAssertDecl *D) {
   Expr *AssertExpr = D->getAssertExpr();
 
   // The expression in a static assertion is not potentially evaluated.
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  OwningExprResult InstantiatedAssertExpr
+  ExprResult InstantiatedAssertExpr
     = SemaRef.SubstExpr(AssertExpr, TemplateArgs);
   if (InstantiatedAssertExpr.isInvalid())
     return 0;
 
-  OwningExprResult Message(SemaRef, D->getMessage());
+  ExprResult Message(D->getMessage());
   D->getMessage()->Retain();
-  Decl *StaticAssert
-    = SemaRef.ActOnStaticAssertDeclaration(D->getLocation(),
-                                           move(InstantiatedAssertExpr),
-                                           move(Message)).getAs<Decl>();
-  return StaticAssert;
+  return SemaRef.ActOnStaticAssertDeclaration(D->getLocation(),
+                                              InstantiatedAssertExpr.get(),
+                                              Message.get());
 }
 
 Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
@@ -611,18 +609,18 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
   if (D->getDeclContext()->isFunctionOrMethod())
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Enum);
     
-  llvm::SmallVector<Sema::DeclPtrTy, 4> Enumerators;
+  llvm::SmallVector<Decl*, 4> Enumerators;
 
   EnumConstantDecl *LastEnumConst = 0;
   for (EnumDecl::enumerator_iterator EC = D->enumerator_begin(),
          ECEnd = D->enumerator_end();
        EC != ECEnd; ++EC) {
     // The specified value for the enumerator.
-    OwningExprResult Value = SemaRef.Owned((Expr *)0);
+    ExprResult Value = SemaRef.Owned((Expr *)0);
     if (Expr *UninstValue = EC->getInitExpr()) {
       // The enumerator's value expression is not potentially evaluated.
       EnterExpressionEvaluationContext Unevaluated(SemaRef,
-                                                   Action::Unevaluated);
+                                                   Sema::Unevaluated);
 
       Value = SemaRef.SubstExpr(UninstValue, TemplateArgs);
     }
@@ -637,7 +635,7 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
     EnumConstantDecl *EnumConst
       = SemaRef.CheckEnumConstant(Enum, LastEnumConst,
                                   EC->getLocation(), EC->getIdentifier(),
-                                  move(Value));
+                                  Value.get());
 
     if (isInvalid) {
       if (EnumConst)
@@ -648,7 +646,7 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
     if (EnumConst) {
       EnumConst->setAccess(Enum->getAccess());
       Enum->addDecl(EnumConst);
-      Enumerators.push_back(Sema::DeclPtrTy::make(EnumConst));
+      Enumerators.push_back(EnumConst);
       LastEnumConst = EnumConst;
       
       if (D->getDeclContext()->isFunctionOrMethod()) {
@@ -662,8 +660,8 @@ Decl *TemplateDeclInstantiator::VisitEnumDecl(EnumDecl *D) {
   // FIXME: Fixup LBraceLoc and RBraceLoc
   // FIXME: Empty Scope and AttributeList (required to handle attribute packed).
   SemaRef.ActOnEnumBody(Enum->getLocation(), SourceLocation(), SourceLocation(),
-                        Sema::DeclPtrTy::make(Enum),
-                        &Enumerators[0], Enumerators.size(),
+                        Enum,
+                        Enumerators.data(), Enumerators.size(),
                         0, 0);
 
   return Enum;
@@ -679,7 +677,7 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
   // Create a local instantiation scope for this class template, which
   // will contain the instantiations of the template parameters.
-  Sema::LocalInstantiationScope Scope(SemaRef);
+  LocalInstantiationScope Scope(SemaRef);
   TemplateParameterList *TempParams = D->getTemplateParameters();
   TemplateParameterList *InstParams = SubstTemplateParams(TempParams);
   if (!InstParams)
@@ -857,16 +855,7 @@ TemplateDeclInstantiator::VisitClassTemplatePartialSpecializationDecl(
   if (!InstClassTemplate)
     return 0;
   
-  Decl *DCanon = D->getCanonicalDecl();
-  for (llvm::FoldingSet<ClassTemplatePartialSpecializationDecl>::iterator
-            P = InstClassTemplate->getPartialSpecializations().begin(),
-         PEnd = InstClassTemplate->getPartialSpecializations().end();
-       P != PEnd; ++P) {
-    if (P->getInstantiatedFromMember()->getCanonicalDecl() == DCanon)
-      return &*P;
-  }
-  
-  return 0;
+  return InstClassTemplate->findPartialSpecInstantiatedFromMember(D);
 }
 
 Decl *
@@ -875,7 +864,7 @@ TemplateDeclInstantiator::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   // will contain the instantiations of the template parameters and then get
   // merged with the local instantiation scope for the function template 
   // itself.
-  Sema::LocalInstantiationScope Scope(SemaRef);
+  LocalInstantiationScope Scope(SemaRef);
 
   TemplateParameterList *TempParams = D->getTemplateParameters();
   TemplateParameterList *InstParams = SubstTemplateParams(TempParams);
@@ -957,7 +946,7 @@ Decl *TemplateDeclInstantiator::VisitCXXRecordDecl(CXXRecordDecl *D) {
   // Make sure that anonymous structs and unions are recorded.
   if (D->isAnonymousStructOrUnion()) {
     Record->setAnonymousStructOrUnion(true);
-    if (Record->getDeclContext()->getLookupContext()->isFunctionOrMethod())
+    if (Record->getDeclContext()->getRedeclContext()->isFunctionOrMethod())
       SemaRef.CurrentInstantiationScope->InstantiatedLocal(D, Record);
   }
 
@@ -977,20 +966,16 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
   FunctionTemplateDecl *FunctionTemplate = D->getDescribedFunctionTemplate();
   void *InsertPos = 0;
   if (FunctionTemplate && !TemplateParams) {
-    llvm::FoldingSetNodeID ID;
     std::pair<const TemplateArgument *, unsigned> Innermost 
       = TemplateArgs.getInnermost();
-    FunctionTemplateSpecializationInfo::Profile(ID, Innermost.first,
-                                                Innermost.second,
-                                                SemaRef.Context);
 
-    FunctionTemplateSpecializationInfo *Info
-      = FunctionTemplate->getSpecializations().FindNodeOrInsertPos(ID,
-                                                                   InsertPos);
+    FunctionDecl *SpecFunc
+      = FunctionTemplate->findSpecialization(Innermost.first, Innermost.second,
+                                             InsertPos);
 
     // If we already have a function template specialization, return it.
-    if (Info)
-      return Info->Function;
+    if (SpecFunc)
+      return SpecFunc;
   }
 
   bool isFriend;
@@ -1003,7 +988,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
     Owner->isFunctionOrMethod() ||
     !(isa<Decl>(Owner) && 
       cast<Decl>(Owner)->isDefinedOutsideFunctionOrMethod());
-  Sema::LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
+  LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
 
   llvm::SmallVector<ParmVarDecl *, 4> Params;
   TypeSourceInfo *TInfo = D->getTypeSourceInfo();
@@ -1181,7 +1166,9 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
 
     PrincipalDecl->setObjectOfFriendDecl(PrevDecl != 0);
     DC->makeDeclVisibleInContext(PrincipalDecl, /*Recoverable=*/ false);
-    
+
+    bool queuedInstantiation = false;
+
     if (!SemaRef.getLangOptions().CPlusPlus0x &&
         D->isThisDeclarationADefinition()) {
       // Check for a function body.
@@ -1198,21 +1185,36 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
       else for (FunctionDecl::redecl_iterator R = Function->redecls_begin(),
                                            REnd = Function->redecls_end();
                 R != REnd; ++R) {
-        if (*R != Function && 
-            ((*R)->getFriendObjectKind() != Decl::FOK_None)) {
+        if (*R == Function)
+          continue;
+        switch (R->getFriendObjectKind()) {
+        case Decl::FOK_None:
+          if (!queuedInstantiation && R->isUsed(false)) {
+            if (MemberSpecializationInfo *MSInfo
+                = Function->getMemberSpecializationInfo()) {
+              if (MSInfo->getPointOfInstantiation().isInvalid()) {
+                SourceLocation Loc = R->getLocation(); // FIXME
+                MSInfo->setPointOfInstantiation(Loc);
+                SemaRef.PendingLocalImplicitInstantiations.push_back(
+                                                 std::make_pair(Function, Loc));
+                queuedInstantiation = true;
+              }
+            }
+          }
+          break;
+        default:
           if (const FunctionDecl *RPattern
-              = (*R)->getTemplateInstantiationPattern())
+              = R->getTemplateInstantiationPattern())
             if (RPattern->hasBody(RPattern)) {
               SemaRef.Diag(Function->getLocation(), diag::err_redefinition) 
                 << Function->getDeclName();
-              SemaRef.Diag((*R)->getLocation(), diag::note_previous_definition);
+              SemaRef.Diag(R->getLocation(), diag::note_previous_definition);
               Function->setInvalidDecl();
               break;
             }
         }
       }
     }
-      
   }
 
   if (Function->isOverloadedOperator() && !DC->isRecord() &&
@@ -1231,20 +1233,16 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     // We are creating a function template specialization from a function
     // template. Check whether there is already a function template
     // specialization for this particular set of template arguments.
-    llvm::FoldingSetNodeID ID;
     std::pair<const TemplateArgument *, unsigned> Innermost 
       = TemplateArgs.getInnermost();
-    FunctionTemplateSpecializationInfo::Profile(ID, Innermost.first,
-                                                Innermost.second,
-                                                SemaRef.Context);
 
-    FunctionTemplateSpecializationInfo *Info
-      = FunctionTemplate->getSpecializations().FindNodeOrInsertPos(ID,
-                                                                   InsertPos);
+    FunctionDecl *SpecFunc
+      = FunctionTemplate->findSpecialization(Innermost.first, Innermost.second,
+                                             InsertPos);
 
     // If we already have a function template specialization, return it.
-    if (Info)
-      return Info->Function;
+    if (SpecFunc)
+      return SpecFunc;
   }
 
   bool isFriend;
@@ -1256,7 +1254,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   bool MergeWithParentScope = (TemplateParams != 0) ||
     !(isa<Decl>(Owner) && 
       cast<Decl>(Owner)->isDefinedOutsideFunctionOrMethod());
-  Sema::LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
+  LocalInstantiationScope Scope(SemaRef, MergeWithParentScope);
 
   llvm::SmallVector<ParmVarDecl *, 4> Params;
   TypeSourceInfo *TInfo = D->getTypeSourceInfo();
@@ -1313,39 +1311,27 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
   CXXMethodDecl *Method = 0;
 
-  DeclarationName Name = D->getDeclName();
+  DeclarationNameInfo NameInfo
+    = SemaRef.SubstDeclarationNameInfo(D->getNameInfo(), TemplateArgs);
   if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(D)) {
-    QualType ClassTy = SemaRef.Context.getTypeDeclType(Record);
-    Name = SemaRef.Context.DeclarationNames.getCXXConstructorName(
-                                    SemaRef.Context.getCanonicalType(ClassTy));
     Method = CXXConstructorDecl::Create(SemaRef.Context, Record,
-                                        Constructor->getLocation(),
-                                        Name, T, TInfo,
+                                        NameInfo, T, TInfo,
                                         Constructor->isExplicit(),
                                         Constructor->isInlineSpecified(),
                                         false);
   } else if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(D)) {
-    QualType ClassTy = SemaRef.Context.getTypeDeclType(Record);
-    Name = SemaRef.Context.DeclarationNames.getCXXDestructorName(
-                                   SemaRef.Context.getCanonicalType(ClassTy));
     Method = CXXDestructorDecl::Create(SemaRef.Context, Record,
-                                       Destructor->getLocation(), Name,
-                                       T, Destructor->isInlineSpecified(),
+                                       NameInfo, T,
+                                       Destructor->isInlineSpecified(),
                                        false);
   } else if (CXXConversionDecl *Conversion = dyn_cast<CXXConversionDecl>(D)) {
-    CanQualType ConvTy
-      = SemaRef.Context.getCanonicalType(
-                                      T->getAs<FunctionType>()->getResultType());
-    Name = SemaRef.Context.DeclarationNames.getCXXConversionFunctionName(
-                                                                      ConvTy);
     Method = CXXConversionDecl::Create(SemaRef.Context, Record,
-                                       Conversion->getLocation(), Name,
-                                       T, TInfo,
+                                       NameInfo, T, TInfo,
                                        Conversion->isInlineSpecified(),
                                        Conversion->isExplicit());
   } else {
-    Method = CXXMethodDecl::Create(SemaRef.Context, Record, D->getLocation(),
-                                   D->getDeclName(), T, TInfo,
+    Method = CXXMethodDecl::Create(SemaRef.Context, Record,
+                                   NameInfo, T, TInfo,
                                    D->isStatic(),
                                    D->getStorageClassAsWritten(),
                                    D->isInlineSpecified());
@@ -1409,8 +1395,8 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   if (InitMethodInstantiation(Method, D))
     Method->setInvalidDecl();
 
-  LookupResult Previous(SemaRef, Name, SourceLocation(),
-                        Sema::LookupOrdinaryName, Sema::ForRedeclaration);
+  LookupResult Previous(SemaRef, NameInfo, Sema::LookupOrdinaryName,
+                        Sema::ForRedeclaration);
 
   if (!FunctionTemplate || TemplateParams || isFriend) {
     SemaRef.LookupQualifiedName(Previous, Record);
@@ -1446,7 +1432,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
     else
       Owner->addDecl(DeclToAdd);
   }
-
+  
   return Method;
 }
 
@@ -1475,8 +1461,8 @@ Decl *TemplateDeclInstantiator::VisitTemplateTypeParmDecl(
 
   TemplateTypeParmDecl *Inst =
     TemplateTypeParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
-                                 TTPT->getDepth() - 1, TTPT->getIndex(),
-                                 TTPT->getName(),
+                                 TTPT->getDepth() - TemplateArgs.getNumLevels(),
+                                 TTPT->getIndex(),TTPT->getName(),
                                  D->wasDeclaredWithTypename(),
                                  D->isParameterPack());
 
@@ -1517,8 +1503,9 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
   
   NonTypeTemplateParmDecl *Param
     = NonTypeTemplateParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
-                                      D->getDepth() - 1, D->getPosition(),
-                                      D->getIdentifier(), T, DI);
+                                    D->getDepth() - TemplateArgs.getNumLevels(), 
+                                      D->getPosition(), D->getIdentifier(), T, 
+                                      DI);
   if (Invalid)
     Param->setInvalidDecl();
   
@@ -1539,7 +1526,7 @@ TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
   {
     // Perform the actual substitution of template parameters within a new,
     // local instantiation scope.
-    Sema::LocalInstantiationScope Scope(SemaRef);
+    LocalInstantiationScope Scope(SemaRef);
     InstParams = SubstTemplateParams(TempParams);
     if (!InstParams)
       return NULL;
@@ -1548,8 +1535,9 @@ TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
   // Build the template template parameter.
   TemplateTemplateParmDecl *Param
     = TemplateTemplateParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
-                                       D->getDepth() - 1, D->getPosition(),
-                                       D->getIdentifier(), InstParams);
+                                   D->getDepth() - TemplateArgs.getNumLevels(), 
+                                       D->getPosition(), D->getIdentifier(), 
+                                       InstParams);
   Param->setDefaultArgument(D->getDefaultArgument(), false);
   
   // Introduce this template parameter's instantiation into the instantiation 
@@ -1575,22 +1563,22 @@ Decl *TemplateDeclInstantiator::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
 
 Decl *TemplateDeclInstantiator::VisitUsingDecl(UsingDecl *D) {
   // The nested name specifier is non-dependent, so no transformation
-  // is required.
+  // is required. The same holds for the name info.
+  DeclarationNameInfo NameInfo = D->getNameInfo();
 
   // We only need to do redeclaration lookups if we're in a class
   // scope (in fact, it's not really even possible in non-class
   // scopes).
   bool CheckRedeclaration = Owner->isRecord();
 
-  LookupResult Prev(SemaRef, D->getDeclName(), D->getLocation(),
-                    Sema::LookupUsingDeclName, Sema::ForRedeclaration);
+  LookupResult Prev(SemaRef, NameInfo, Sema::LookupUsingDeclName,
+                    Sema::ForRedeclaration);
 
   UsingDecl *NewUD = UsingDecl::Create(SemaRef.Context, Owner,
-                                       D->getLocation(),
                                        D->getNestedNameRange(),
                                        D->getUsingLocation(),
                                        D->getTargetNestedNameDecl(),
-                                       D->getDeclName(),
+                                       NameInfo,
                                        D->isTypeName());
 
   CXXScopeSpec SS;
@@ -1666,10 +1654,12 @@ Decl * TemplateDeclInstantiator
   SS.setRange(D->getTargetNestedNameRange());
   SS.setScopeRep(NNS);
 
+  // Since NameInfo refers to a typename, it cannot be a C++ special name.
+  // Hence, no tranformation is required for it.
+  DeclarationNameInfo NameInfo(D->getDeclName(), D->getLocation());
   NamedDecl *UD =
     SemaRef.BuildUsingDeclaration(/*Scope*/ 0, D->getAccess(),
-                                  D->getUsingLoc(), SS, D->getLocation(),
-                                  D->getDeclName(), 0,
+                                  D->getUsingLoc(), SS, NameInfo, 0,
                                   /*instantiation*/ true,
                                   /*typename*/ true, D->getTypenameLoc());
   if (UD)
@@ -1691,10 +1681,12 @@ Decl * TemplateDeclInstantiator
   SS.setRange(D->getTargetNestedNameRange());
   SS.setScopeRep(NNS);
 
+  DeclarationNameInfo NameInfo
+    = SemaRef.SubstDeclarationNameInfo(D->getNameInfo(), TemplateArgs);
+
   NamedDecl *UD =
     SemaRef.BuildUsingDeclaration(/*Scope*/ 0, D->getAccess(),
-                                  D->getUsingLoc(), SS, D->getLocation(),
-                                  D->getDeclName(), 0,
+                                  D->getUsingLoc(), SS, NameInfo, 0,
                                   /*instantiation*/ true,
                                   /*typename*/ false, SourceLocation());
   if (UD)
@@ -1735,13 +1727,8 @@ TemplateDeclInstantiator::SubstTemplateParams(TemplateParameterList *L) {
   }
 
   // Clean up if we had an error.
-  if (Invalid) {
-    for (ParamVector::iterator PI = Params.begin(), PE = Params.end();
-         PI != PE; ++PI)
-      if (*PI)
-        (*PI)->Destroy(SemaRef.Context);
+  if (Invalid)
     return NULL;
-  }
 
   TemplateParameterList *InstL
     = TemplateParameterList::Create(SemaRef.Context, L->getTemplateLoc(),
@@ -1767,7 +1754,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   // Create a local instantiation scope for this class template partial
   // specialization, which will contain the instantiations of the template
   // parameters.
-  Sema::LocalInstantiationScope Scope(SemaRef);
+  LocalInstantiationScope Scope(SemaRef);
   
   // Substitute into the template parameters of the class template partial
   // specialization.
@@ -1804,15 +1791,10 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
 
   // Figure out where to insert this class template partial specialization
   // in the member template's set of class template partial specializations.
-  llvm::FoldingSetNodeID ID;
-  ClassTemplatePartialSpecializationDecl::Profile(ID,
-                                                  Converted.getFlatArguments(),
-                                                  Converted.flatSize(),
-                                                  SemaRef.Context);
   void *InsertPos = 0;
   ClassTemplateSpecializationDecl *PrevDecl
-    = ClassTemplate->getPartialSpecializations().FindNodeOrInsertPos(ID,
-                                                                     InsertPos);
+    = ClassTemplate->findPartialSpecialization(Converted.getFlatArguments(),
+                                                Converted.flatSize(), InsertPos);
   
   // Build the canonical type that describes the converted template
   // arguments of the class template partial specialization.
@@ -1871,7 +1853,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
                                                      InstTemplateArgs,
                                                      CanonType,
                                                      0,
-                             ClassTemplate->getPartialSpecializations().size());
+                             ClassTemplate->getNextPartialSpecSequenceNumber());
   // Substitute the nested name specifier, if any.
   if (SubstQualifier(PartialSpec, InstPartialSpec))
     return 0;
@@ -1881,8 +1863,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
   
   // Add this partial specialization to the set of class template partial
   // specializations.
-  ClassTemplate->getPartialSpecializations().InsertNode(InstPartialSpec,
-                                                        InsertPos);
+  ClassTemplate->AddPartialSpecialization(InstPartialSpec, InsertPos);
   return false;
 }
 
@@ -2003,7 +1984,7 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
                                                  Proto->getExtInfo()));
   }
 
-  InstantiateAttrs(Tmpl, New);
+  SemaRef.InstantiateAttrs(TemplateArgs, Tmpl, New);
 
   return false;
 }
@@ -2078,8 +2059,12 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
         Diag(PatternDecl->getLocation(), 
              diag::note_explicit_instantiation_here);
       Function->setInvalidDecl();
+    } else if (Function->getTemplateSpecializationKind()
+                 == TSK_ExplicitInstantiationDefinition) {
+      PendingInstantiations.push_back(
+        std::make_pair(Function, PointOfInstantiation));
     }
-      
+
     return;
   }
 
@@ -2099,13 +2084,13 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   // If we're performing recursive template instantiation, create our own
   // queue of pending implicit instantiations that we will instantiate later,
   // while we're still within our own instantiation context.
-  std::deque<PendingImplicitInstantiation> SavedPendingImplicitInstantiations;
+  std::deque<PendingImplicitInstantiation> SavedPendingInstantiations;
   if (Recursive)
-    PendingImplicitInstantiations.swap(SavedPendingImplicitInstantiations);
+    PendingInstantiations.swap(SavedPendingInstantiations);
 
   EnterExpressionEvaluationContext EvalContext(*this, 
-                                               Action::PotentiallyEvaluated);
-  ActOnStartOfFunctionDef(0, DeclPtrTy::make(Function));
+                                               Sema::PotentiallyEvaluated);
+  ActOnStartOfFunctionDef(0, Function);
 
   // Introduce a new scope where local variable instantiations will be
   // recorded, unless we're actually a member function within a local
@@ -2118,10 +2103,14 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   LocalInstantiationScope Scope(*this, MergeWithParentScope);
 
   // Introduce the instantiated function parameters into the local
-  // instantiation scope.
-  for (unsigned I = 0, N = PatternDecl->getNumParams(); I != N; ++I)
-    Scope.InstantiatedLocal(PatternDecl->getParamDecl(I),
-                            Function->getParamDecl(I));
+  // instantiation scope, and set the parameter names to those used
+  // in the template.
+  for (unsigned I = 0, N = PatternDecl->getNumParams(); I != N; ++I) {
+    const ParmVarDecl *PatternParam = PatternDecl->getParamDecl(I);
+    ParmVarDecl *FunctionParam = Function->getParamDecl(I);
+    FunctionParam->setDeclName(PatternParam->getDeclName());
+    Scope.InstantiatedLocal(PatternParam, FunctionParam);
+  }
 
   // Enter the scope of this instantiation. We don't use
   // PushDeclContext because we don't have a scope.
@@ -2139,12 +2128,12 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   }
 
   // Instantiate the function body.
-  OwningStmtResult Body = SubstStmt(Pattern, TemplateArgs);
+  StmtResult Body = SubstStmt(Pattern, TemplateArgs);
 
   if (Body.isInvalid())
     Function->setInvalidDecl();
   
-  ActOnFinishFunctionBody(DeclPtrTy::make(Function), move(Body),
+  ActOnFinishFunctionBody(Function, Body.get(),
                           /*IsInstantiation=*/true);
 
   PerformDependentDiagnostics(PatternDecl, TemplateArgs);
@@ -2156,16 +2145,16 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
 
   // This class may have local implicit instantiations that need to be
   // instantiation within this scope.
-  PerformPendingImplicitInstantiations(/*LocalOnly=*/true);
+  PerformPendingInstantiations(/*LocalOnly=*/true);
   Scope.Exit();
 
   if (Recursive) {
     // Instantiate any pending implicit instantiations found during the
     // instantiation of this template.
-    PerformPendingImplicitInstantiations();
+    PerformPendingInstantiations();
 
     // Restore the set of pending implicit instantiations.
-    PendingImplicitInstantiations.swap(SavedPendingImplicitInstantiations);
+    PendingInstantiations.swap(SavedPendingInstantiations);
   }
 }
 
@@ -2210,8 +2199,12 @@ void Sema::InstantiateStaticDataMemberDefinition(
            diag::err_explicit_instantiation_undefined_member)
         << 2 << Var->getDeclName() << Var->getDeclContext();
       Diag(Def->getLocation(), diag::note_explicit_instantiation_here);
-    }    
-    
+    } else if (Var->getTemplateSpecializationKind()
+                 == TSK_ExplicitInstantiationDefinition) {
+      PendingInstantiations.push_back(
+        std::make_pair(Var, PointOfInstantiation));
+    }
+
     return;
   }
 
@@ -2234,9 +2227,9 @@ void Sema::InstantiateStaticDataMemberDefinition(
   // If we're performing recursive template instantiation, create our own
   // queue of pending implicit instantiations that we will instantiate later,
   // while we're still within our own instantiation context.
-  std::deque<PendingImplicitInstantiation> SavedPendingImplicitInstantiations;
+  std::deque<PendingImplicitInstantiation> SavedPendingInstantiations;
   if (Recursive)
-    PendingImplicitInstantiations.swap(SavedPendingImplicitInstantiations);
+    PendingInstantiations.swap(SavedPendingInstantiations);
 
   // Enter the scope of this instantiation. We don't use
   // PushDeclContext because we don't have a scope.
@@ -2260,10 +2253,10 @@ void Sema::InstantiateStaticDataMemberDefinition(
   if (Recursive) {
     // Instantiate any pending implicit instantiations found during the
     // instantiation of this template.
-    PerformPendingImplicitInstantiations();
+    PerformPendingInstantiations();
 
     // Restore the set of pending implicit instantiations.
-    PendingImplicitInstantiations.swap(SavedPendingImplicitInstantiations);
+    PendingInstantiations.swap(SavedPendingInstantiations);
   }
 }
 
@@ -2281,8 +2274,13 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
        Inits != InitsEnd; ++Inits) {
     CXXBaseOrMemberInitializer *Init = *Inits;
 
+    // Only instantiate written initializers, let Sema re-construct implicit
+    // ones.
+    if (!Init->isWritten())
+      continue;
+
     SourceLocation LParenLoc, RParenLoc;
-    ASTOwningVector<&ActionBase::DeleteExpr> NewArgs(*this);
+    ASTOwningVector<Expr*> NewArgs(*this);
     llvm::SmallVector<SourceLocation, 4> CommaLocs;
 
     // Instantiate the initializer.
@@ -2341,7 +2339,7 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
   }
 
   // Assign all the initializers to the new constructor.
-  ActOnMemInitializers(DeclPtrTy::make(New),
+  ActOnMemInitializers(New,
                        /*FIXME: ColonLoc */
                        SourceLocation(),
                        NewInits.data(), NewInits.size(),
@@ -2588,7 +2586,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
   DeclContext *ParentDC = D->getDeclContext();
   if (isa<ParmVarDecl>(D) || isa<NonTypeTemplateParmDecl>(D) ||
       isa<TemplateTypeParmDecl>(D) || isa<TemplateTemplateParmDecl>(D) ||
-      ParentDC->isFunctionOrMethod()) {
+      (ParentDC->isFunctionOrMethod() && ParentDC->isDependentContext())) {
     // D is a local of some kind. Look into the map of local
     // declarations to their instantiations.
     return cast<NamedDecl>(CurrentInstantiationScope->getInstantiationOf(D));
@@ -2729,14 +2727,14 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
 
 /// \brief Performs template instantiation for all implicit template
 /// instantiations we have seen until this point.
-void Sema::PerformPendingImplicitInstantiations(bool LocalOnly) {
+void Sema::PerformPendingInstantiations(bool LocalOnly) {
   while (!PendingLocalImplicitInstantiations.empty() ||
-         (!LocalOnly && !PendingImplicitInstantiations.empty())) {
+         (!LocalOnly && !PendingInstantiations.empty())) {
     PendingImplicitInstantiation Inst;
 
     if (PendingLocalImplicitInstantiations.empty()) {
-      Inst = PendingImplicitInstantiations.front();
-      PendingImplicitInstantiations.pop_front();
+      Inst = PendingInstantiations.front();
+      PendingInstantiations.pop_front();
     } else {
       Inst = PendingLocalImplicitInstantiations.front();
       PendingLocalImplicitInstantiations.pop_front();
@@ -2744,12 +2742,12 @@ void Sema::PerformPendingImplicitInstantiations(bool LocalOnly) {
 
     // Instantiate function definitions
     if (FunctionDecl *Function = dyn_cast<FunctionDecl>(Inst.first)) {
-      PrettyStackTraceActionsDecl CrashInfo(DeclPtrTy::make(Function),
-                                            Function->getLocation(), *this,
-                                            Context.getSourceManager(),
-                                           "instantiating function definition");
-
-      InstantiateFunctionDefinition(/*FIXME:*/Inst.second, Function, true);
+      PrettyDeclStackTraceEntry CrashInfo(*this, Function, SourceLocation(),
+                                          "instantiating function definition");
+      bool DefinitionRequired = Function->getTemplateSpecializationKind() ==
+                                TSK_ExplicitInstantiationDefinition;
+      InstantiateFunctionDefinition(/*FIXME:*/Inst.second, Function, true,
+                                    DefinitionRequired);
       continue;
     }
 
@@ -2768,20 +2766,24 @@ void Sema::PerformPendingImplicitInstantiations(bool LocalOnly) {
     case TSK_Undeclared:
       assert(false && "Cannot instantitiate an undeclared specialization.");
     case TSK_ExplicitInstantiationDeclaration:
-    case TSK_ExplicitInstantiationDefinition:
     case TSK_ExplicitSpecialization:
-      continue;  // No longer need implicit instantiation.
+      continue;  // No longer need to instantiate this type.
+    case TSK_ExplicitInstantiationDefinition:
+      // We only need an instantiation if the pending instantiation *is* the
+      // explicit instantiation.
+      if (Var != Var->getMostRecentDeclaration()) continue;
     case TSK_ImplicitInstantiation:
       break;
     }
 
-    PrettyStackTraceActionsDecl CrashInfo(DeclPtrTy::make(Var),
-                                          Var->getLocation(), *this,
-                                          Context.getSourceManager(),
-                                          "instantiating static data member "
-                                          "definition");
+    PrettyDeclStackTraceEntry CrashInfo(*this, Var, Var->getLocation(),
+                                        "instantiating static data member "
+                                        "definition");
 
-    InstantiateStaticDataMemberDefinition(/*FIXME:*/Inst.second, Var, true);
+    bool DefinitionRequired = Var->getTemplateSpecializationKind() ==
+                              TSK_ExplicitInstantiationDefinition;
+    InstantiateStaticDataMemberDefinition(/*FIXME:*/Inst.second, Var, true,
+                                          DefinitionRequired);
   }
 }
 
@@ -2798,3 +2800,4 @@ void Sema::PerformDependentDiagnostics(const DeclContext *Pattern,
     }
   }
 }
+
diff --git a/lib/Sema/SemaType.cpp b/lib/Sema/SemaType.cpp
index a4fc98cd958e..aa30b5c2da30 100644
--- a/lib/Sema/SemaType.cpp
+++ b/lib/Sema/SemaType.cpp
@@ -11,7 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Template.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclObjC.h"
@@ -20,13 +21,12 @@
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/AST/Expr.h"
 #include "clang/Basic/PartialDiagnostic.h"
-#include "clang/Parse/DeclSpec.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/DeclSpec.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/ErrorHandling.h"
 using namespace clang;
 
-#include <iostream>
-
 /// \brief Perform adjustment on the parameter type of a function.
 ///
 /// This routine adjusts the given parameter type @p T to the actual
@@ -266,8 +266,7 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
   case DeclSpec::TST_enum:
   case DeclSpec::TST_union:
   case DeclSpec::TST_struct: {
-    TypeDecl *D 
-      = dyn_cast_or_null<TypeDecl>(static_cast<Decl *>(DS.getTypeRep()));
+    TypeDecl *D = dyn_cast_or_null<TypeDecl>(DS.getRepAsDecl());
     if (!D) {
       // This can happen in C++ with ambiguous lookups.
       Result = Context.IntTy;
@@ -299,9 +298,11 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     assert(DS.getTypeSpecWidth() == 0 && DS.getTypeSpecComplex() == 0 &&
            DS.getTypeSpecSign() == 0 &&
            "Can't handle qualifiers on typedef names yet!");
-    Result = TheSema.GetTypeFromParser(DS.getTypeRep());
-
-    if (DeclSpec::ProtocolQualifierListTy PQ = DS.getProtocolQualifiers()) {
+    Result = TheSema.GetTypeFromParser(DS.getRepAsType());
+    if (Result.isNull())
+      TheDeclarator.setInvalidType(true);
+    else if (DeclSpec::ProtocolQualifierListTy PQ
+               = DS.getProtocolQualifiers()) {
       if (const ObjCObjectType *ObjT = Result->getAs<ObjCObjectType>()) {
         // Silently drop any existing protocol qualifiers.
         // TODO: determine whether that's the right thing to do.
@@ -336,13 +337,13 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
   }
   case DeclSpec::TST_typeofType:
     // FIXME: Preserve type source info.
-    Result = TheSema.GetTypeFromParser(DS.getTypeRep());
+    Result = TheSema.GetTypeFromParser(DS.getRepAsType());
     assert(!Result.isNull() && "Didn't get a type for typeof?");
     // TypeQuals handled by caller.
     Result = Context.getTypeOfType(Result);
     break;
   case DeclSpec::TST_typeofExpr: {
-    Expr *E = static_cast<Expr *>(DS.getTypeRep());
+    Expr *E = DS.getRepAsExpr();
     assert(E && "Didn't get an expression for typeof?");
     // TypeQuals handled by caller.
     Result = TheSema.BuildTypeofExprType(E);
@@ -353,7 +354,7 @@ static QualType ConvertDeclSpecToType(Sema &TheSema,
     break;
   }
   case DeclSpec::TST_decltype: {
-    Expr *E = static_cast<Expr *>(DS.getTypeRep());
+    Expr *E = DS.getRepAsExpr();
     assert(E && "Didn't get an expression for decltype?");
     // TypeQuals handled by caller.
     Result = TheSema.BuildDecltypeType(E);
@@ -674,10 +675,9 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
       !ArraySize->getType()->isIntegerType()) {
     Diag(ArraySize->getLocStart(), diag::err_array_size_non_int)
       << ArraySize->getType() << ArraySize->getSourceRange();
-    ArraySize->Destroy(Context);
     return QualType();
   }
-  llvm::APSInt ConstVal(32);
+  llvm::APSInt ConstVal(Context.getTypeSize(Context.getSizeType()));
   if (!ArraySize) {
     if (ASM == ArrayType::Star)
       T = Context.getVariableArrayType(T, 0, ASM, Quals, Brackets);
@@ -707,7 +707,17 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
            isSFINAEContext()? diag::err_typecheck_zero_array_size
                             : diag::ext_typecheck_zero_array_size)
         << ArraySize->getSourceRange();
+    } else if (!T->isDependentType() && !T->isVariablyModifiedType() && 
+               !T->isIncompleteType()) {
+      // Is the array too large?      
+      unsigned ActiveSizeBits
+        = ConstantArrayType::getNumAddressingBits(Context, T, ConstVal);
+      if (ActiveSizeBits > ConstantArrayType::getMaxSizeBits(Context))
+        Diag(ArraySize->getLocStart(), diag::err_array_too_large)
+          << ConstVal.toString(10)
+          << ArraySize->getSourceRange();
     }
+    
     T = Context.getConstantArrayType(T, ConstVal, ASM, Quals);
   }
   // If this is not C99, extwarn about VLA's and C99 array size modifiers.
@@ -740,11 +750,8 @@ QualType Sema::BuildArrayType(QualType T, ArrayType::ArraySizeModifier ASM,
 /// \brief Build an ext-vector type.
 ///
 /// Run the required checks for the extended vector type.
-QualType Sema::BuildExtVectorType(QualType T, ExprArg ArraySize,
+QualType Sema::BuildExtVectorType(QualType T, Expr *ArraySize,
                                   SourceLocation AttrLoc) {
-
-  Expr *Arg = (Expr *)ArraySize.get();
-
   // unlike gcc's vector_size attribute, we do not allow vectors to be defined
   // in conjunction with complex types (pointers, arrays, functions, etc.).
   if (!T->isDependentType() &&
@@ -753,11 +760,11 @@ QualType Sema::BuildExtVectorType(QualType T, ExprArg ArraySize,
     return QualType();
   }
 
-  if (!Arg->isTypeDependent() && !Arg->isValueDependent()) {
+  if (!ArraySize->isTypeDependent() && !ArraySize->isValueDependent()) {
     llvm::APSInt vecSize(32);
-    if (!Arg->isIntegerConstantExpr(vecSize, Context)) {
+    if (!ArraySize->isIntegerConstantExpr(vecSize, Context)) {
       Diag(AttrLoc, diag::err_attribute_argument_not_int)
-      << "ext_vector_type" << Arg->getSourceRange();
+        << "ext_vector_type" << ArraySize->getSourceRange();
       return QualType();
     }
 
@@ -767,7 +774,7 @@ QualType Sema::BuildExtVectorType(QualType T, ExprArg ArraySize,
 
     if (vectorSize == 0) {
       Diag(AttrLoc, diag::err_attribute_zero_size)
-      << Arg->getSourceRange();
+      << ArraySize->getSourceRange();
       return QualType();
     }
 
@@ -775,8 +782,7 @@ QualType Sema::BuildExtVectorType(QualType T, ExprArg ArraySize,
       return Context.getExtVectorType(T, vectorSize);
   }
 
-  return Context.getDependentSizedExtVectorType(T, ArraySize.takeAs<Expr>(),
-                                                AttrLoc);
+  return Context.getDependentSizedExtVectorType(T, ArraySize, AttrLoc);
 }
 
 /// \brief Build a function type.
@@ -812,7 +818,8 @@ QualType Sema::BuildFunctionType(QualType T,
                                  QualType *ParamTypes,
                                  unsigned NumParamTypes,
                                  bool Variadic, unsigned Quals,
-                                 SourceLocation Loc, DeclarationName Entity) {
+                                 SourceLocation Loc, DeclarationName Entity,
+                                 const FunctionType::ExtInfo &Info) {
   if (T->isArrayType() || T->isFunctionType()) {
     Diag(Loc, diag::err_func_returning_array_function) 
       << T->isFunctionType() << T;
@@ -834,8 +841,7 @@ QualType Sema::BuildFunctionType(QualType T,
     return QualType();
 
   return Context.getFunctionType(T, ParamTypes, NumParamTypes, Variadic,
-                                 Quals, false, false, 0, 0,
-                                 FunctionType::ExtInfo());
+                                 Quals, false, false, 0, 0, Info);
 }
 
 /// \brief Build a member pointer type \c T Class::*.
@@ -883,6 +889,14 @@ QualType Sema::BuildMemberPointerType(QualType T, QualType Class,
     return QualType();
   }
 
+  // In the Microsoft ABI, the class is allowed to be an incomplete
+  // type. In such cases, the compiler makes a worst-case assumption.
+  // We make no such assumption right now, so emit an error if the
+  // class isn't a complete type.
+  if (Context.Target.getCXXABI() == CXXABI_Microsoft &&
+      RequireCompleteType(Loc, Class, diag::err_incomplete_type))
+    return QualType();
+
   return Context.getMemberPointerType(T, Class.getTypePtr());
 }
 
@@ -912,8 +926,8 @@ QualType Sema::BuildBlockPointerType(QualType T,
   return Context.getBlockPointerType(T);
 }
 
-QualType Sema::GetTypeFromParser(TypeTy *Ty, TypeSourceInfo **TInfo) {
-  QualType QT = QualType::getFromOpaquePtr(Ty);
+QualType Sema::GetTypeFromParser(ParsedType Ty, TypeSourceInfo **TInfo) {
+  QualType QT = Ty.get();
   if (QT.isNull()) {
     if (TInfo) *TInfo = 0;
     return QualType();
@@ -955,7 +969,7 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
     T = ConvertDeclSpecToType(*this, D, FnAttrsFromDeclSpec);
     
     if (!D.isInvalidType() && D.getDeclSpec().isTypeSpecOwned()) {
-      TagDecl* Owned = cast<TagDecl>((Decl *)D.getDeclSpec().getTypeRep());
+      TagDecl* Owned = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
       // Owned is embedded if it was defined here, or if it is the
       // very first (i.e., canonical) declaration of this tag type.
       Owned->setEmbeddedInDeclarator(Owned->isDefinition() ||
@@ -1174,7 +1188,7 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
       if (getLangOptions().CPlusPlus && D.getDeclSpec().isTypeSpecOwned()) {
         // C++ [dcl.fct]p6:
         //   Types shall not be defined in return or parameter types.
-        TagDecl *Tag = cast<TagDecl>((Decl *)D.getDeclSpec().getTypeRep());
+        TagDecl *Tag = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
         if (Tag->isDefinition())
           Diag(Tag->getLocation(), diag::err_type_defined_in_result_type)
             << Context.getTypeDeclType(Tag);
@@ -1221,8 +1235,7 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
         ArgTys.reserve(FTI.NumArgs);
 
         for (unsigned i = 0, e = FTI.NumArgs; i != e; ++i) {
-          ParmVarDecl *Param =
-            cast<ParmVarDecl>(FTI.ArgInfo[i].Param.getAs<Decl>());
+          ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[i].Param);
           QualType ArgTy = Param->getType();
           assert(!ArgTy.isNull() && "Couldn't parse type?");
 
@@ -1295,19 +1308,19 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
     }
     case DeclaratorChunk::MemberPointer:
       // The scope spec must refer to a class, or be dependent.
+      CXXScopeSpec &SS = DeclType.Mem.Scope();
       QualType ClsType;
-      if (DeclType.Mem.Scope().isInvalid()) {
+      if (SS.isInvalid()) {
         // Avoid emitting extra errors if we already errored on the scope.
         D.setInvalidType(true);
-      } else if (isDependentScopeSpecifier(DeclType.Mem.Scope())
-                 || dyn_cast_or_null<CXXRecordDecl>(
-                                   computeDeclContext(DeclType.Mem.Scope()))) {
+      } else if (isDependentScopeSpecifier(SS) ||
+                 dyn_cast_or_null<CXXRecordDecl>(computeDeclContext(SS))) {
         NestedNameSpecifier *NNS
-          = (NestedNameSpecifier *)DeclType.Mem.Scope().getScopeRep();
+          = static_cast<NestedNameSpecifier*>(SS.getScopeRep());
         NestedNameSpecifier *NNSPrefix = NNS->getPrefix();
         switch (NNS->getKind()) {
         case NestedNameSpecifier::Identifier:
-          ClsType = Context.getDependentNameType(ETK_None, NNSPrefix, 
+          ClsType = Context.getDependentNameType(ETK_None, NNSPrefix,
                                                  NNS->getAsIdentifier());
           break;
 
@@ -1315,11 +1328,15 @@ TypeSourceInfo *Sema::GetTypeForDeclarator(Declarator &D, Scope *S,
         case NestedNameSpecifier::Global:
           llvm_unreachable("Nested-name-specifier must name a type");
           break;
-            
+
         case NestedNameSpecifier::TypeSpec:
         case NestedNameSpecifier::TypeSpecWithTemplate:
           ClsType = QualType(NNS->getAsType(), 0);
-          if (NNSPrefix)
+          // Note: if NNS is dependent, then its prefix (if any) is already
+          // included in ClsType; this does not hold if the NNS is
+          // nondependent: in this case (if there is indeed a prefix)
+          // ClsType needs to be wrapped into an elaborated type.
+          if (NNSPrefix && !NNS->isDependent())
             ClsType = Context.getElaboratedType(ETK_None, NNSPrefix, ClsType);
           break;
         }
@@ -1455,7 +1472,7 @@ namespace {
     }
     void VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc TL) {
       TypeSourceInfo *TInfo = 0;
-      Sema::GetTypeFromParser(DS.getTypeRep(), &TInfo);
+      Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
 
       // If we got no declarator info from previous Sema routines,
       // just fill with the typespec loc.
@@ -1483,9 +1500,9 @@ namespace {
       assert(DS.getTypeSpecType() == DeclSpec::TST_typeofType);
       TL.setTypeofLoc(DS.getTypeSpecTypeLoc());
       TL.setParensRange(DS.getTypeofParensRange());
-      assert(DS.getTypeRep());
+      assert(DS.getRepAsType());
       TypeSourceInfo *TInfo = 0;
-      Sema::GetTypeFromParser(DS.getTypeRep(), &TInfo);
+      Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
       TL.setUnderlyingTInfo(TInfo);
     }
     void VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
@@ -1508,7 +1525,7 @@ namespace {
         = TypeWithKeyword::getKeywordForTypeSpec(DS.getTypeSpecType());
       if (Keyword == ETK_Typename) {
         TypeSourceInfo *TInfo = 0;
-        Sema::GetTypeFromParser(DS.getTypeRep(), &TInfo);
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
         if (TInfo) {
           TL.copy(cast<ElaboratedTypeLoc>(TInfo->getTypeLoc()));
           return;
@@ -1526,7 +1543,7 @@ namespace {
         = TypeWithKeyword::getKeywordForTypeSpec(DS.getTypeSpecType());
       if (Keyword == ETK_Typename) {
         TypeSourceInfo *TInfo = 0;
-        Sema::GetTypeFromParser(DS.getTypeRep(), &TInfo);
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
         if (TInfo) {
           TL.copy(cast<DependentNameTypeLoc>(TInfo->getTypeLoc()));
           return;
@@ -1546,7 +1563,7 @@ namespace {
         = TypeWithKeyword::getKeywordForTypeSpec(DS.getTypeSpecType());
       if (Keyword == ETK_Typename) {
         TypeSourceInfo *TInfo = 0;
-        Sema::GetTypeFromParser(DS.getTypeRep(), &TInfo);
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
         if (TInfo) {
           TL.copy(cast<DependentTemplateSpecializationTypeLoc>(
                     TInfo->getTypeLoc()));
@@ -1620,7 +1637,7 @@ namespace {
 
       const DeclaratorChunk::FunctionTypeInfo &FTI = Chunk.Fun;
       for (unsigned i = 0, e = TL.getNumArgs(), tpi = 0; i != e; ++i) {
-        ParmVarDecl *Param = FTI.ArgInfo[i].Param.getAs<ParmVarDecl>();
+        ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[i].Param);
         TL.setArg(tpi++, Param);
       }
       // FIXME: exception specs
@@ -1651,24 +1668,21 @@ Sema::GetTypeSourceInfoForDeclarator(Declarator &D, QualType T,
     CurrTL = CurrTL.getNextTypeLoc().getUnqualifiedLoc();
   }
   
-  TypeSpecLocFiller(D.getDeclSpec()).Visit(CurrTL);
-  
-  // We have source information for the return type that was not in the
-  // declaration specifiers; copy that information into the current type
-  // location so that it will be retained. This occurs, for example, with 
-  // a C++ conversion function, where the return type occurs within the
-  // declarator-id rather than in the declaration specifiers.
-  if (ReturnTypeInfo && D.getDeclSpec().getTypeSpecType() == TST_unspecified) {
+  // If we have different source information for the return type, use
+  // that.  This really only applies to C++ conversion functions.
+  if (ReturnTypeInfo) {
     TypeLoc TL = ReturnTypeInfo->getTypeLoc();
     assert(TL.getFullDataSize() == CurrTL.getFullDataSize());
     memcpy(CurrTL.getOpaqueData(), TL.getOpaqueData(), TL.getFullDataSize());
+  } else {
+    TypeSpecLocFiller(D.getDeclSpec()).Visit(CurrTL);
   }
       
   return TInfo;
 }
 
 /// \brief Create a LocInfoType to hold the given QualType and TypeSourceInfo.
-QualType Sema::CreateLocInfoType(QualType T, TypeSourceInfo *TInfo) {
+ParsedType Sema::CreateParsedType(QualType T, TypeSourceInfo *TInfo) {
   // FIXME: LocInfoTypes are "transient", only needed for passing to/from Parser
   // and Sema during declaration parsing. Try deallocating/caching them when
   // it's appropriate, instead of allocating them and keeping them around.
@@ -1676,7 +1690,7 @@ QualType Sema::CreateLocInfoType(QualType T, TypeSourceInfo *TInfo) {
   new (LocT) LocInfoType(T, TInfo);
   assert(LocT->getTypeClass() != T->getTypeClass() &&
          "LocInfoType's TypeClass conflicts with an existing Type class");
-  return QualType(LocT, 0);
+  return ParsedType::make(QualType(LocT, 0));
 }
 
 void LocInfoType::getAsStringInternal(std::string &Str,
@@ -1686,7 +1700,7 @@ void LocInfoType::getAsStringInternal(std::string &Str,
          " GetTypeFromParser");
 }
 
-Sema::TypeResult Sema::ActOnTypeName(Scope *S, Declarator &D) {
+TypeResult Sema::ActOnTypeName(Scope *S, Declarator &D) {
   // C99 6.7.6: Type names have no identifier.  This is already validated by
   // the parser.
   assert(D.getIdentifier() == 0 && "Type name should have no identifier!");
@@ -1710,8 +1724,7 @@ Sema::TypeResult Sema::ActOnTypeName(Scope *S, Declarator &D) {
         << Context.getTypeDeclType(OwnedTag);
   }
 
-  T = CreateLocInfoType(T, TInfo);
-  return T.getAsOpaquePtr();
+  return CreateParsedType(T, TInfo);
 }
 
 
@@ -1825,9 +1838,10 @@ bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr) {
     // Delay if this is not a function or pointer to block.
     if (!Type->isFunctionPointerType()
         && !Type->isBlockPointerType()
-        && !Type->isFunctionType())
+        && !Type->isFunctionType()
+        && !Type->isMemberFunctionPointerType())
       return true;
-
+    
     // Otherwise we can process right away.
     Type = S.Context.getNoReturnType(Type);
     return false;
@@ -1842,7 +1856,8 @@ bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr) {
     // Delay if this is not a function or pointer to block.
     if (!Type->isFunctionPointerType()
         && !Type->isBlockPointerType()
-        && !Type->isFunctionType())
+        && !Type->isFunctionType()
+        && !Type->isMemberFunctionPointerType())
       return true;
 
     // Otherwise we can process right away.
@@ -1868,6 +1883,12 @@ bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr) {
   QualType T = Type;
   if (const PointerType *PT = Type->getAs<PointerType>())
     T = PT->getPointeeType();
+  else if (const BlockPointerType *BPT = Type->getAs<BlockPointerType>())
+    T = BPT->getPointeeType();
+  else if (const MemberPointerType *MPT = Type->getAs<MemberPointerType>())
+    T = MPT->getPointeeType();
+  else if (const ReferenceType *RT = Type->getAs<ReferenceType>())
+    T = RT->getPointeeType();
   const FunctionType *Fn = T->getAs<FunctionType>();
 
   // Delay if the type didn't work out to a function.
@@ -1880,6 +1901,7 @@ bool ProcessFnAttr(Sema &S, QualType &Type, const AttributeList &Attr) {
   case AttributeList::AT_fastcall: CC = CC_X86FastCall; break;
   case AttributeList::AT_stdcall: CC = CC_X86StdCall; break;
   case AttributeList::AT_thiscall: CC = CC_X86ThisCall; break;
+  case AttributeList::AT_pascal: CC = CC_X86Pascal; break;
   default: llvm_unreachable("unexpected attribute kind"); return false;
   }
 
@@ -1946,8 +1968,7 @@ static void HandleVectorSizeAttr(QualType& CurType, const AttributeList &Attr,
     return;
   }
   // the base type must be integer or float, and can't already be a vector.
-  if (CurType->isVectorType() ||
-      (!CurType->isIntegerType() && !CurType->isRealFloatingType())) {
+  if (!CurType->isIntegerType() && !CurType->isRealFloatingType()) {
     S.Diag(Attr.getLoc(), diag::err_attribute_invalid_vector_type) << CurType;
     Attr.setInvalid();
     return;
@@ -2008,6 +2029,7 @@ void ProcessTypeAttributeList(Sema &S, QualType &Result,
     case AttributeList::AT_fastcall:
     case AttributeList::AT_stdcall:
     case AttributeList::AT_thiscall:
+    case AttributeList::AT_pascal:
     case AttributeList::AT_regparm:
       // Don't process these on the DeclSpec.
       if (IsDeclSpec ||
diff --git a/lib/Sema/TargetAttributesSema.cpp b/lib/Sema/TargetAttributesSema.cpp
index 87e7b9d00b09..1854e7430853 100644
--- a/lib/Sema/TargetAttributesSema.cpp
+++ b/lib/Sema/TargetAttributesSema.cpp
@@ -12,9 +12,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Sema.h"
 #include "TargetAttributesSema.h"
+#include "clang/Sema/SemaInternal.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/AST/DeclCXX.h"
 #include "llvm/ADT/Triple.h"
 
 using namespace clang;
@@ -51,8 +52,8 @@ static void HandleMSP430InterruptAttr(Decl *d,
       return;
     }
 
-    d->addAttr(::new (S.Context) MSP430InterruptAttr(Num));
-    d->addAttr(::new (S.Context) UsedAttr());
+    d->addAttr(::new (S.Context) MSP430InterruptAttr(Attr.getLoc(), S.Context, Num));
+    d->addAttr(::new (S.Context) UsedAttr(Attr.getLoc(), S.Context));
   }
 
 namespace {
@@ -97,7 +98,7 @@ static void HandleX86ForceAlignArgPointerAttr(Decl *D,
     return;
   }
 
-  D->addAttr(::new (S.Context) X86ForceAlignArgPointerAttr());
+  D->addAttr(::new (S.Context) X86ForceAlignArgPointerAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleDLLImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -109,7 +110,7 @@ static void HandleDLLImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
 
   // Attribute can be applied only to functions or variables.
   if (isa<VarDecl>(D)) {
-    D->addAttr(::new (S.Context) DLLImportAttr());
+    D->addAttr(::new (S.Context) DLLImportAttr(Attr.getLoc(), S.Context));
     return;
   }
 
@@ -146,7 +147,7 @@ static void HandleDLLImportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  D->addAttr(::new (S.Context) DLLImportAttr());
+  D->addAttr(::new (S.Context) DLLImportAttr(Attr.getLoc(), S.Context));
 }
 
 static void HandleDLLExportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
@@ -158,7 +159,7 @@ static void HandleDLLExportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
 
   // Attribute can be applied only to functions or variables.
   if (isa<VarDecl>(D)) {
-    D->addAttr(::new (S.Context) DLLExportAttr());
+    D->addAttr(::new (S.Context) DLLExportAttr(Attr.getLoc(), S.Context));
     return;
   }
 
@@ -177,7 +178,7 @@ static void HandleDLLExportAttr(Decl *D, const AttributeList &Attr, Sema &S) {
     return;
   }
 
-  D->addAttr(::new (S.Context) DLLExportAttr());
+  D->addAttr(::new (S.Context) DLLExportAttr(Attr.getLoc(), S.Context));
 }
 
 namespace {
diff --git a/lib/Sema/TargetAttributesSema.h b/lib/Sema/TargetAttributesSema.h
index 8794e4013ec7..410c900222f6 100644
--- a/lib/Sema/TargetAttributesSema.h
+++ b/lib/Sema/TargetAttributesSema.h
@@ -13,7 +13,7 @@
 namespace clang {
   class Scope;
   class Decl;
-  class Attr;
+  class AttributeList;
   class Sema;
 
   class TargetAttributesSema {
diff --git a/lib/Sema/TreeTransform.h b/lib/Sema/TreeTransform.h
index 17103c515f8b..e7bfbe6fe955 100644
--- a/lib/Sema/TreeTransform.h
+++ b/lib/Sema/TreeTransform.h
@@ -13,10 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_TREETRANSFORM_H
 #define LLVM_CLANG_SEMA_TREETRANSFORM_H
 
-#include "Sema.h"
-#include "Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/Sema/SemaDiagnostic.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
@@ -24,13 +26,14 @@
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtObjC.h"
 #include "clang/AST/TypeLocBuilder.h"
-#include "clang/Parse/Ownership.h"
-#include "clang/Parse/Designator.h"
+#include "clang/Sema/Ownership.h"
+#include "clang/Sema/Designator.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <algorithm>
 
 namespace clang {
+using namespace sema;
 
 /// \brief A semantic tree transformation that allows one to transform one
 /// abstract syntax tree into another.
@@ -89,14 +92,6 @@ protected:
   Sema &SemaRef;
 
 public:
-  typedef Sema::OwningStmtResult OwningStmtResult;
-  typedef Sema::OwningExprResult OwningExprResult;
-  typedef Sema::StmtArg StmtArg;
-  typedef Sema::ExprArg ExprArg;
-  typedef Sema::MultiExprArg MultiExprArg;
-  typedef Sema::MultiStmtArg MultiStmtArg;
-  typedef Sema::DeclPtrTy DeclPtrTy;
-  
   /// \brief Initializes a new tree transformer.
   TreeTransform(Sema &SemaRef) : SemaRef(SemaRef) { }
 
@@ -108,6 +103,9 @@ public:
     return static_cast<const Derived&>(*this);
   }
 
+  static inline ExprResult Owned(Expr *E) { return E; }
+  static inline StmtResult Owned(Stmt *S) { return S; }
+
   /// \brief Retrieves a reference to the semantic analysis object used for
   /// this tree transform.
   Sema &getSema() const { return SemaRef; }
@@ -220,7 +218,7 @@ public:
   /// other mechanism.
   ///
   /// \returns the transformed statement.
-  OwningStmtResult TransformStmt(Stmt *S);
+  StmtResult TransformStmt(Stmt *S);
 
   /// \brief Transform the given expression.
   ///
@@ -230,7 +228,7 @@ public:
   /// other mechanism.
   ///
   /// \returns the transformed expression.
-  OwningExprResult TransformExpr(Expr *E);
+  ExprResult TransformExpr(Expr *E);
 
   /// \brief Transform the given declaration, which is referenced from a type
   /// or expression.
@@ -276,9 +274,9 @@ public:
   /// and destructor names and then (if needed) rebuilds the declaration name.
   /// Identifiers and selectors are returned unmodified. Sublcasses may
   /// override this function to provide alternate behavior.
-  DeclarationName TransformDeclarationName(DeclarationName Name,
-                                           SourceLocation Loc,
-                                           QualType ObjectType = QualType());
+  DeclarationNameInfo
+  TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
+                               QualType ObjectType = QualType());
 
   /// \brief Transform the given template name.
   ///
@@ -337,13 +335,13 @@ public:
   TransformTemplateSpecializationType(const TemplateSpecializationType *T,
                                       QualType ObjectType);
 
-  OwningStmtResult TransformCompoundStmt(CompoundStmt *S, bool IsStmtExpr);
-  OwningExprResult TransformCXXNamedCastExpr(CXXNamedCastExpr *E);
+  StmtResult TransformCompoundStmt(CompoundStmt *S, bool IsStmtExpr);
+  ExprResult TransformCXXNamedCastExpr(CXXNamedCastExpr *E);
 
 #define STMT(Node, Parent)                        \
-  OwningStmtResult Transform##Node(Node *S);
+  StmtResult Transform##Node(Node *S);
 #define EXPR(Node, Parent)                        \
-  OwningExprResult Transform##Node(Node *E);
+  ExprResult Transform##Node(Node *E);
 #define ABSTRACT_STMT(Stmt)
 #include "clang/AST/StmtNodes.inc"
 
@@ -421,7 +419,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   QualType RebuildVariableArrayType(QualType ElementType,
                                     ArrayType::ArraySizeModifier SizeMod,
-                                    ExprArg SizeExpr,
+                                    Expr *SizeExpr,
                                     unsigned IndexTypeQuals,
                                     SourceRange BracketsRange);
 
@@ -432,7 +430,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   QualType RebuildDependentSizedArrayType(QualType ElementType,
                                           ArrayType::ArraySizeModifier SizeMod,
-                                          ExprArg SizeExpr,
+                                          Expr *SizeExpr,
                                           unsigned IndexTypeQuals,
                                           SourceRange BracketsRange);
 
@@ -458,7 +456,7 @@ public:
   /// By default, performs semantic analysis when building the vector type.
   /// Subclasses may override this routine to provide different behavior.
   QualType RebuildDependentSizedExtVectorType(QualType ElementType,
-                                              ExprArg SizeExpr,
+                                              Expr *SizeExpr,
                                               SourceLocation AttributeLoc);
 
   /// \brief Build a new function type.
@@ -468,7 +466,8 @@ public:
   QualType RebuildFunctionProtoType(QualType T,
                                     QualType *ParamTypes,
                                     unsigned NumParamTypes,
-                                    bool Variadic, unsigned Quals);
+                                    bool Variadic, unsigned Quals,
+                                    const FunctionType::ExtInfo &Info);
 
   /// \brief Build a new unprototyped function type.
   QualType RebuildFunctionNoProtoType(QualType ResultType);
@@ -496,7 +495,7 @@ public:
   ///
   /// By default, performs semantic analysis when building the typeof type.
   /// Subclasses may override this routine to provide different behavior.
-  QualType RebuildTypeOfExprType(ExprArg Underlying);
+  QualType RebuildTypeOfExprType(Expr *Underlying);
 
   /// \brief Build a new typeof(type) type.
   ///
@@ -507,7 +506,7 @@ public:
   ///
   /// By default, performs semantic analysis when building the decltype type.
   /// Subclasses may override this routine to provide different behavior.
-  QualType RebuildDecltypeType(ExprArg Underlying);
+  QualType RebuildDecltypeType(Expr *Underlying);
 
   /// \brief Build a new template specialization type.
   ///
@@ -558,7 +557,8 @@ public:
       getDerived().RebuildTemplateSpecializationType(InstName, NameLoc, Args);
     if (T.isNull()) return QualType();
 
-    return SemaRef.Context.getElaboratedType(Keyword, NNS, T);
+    // NOTE: NNS is already recorded in template specialization type T.
+    return SemaRef.Context.getElaboratedType(Keyword, /*NNS=*/0, T);
   }
 
   /// \brief Build a new typename type that refers to an identifier.
@@ -707,11 +707,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildCompoundStmt(SourceLocation LBraceLoc,
+  StmtResult RebuildCompoundStmt(SourceLocation LBraceLoc,
                                        MultiStmtArg Statements,
                                        SourceLocation RBraceLoc,
                                        bool IsStmtExpr) {
-    return getSema().ActOnCompoundStmt(LBraceLoc, RBraceLoc, move(Statements),
+    return getSema().ActOnCompoundStmt(LBraceLoc, RBraceLoc, Statements,
                                        IsStmtExpr);
   }
 
@@ -719,12 +719,12 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildCaseStmt(SourceLocation CaseLoc,
-                                   ExprArg LHS,
+  StmtResult RebuildCaseStmt(SourceLocation CaseLoc,
+                                   Expr *LHS,
                                    SourceLocation EllipsisLoc,
-                                   ExprArg RHS,
+                                   Expr *RHS,
                                    SourceLocation ColonLoc) {
-    return getSema().ActOnCaseStmt(CaseLoc, move(LHS), EllipsisLoc, move(RHS),
+    return getSema().ActOnCaseStmt(CaseLoc, LHS, EllipsisLoc, RHS,
                                    ColonLoc);
   }
 
@@ -732,19 +732,19 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildCaseStmtBody(StmtArg S, StmtArg Body) {
-    getSema().ActOnCaseStmtBody(S.get(), move(Body));
-    return move(S);
+  StmtResult RebuildCaseStmtBody(Stmt *S, Stmt *Body) {
+    getSema().ActOnCaseStmtBody(S, Body);
+    return S;
   }
 
   /// \brief Build a new default statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildDefaultStmt(SourceLocation DefaultLoc,
+  StmtResult RebuildDefaultStmt(SourceLocation DefaultLoc,
                                       SourceLocation ColonLoc,
-                                      StmtArg SubStmt) {
-    return getSema().ActOnDefaultStmt(DefaultLoc, ColonLoc, move(SubStmt),
+                                      Stmt *SubStmt) {
+    return getSema().ActOnDefaultStmt(DefaultLoc, ColonLoc, SubStmt,
                                       /*CurScope=*/0);
   }
 
@@ -752,89 +752,85 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildLabelStmt(SourceLocation IdentLoc,
+  StmtResult RebuildLabelStmt(SourceLocation IdentLoc,
                                     IdentifierInfo *Id,
                                     SourceLocation ColonLoc,
-                                    StmtArg SubStmt) {
-    return SemaRef.ActOnLabelStmt(IdentLoc, Id, ColonLoc, move(SubStmt));
+                                    Stmt *SubStmt) {
+    return SemaRef.ActOnLabelStmt(IdentLoc, Id, ColonLoc, SubStmt);
   }
 
   /// \brief Build a new "if" statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildIfStmt(SourceLocation IfLoc, Sema::FullExprArg Cond,
-                                 VarDecl *CondVar, StmtArg Then, 
-                                 SourceLocation ElseLoc, StmtArg Else) {
-    return getSema().ActOnIfStmt(IfLoc, Cond, DeclPtrTy::make(CondVar), 
-                                 move(Then), ElseLoc, move(Else));
+  StmtResult RebuildIfStmt(SourceLocation IfLoc, Sema::FullExprArg Cond,
+                                 VarDecl *CondVar, Stmt *Then, 
+                                 SourceLocation ElseLoc, Stmt *Else) {
+    return getSema().ActOnIfStmt(IfLoc, Cond, CondVar, Then, ElseLoc, Else);
   }
 
   /// \brief Start building a new switch statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildSwitchStmtStart(SourceLocation SwitchLoc,
-                                          Sema::ExprArg Cond, 
-                                          VarDecl *CondVar) {
-    return getSema().ActOnStartOfSwitchStmt(SwitchLoc, move(Cond), 
-                                            DeclPtrTy::make(CondVar));
+  StmtResult RebuildSwitchStmtStart(SourceLocation SwitchLoc,
+                                          Expr *Cond, VarDecl *CondVar) {
+    return getSema().ActOnStartOfSwitchStmt(SwitchLoc, Cond, 
+                                            CondVar);
   }
 
   /// \brief Attach the body to the switch statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildSwitchStmtBody(SourceLocation SwitchLoc,
-                                         StmtArg Switch, StmtArg Body) {
-    return getSema().ActOnFinishSwitchStmt(SwitchLoc, move(Switch),
-                                         move(Body));
+  StmtResult RebuildSwitchStmtBody(SourceLocation SwitchLoc,
+                                         Stmt *Switch, Stmt *Body) {
+    return getSema().ActOnFinishSwitchStmt(SwitchLoc, Switch, Body);
   }
 
   /// \brief Build a new while statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildWhileStmt(SourceLocation WhileLoc,
+  StmtResult RebuildWhileStmt(SourceLocation WhileLoc,
                                     Sema::FullExprArg Cond,
                                     VarDecl *CondVar,
-                                    StmtArg Body) {
-    return getSema().ActOnWhileStmt(WhileLoc, Cond, 
-                                    DeclPtrTy::make(CondVar), move(Body));
+                                    Stmt *Body) {
+    return getSema().ActOnWhileStmt(WhileLoc, Cond, CondVar, Body);
   }
 
   /// \brief Build a new do-while statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildDoStmt(SourceLocation DoLoc, StmtArg Body,
+  StmtResult RebuildDoStmt(SourceLocation DoLoc, Stmt *Body,
                                  SourceLocation WhileLoc,
                                  SourceLocation LParenLoc,
-                                 ExprArg Cond,
+                                 Expr *Cond,
                                  SourceLocation RParenLoc) {
-    return getSema().ActOnDoStmt(DoLoc, move(Body), WhileLoc, LParenLoc,
-                                 move(Cond), RParenLoc);
+    return getSema().ActOnDoStmt(DoLoc, Body, WhileLoc, LParenLoc,
+                                 Cond, RParenLoc);
   }
 
   /// \brief Build a new for statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildForStmt(SourceLocation ForLoc,
+  StmtResult RebuildForStmt(SourceLocation ForLoc,
                                   SourceLocation LParenLoc,
-                                  StmtArg Init, Sema::FullExprArg Cond, 
+                                  Stmt *Init, Sema::FullExprArg Cond, 
                                   VarDecl *CondVar, Sema::FullExprArg Inc,
-                                  SourceLocation RParenLoc, StmtArg Body) {
-    return getSema().ActOnForStmt(ForLoc, LParenLoc, move(Init), Cond, 
-                                  DeclPtrTy::make(CondVar),
-                                  Inc, RParenLoc, move(Body));
+                                  SourceLocation RParenLoc, Stmt *Body) {
+    return getSema().ActOnForStmt(ForLoc, LParenLoc, Init, Cond, 
+                                  CondVar,
+                                  Inc, RParenLoc, Body);
   }
 
   /// \brief Build a new goto statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildGotoStmt(SourceLocation GotoLoc,
+  StmtResult RebuildGotoStmt(SourceLocation GotoLoc,
                                    SourceLocation LabelLoc,
                                    LabelStmt *Label) {
     return getSema().ActOnGotoStmt(GotoLoc, LabelLoc, Label->getID());
@@ -844,27 +840,27 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildIndirectGotoStmt(SourceLocation GotoLoc,
+  StmtResult RebuildIndirectGotoStmt(SourceLocation GotoLoc,
                                            SourceLocation StarLoc,
-                                           ExprArg Target) {
-    return getSema().ActOnIndirectGotoStmt(GotoLoc, StarLoc, move(Target));
+                                           Expr *Target) {
+    return getSema().ActOnIndirectGotoStmt(GotoLoc, StarLoc, Target);
   }
 
   /// \brief Build a new return statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildReturnStmt(SourceLocation ReturnLoc,
-                                     ExprArg Result) {
+  StmtResult RebuildReturnStmt(SourceLocation ReturnLoc,
+                                     Expr *Result) {
 
-    return getSema().ActOnReturnStmt(ReturnLoc, move(Result));
+    return getSema().ActOnReturnStmt(ReturnLoc, Result);
   }
 
   /// \brief Build a new declaration statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildDeclStmt(Decl **Decls, unsigned NumDecls,
+  StmtResult RebuildDeclStmt(Decl **Decls, unsigned NumDecls,
                                    SourceLocation StartLoc,
                                    SourceLocation EndLoc) {
     return getSema().Owned(
@@ -878,7 +874,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildAsmStmt(SourceLocation AsmLoc,
+  StmtResult RebuildAsmStmt(SourceLocation AsmLoc,
                                   bool IsSimple,
                                   bool IsVolatile,
                                   unsigned NumOutputs,
@@ -886,13 +882,13 @@ public:
                                   IdentifierInfo **Names,
                                   MultiExprArg Constraints,
                                   MultiExprArg Exprs,
-                                  ExprArg AsmString,
+                                  Expr *AsmString,
                                   MultiExprArg Clobbers,
                                   SourceLocation RParenLoc,
                                   bool MSAsm) {
     return getSema().ActOnAsmStmt(AsmLoc, IsSimple, IsVolatile, NumOutputs, 
                                   NumInputs, Names, move(Constraints),
-                                  move(Exprs), move(AsmString), move(Clobbers),
+                                  Exprs, AsmString, Clobbers,
                                   RParenLoc, MSAsm);
   }
 
@@ -900,12 +896,12 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildObjCAtTryStmt(SourceLocation AtLoc,
-                                        StmtArg TryBody,
+  StmtResult RebuildObjCAtTryStmt(SourceLocation AtLoc,
+                                        Stmt *TryBody,
                                         MultiStmtArg CatchStmts,
-                                        StmtArg Finally) {
-    return getSema().ActOnObjCAtTryStmt(AtLoc, move(TryBody), move(CatchStmts),
-                                        move(Finally));
+                                        Stmt *Finally) {
+    return getSema().ActOnObjCAtTryStmt(AtLoc, TryBody, move(CatchStmts),
+                                        Finally);
   }
 
   /// \brief Rebuild an Objective-C exception declaration.
@@ -923,59 +919,58 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildObjCAtCatchStmt(SourceLocation AtLoc,
+  StmtResult RebuildObjCAtCatchStmt(SourceLocation AtLoc,
                                           SourceLocation RParenLoc,
                                           VarDecl *Var,
-                                          StmtArg Body) {
+                                          Stmt *Body) {
     return getSema().ActOnObjCAtCatchStmt(AtLoc, RParenLoc,
-                                          Sema::DeclPtrTy::make(Var),
-                                          move(Body));
+                                          Var, Body);
   }
   
   /// \brief Build a new Objective-C @finally statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildObjCAtFinallyStmt(SourceLocation AtLoc,
-                                            StmtArg Body) {
-    return getSema().ActOnObjCAtFinallyStmt(AtLoc, move(Body));
+  StmtResult RebuildObjCAtFinallyStmt(SourceLocation AtLoc,
+                                            Stmt *Body) {
+    return getSema().ActOnObjCAtFinallyStmt(AtLoc, Body);
   }
   
   /// \brief Build a new Objective-C @throw statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildObjCAtThrowStmt(SourceLocation AtLoc,
-                                          ExprArg Operand) {
-    return getSema().BuildObjCAtThrowStmt(AtLoc, move(Operand));
+  StmtResult RebuildObjCAtThrowStmt(SourceLocation AtLoc,
+                                          Expr *Operand) {
+    return getSema().BuildObjCAtThrowStmt(AtLoc, Operand);
   }
   
   /// \brief Build a new Objective-C @synchronized statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildObjCAtSynchronizedStmt(SourceLocation AtLoc,
-                                                 ExprArg Object,
-                                                 StmtArg Body) {
-    return getSema().ActOnObjCAtSynchronizedStmt(AtLoc, move(Object),
-                                                 move(Body));
+  StmtResult RebuildObjCAtSynchronizedStmt(SourceLocation AtLoc,
+                                                 Expr *Object,
+                                                 Stmt *Body) {
+    return getSema().ActOnObjCAtSynchronizedStmt(AtLoc, Object,
+                                                 Body);
   }
 
   /// \brief Build a new Objective-C fast enumeration statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildObjCForCollectionStmt(SourceLocation ForLoc,
-                                                SourceLocation LParenLoc,
-                                                StmtArg Element,
-                                                ExprArg Collection,
-                                                SourceLocation RParenLoc,
-                                                StmtArg Body) {
+  StmtResult RebuildObjCForCollectionStmt(SourceLocation ForLoc,
+                                          SourceLocation LParenLoc,
+                                          Stmt *Element,
+                                          Expr *Collection,
+                                          SourceLocation RParenLoc,
+                                          Stmt *Body) {
     return getSema().ActOnObjCForCollectionStmt(ForLoc, LParenLoc,
-                                                move(Element), 
-                                                move(Collection),
+                                                Element, 
+                                                Collection,
                                                 RParenLoc,
-                                                move(Body));
+                                                Body);
   }
   
   /// \brief Build a new C++ exception declaration.
@@ -995,31 +990,30 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildCXXCatchStmt(SourceLocation CatchLoc,
-                                       VarDecl *ExceptionDecl,
-                                       StmtArg Handler) {
-    return getSema().Owned(
-             new (getSema().Context) CXXCatchStmt(CatchLoc, ExceptionDecl,
-                                                  Handler.takeAs<Stmt>()));
+  StmtResult RebuildCXXCatchStmt(SourceLocation CatchLoc,
+                                 VarDecl *ExceptionDecl,
+                                 Stmt *Handler) {
+    return Owned(new (getSema().Context) CXXCatchStmt(CatchLoc, ExceptionDecl,
+                                                      Handler));
   }
 
   /// \brief Build a new C++ try statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  OwningStmtResult RebuildCXXTryStmt(SourceLocation TryLoc,
-                                     StmtArg TryBlock,
-                                     MultiStmtArg Handlers) {
-    return getSema().ActOnCXXTryBlock(TryLoc, move(TryBlock), move(Handlers));
+  StmtResult RebuildCXXTryStmt(SourceLocation TryLoc,
+                               Stmt *TryBlock,
+                               MultiStmtArg Handlers) {
+    return getSema().ActOnCXXTryBlock(TryLoc, TryBlock, move(Handlers));
   }
 
   /// \brief Build a new expression that references a declaration.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildDeclarationNameExpr(const CXXScopeSpec &SS,
-                                              LookupResult &R,
-                                              bool RequiresADL) {
+  ExprResult RebuildDeclarationNameExpr(const CXXScopeSpec &SS,
+                                        LookupResult &R,
+                                        bool RequiresADL) {
     return getSema().BuildDeclarationNameExpr(SS, R, RequiresADL);
   }
 
@@ -1028,33 +1022,34 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildDeclRefExpr(NestedNameSpecifier *Qualifier,
-                                      SourceRange QualifierRange,
-                                      ValueDecl *VD, SourceLocation Loc,
-                                      TemplateArgumentListInfo *TemplateArgs) {
+  ExprResult RebuildDeclRefExpr(NestedNameSpecifier *Qualifier,
+                                SourceRange QualifierRange,
+                                ValueDecl *VD,
+                                const DeclarationNameInfo &NameInfo,
+                                TemplateArgumentListInfo *TemplateArgs) {
     CXXScopeSpec SS;
     SS.setScopeRep(Qualifier);
     SS.setRange(QualifierRange);
 
     // FIXME: loses template args.
-    
-    return getSema().BuildDeclarationNameExpr(SS, Loc, VD);
+
+    return getSema().BuildDeclarationNameExpr(SS, NameInfo, VD);
   }
 
   /// \brief Build a new expression in parentheses.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildParenExpr(ExprArg SubExpr, SourceLocation LParen,
+  ExprResult RebuildParenExpr(Expr *SubExpr, SourceLocation LParen,
                                     SourceLocation RParen) {
-    return getSema().ActOnParenExpr(LParen, RParen, move(SubExpr));
+    return getSema().ActOnParenExpr(LParen, RParen, SubExpr);
   }
 
   /// \brief Build a new pseudo-destructor expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXPseudoDestructorExpr(ExprArg Base,
+  ExprResult RebuildCXXPseudoDestructorExpr(Expr *Base,
                                                   SourceLocation OperatorLoc,
                                                   bool isArrow,
                                                 NestedNameSpecifier *Qualifier,
@@ -1068,19 +1063,19 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildUnaryOperator(SourceLocation OpLoc,
-                                        UnaryOperator::Opcode Opc,
-                                        ExprArg SubExpr) {
-    return getSema().BuildUnaryOp(/*Scope=*/0, OpLoc, Opc, move(SubExpr));
+  ExprResult RebuildUnaryOperator(SourceLocation OpLoc,
+                                        UnaryOperatorKind Opc,
+                                        Expr *SubExpr) {
+    return getSema().BuildUnaryOp(/*Scope=*/0, OpLoc, Opc, SubExpr);
   }
 
   /// \brief Build a new builtin offsetof expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildOffsetOfExpr(SourceLocation OperatorLoc,
+  ExprResult RebuildOffsetOfExpr(SourceLocation OperatorLoc,
                                        TypeSourceInfo *Type,
-                                       Action::OffsetOfComponent *Components,
+                                       Sema::OffsetOfComponent *Components,
                                        unsigned NumComponents,
                                        SourceLocation RParenLoc) {
     return getSema().BuildBuiltinOffsetOf(OperatorLoc, Type, Components,
@@ -1091,7 +1086,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildSizeOfAlignOf(TypeSourceInfo *TInfo,
+  ExprResult RebuildSizeOfAlignOf(TypeSourceInfo *TInfo,
                                         SourceLocation OpLoc,
                                         bool isSizeOf, SourceRange R) {
     return getSema().CreateSizeOfAlignOfExpr(TInfo, OpLoc, isSizeOf, R);
@@ -1102,15 +1097,13 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildSizeOfAlignOf(ExprArg SubExpr, SourceLocation OpLoc,
+  ExprResult RebuildSizeOfAlignOf(Expr *SubExpr, SourceLocation OpLoc,
                                         bool isSizeOf, SourceRange R) {
-    OwningExprResult Result
-      = getSema().CreateSizeOfAlignOfExpr((Expr *)SubExpr.get(),
-                                          OpLoc, isSizeOf, R);
+    ExprResult Result
+      = getSema().CreateSizeOfAlignOfExpr(SubExpr, OpLoc, isSizeOf, R);
     if (Result.isInvalid())
-      return getSema().ExprError();
+      return ExprError();
 
-    SubExpr.release();
     return move(Result);
   }
 
@@ -1118,12 +1111,12 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildArraySubscriptExpr(ExprArg LHS,
+  ExprResult RebuildArraySubscriptExpr(Expr *LHS,
                                              SourceLocation LBracketLoc,
-                                             ExprArg RHS,
+                                             Expr *RHS,
                                              SourceLocation RBracketLoc) {
-    return getSema().ActOnArraySubscriptExpr(/*Scope=*/0, move(LHS),
-                                             LBracketLoc, move(RHS),
+    return getSema().ActOnArraySubscriptExpr(/*Scope=*/0, LHS,
+                                             LBracketLoc, RHS,
                                              RBracketLoc);
   }
 
@@ -1131,11 +1124,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCallExpr(ExprArg Callee, SourceLocation LParenLoc,
+  ExprResult RebuildCallExpr(Expr *Callee, SourceLocation LParenLoc,
                                    MultiExprArg Args,
                                    SourceLocation *CommaLocs,
                                    SourceLocation RParenLoc) {
-    return getSema().ActOnCallExpr(/*Scope=*/0, move(Callee), LParenLoc,
+    return getSema().ActOnCallExpr(/*Scope=*/0, Callee, LParenLoc,
                                    move(Args), CommaLocs, RParenLoc);
   }
 
@@ -1143,11 +1136,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildMemberExpr(ExprArg Base, SourceLocation OpLoc,
+  ExprResult RebuildMemberExpr(Expr *Base, SourceLocation OpLoc,
                                      bool isArrow,
                                      NestedNameSpecifier *Qualifier,
                                      SourceRange QualifierRange,
-                                     SourceLocation MemberLoc,
+                                     const DeclarationNameInfo &MemberNameInfo,
                                      ValueDecl *Member,
                                      NamedDecl *FoundDecl,
                         const TemplateArgumentListInfo *ExplicitTemplateArgs,
@@ -1156,14 +1149,13 @@ public:
       // We have a reference to an unnamed field.
       assert(!Qualifier && "Can't have an unnamed field with a qualifier!");
 
-      Expr *BaseExpr = Base.takeAs<Expr>();
-      if (getSema().PerformObjectMemberConversion(BaseExpr, Qualifier,
+      if (getSema().PerformObjectMemberConversion(Base, Qualifier,
                                                   FoundDecl, Member))
-        return getSema().ExprError();
+        return ExprError();
 
       MemberExpr *ME =
-        new (getSema().Context) MemberExpr(BaseExpr, isArrow,
-                                           Member, MemberLoc,
+        new (getSema().Context) MemberExpr(Base, isArrow,
+                                           Member, MemberNameInfo,
                                            cast<FieldDecl>(Member)->getType());
       return getSema().Owned(ME);
     }
@@ -1174,19 +1166,16 @@ public:
       SS.setScopeRep(Qualifier);
     }
 
-    Expr *BaseExpr = Base.takeAs<Expr>();
-    getSema().DefaultFunctionArrayConversion(BaseExpr);
-    QualType BaseType = BaseExpr->getType();
+    getSema().DefaultFunctionArrayConversion(Base);
+    QualType BaseType = Base->getType();
 
     // FIXME: this involves duplicating earlier analysis in a lot of
     // cases; we should avoid this when possible.
-    LookupResult R(getSema(), Member->getDeclName(), MemberLoc,
-                   Sema::LookupMemberName);
+    LookupResult R(getSema(), MemberNameInfo, Sema::LookupMemberName);
     R.addDecl(FoundDecl);
     R.resolveKind();
 
-    return getSema().BuildMemberReferenceExpr(getSema().Owned(BaseExpr),
-                                              BaseType, OpLoc, isArrow,
+    return getSema().BuildMemberReferenceExpr(Base, BaseType, OpLoc, isArrow,
                                               SS, FirstQualifierInScope,
                                               R, ExplicitTemplateArgs);
   }
@@ -1195,66 +1184,64 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildBinaryOperator(SourceLocation OpLoc,
-                                         BinaryOperator::Opcode Opc,
-                                         ExprArg LHS, ExprArg RHS) {
-    return getSema().BuildBinOp(/*Scope=*/0, OpLoc, Opc, 
-                                LHS.takeAs<Expr>(), RHS.takeAs<Expr>());
+  ExprResult RebuildBinaryOperator(SourceLocation OpLoc,
+                                         BinaryOperatorKind Opc,
+                                         Expr *LHS, Expr *RHS) {
+    return getSema().BuildBinOp(/*Scope=*/0, OpLoc, Opc, LHS, RHS);
   }
 
   /// \brief Build a new conditional operator expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildConditionalOperator(ExprArg Cond,
+  ExprResult RebuildConditionalOperator(Expr *Cond,
                                               SourceLocation QuestionLoc,
-                                              ExprArg LHS,
+                                              Expr *LHS,
                                               SourceLocation ColonLoc,
-                                              ExprArg RHS) {
-    return getSema().ActOnConditionalOp(QuestionLoc, ColonLoc, move(Cond),
-                                        move(LHS), move(RHS));
+                                              Expr *RHS) {
+    return getSema().ActOnConditionalOp(QuestionLoc, ColonLoc, Cond,
+                                        LHS, RHS);
   }
 
   /// \brief Build a new C-style cast expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCStyleCastExpr(SourceLocation LParenLoc,
+  ExprResult RebuildCStyleCastExpr(SourceLocation LParenLoc,
                                          TypeSourceInfo *TInfo,
                                          SourceLocation RParenLoc,
-                                         ExprArg SubExpr) {
+                                         Expr *SubExpr) {
     return getSema().BuildCStyleCastExpr(LParenLoc, TInfo, RParenLoc,
-                                         move(SubExpr));
+                                         SubExpr);
   }
 
   /// \brief Build a new compound literal expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCompoundLiteralExpr(SourceLocation LParenLoc,
+  ExprResult RebuildCompoundLiteralExpr(SourceLocation LParenLoc,
                                               TypeSourceInfo *TInfo,
                                               SourceLocation RParenLoc,
-                                              ExprArg Init) {
+                                              Expr *Init) {
     return getSema().BuildCompoundLiteralExpr(LParenLoc, TInfo, RParenLoc,
-                                              move(Init));
+                                              Init);
   }
 
   /// \brief Build a new extended vector element access expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildExtVectorElementExpr(ExprArg Base,
+  ExprResult RebuildExtVectorElementExpr(Expr *Base,
                                                SourceLocation OpLoc,
                                                SourceLocation AccessorLoc,
                                                IdentifierInfo &Accessor) {
 
     CXXScopeSpec SS;
-    QualType BaseType = ((Expr*) Base.get())->getType();
-    return getSema().BuildMemberReferenceExpr(move(Base), BaseType,
+    DeclarationNameInfo NameInfo(&Accessor, AccessorLoc);
+    return getSema().BuildMemberReferenceExpr(Base, Base->getType(),
                                               OpLoc, /*IsArrow*/ false,
                                               SS, /*FirstQualifierInScope*/ 0,
-                                              DeclarationName(&Accessor),
-                                              AccessorLoc,
+                                              NameInfo,
                                               /* TemplateArgs */ 0);
   }
 
@@ -1262,11 +1249,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildInitList(SourceLocation LBraceLoc,
+  ExprResult RebuildInitList(SourceLocation LBraceLoc,
                                    MultiExprArg Inits,
                                    SourceLocation RBraceLoc,
                                    QualType ResultTy) {
-    OwningExprResult Result
+    ExprResult Result
       = SemaRef.ActOnInitList(LBraceLoc, move(Inits), RBraceLoc);
     if (Result.isInvalid() || ResultTy->isDependentType())
       return move(Result);
@@ -1282,16 +1269,16 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildDesignatedInitExpr(Designation &Desig,
+  ExprResult RebuildDesignatedInitExpr(Designation &Desig,
                                              MultiExprArg ArrayExprs,
                                              SourceLocation EqualOrColonLoc,
                                              bool GNUSyntax,
-                                             ExprArg Init) {
-    OwningExprResult Result
+                                             Expr *Init) {
+    ExprResult Result
       = SemaRef.ActOnDesignatedInitializer(Desig, EqualOrColonLoc, GNUSyntax,
-                                           move(Init));
+                                           Init);
     if (Result.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArrayExprs.release();
     return move(Result);
@@ -1302,7 +1289,7 @@ public:
   /// By default, builds the implicit value initialization without performing
   /// any semantic analysis. Subclasses may override this routine to provide
   /// different behavior.
-  OwningExprResult RebuildImplicitValueInitExpr(QualType T) {
+  ExprResult RebuildImplicitValueInitExpr(QualType T) {
     return SemaRef.Owned(new (SemaRef.Context) ImplicitValueInitExpr(T));
   }
 
@@ -1310,17 +1297,19 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildVAArgExpr(SourceLocation BuiltinLoc, ExprArg SubExpr,
-                                    QualType T, SourceLocation RParenLoc) {
-    return getSema().ActOnVAArg(BuiltinLoc, move(SubExpr), T.getAsOpaquePtr(),
-                                RParenLoc);
+  ExprResult RebuildVAArgExpr(SourceLocation BuiltinLoc,
+                                    Expr *SubExpr, TypeSourceInfo *TInfo,
+                                    SourceLocation RParenLoc) {
+    return getSema().BuildVAArgExpr(BuiltinLoc,
+                                    SubExpr, TInfo,
+                                    RParenLoc);
   }
 
   /// \brief Build a new expression list in parentheses.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildParenListExpr(SourceLocation LParenLoc,
+  ExprResult RebuildParenListExpr(SourceLocation LParenLoc,
                                         MultiExprArg SubExprs,
                                         SourceLocation RParenLoc) {
     return getSema().ActOnParenOrParenListExpr(LParenLoc, RParenLoc, 
@@ -1332,7 +1321,7 @@ public:
   /// By default, performs semantic analysis, using the name of the label
   /// rather than attempting to map the label statement itself.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildAddrLabelExpr(SourceLocation AmpAmpLoc,
+  ExprResult RebuildAddrLabelExpr(SourceLocation AmpAmpLoc,
                                         SourceLocation LabelLoc,
                                         LabelStmt *Label) {
     return getSema().ActOnAddrLabel(AmpAmpLoc, LabelLoc, Label->getID());
@@ -1342,22 +1331,22 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildStmtExpr(SourceLocation LParenLoc,
-                                   StmtArg SubStmt,
+  ExprResult RebuildStmtExpr(SourceLocation LParenLoc,
+                                   Stmt *SubStmt,
                                    SourceLocation RParenLoc) {
-    return getSema().ActOnStmtExpr(LParenLoc, move(SubStmt), RParenLoc);
+    return getSema().ActOnStmtExpr(LParenLoc, SubStmt, RParenLoc);
   }
 
   /// \brief Build a new __builtin_types_compatible_p expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildTypesCompatibleExpr(SourceLocation BuiltinLoc,
-                                              QualType T1, QualType T2,
+  ExprResult RebuildTypesCompatibleExpr(SourceLocation BuiltinLoc,
+                                              TypeSourceInfo *TInfo1,
+                                              TypeSourceInfo *TInfo2,
                                               SourceLocation RParenLoc) {
-    return getSema().ActOnTypesCompatibleExpr(BuiltinLoc,
-                                              T1.getAsOpaquePtr(),
-                                              T2.getAsOpaquePtr(),
+    return getSema().BuildTypesCompatibleExpr(BuiltinLoc,
+                                              TInfo1, TInfo2,
                                               RParenLoc);
   }
 
@@ -1365,11 +1354,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildChooseExpr(SourceLocation BuiltinLoc,
-                                     ExprArg Cond, ExprArg LHS, ExprArg RHS,
+  ExprResult RebuildChooseExpr(SourceLocation BuiltinLoc,
+                                     Expr *Cond, Expr *LHS, Expr *RHS,
                                      SourceLocation RParenLoc) {
     return SemaRef.ActOnChooseExpr(BuiltinLoc,
-                                   move(Cond), move(LHS), move(RHS),
+                                   Cond, LHS, RHS,
                                    RParenLoc);
   }
 
@@ -1381,11 +1370,11 @@ public:
   /// operator call into a use of a builtin operator, performing
   /// argument-dependent lookup, etc. Subclasses may override this routine to
   /// provide different behavior.
-  OwningExprResult RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
+  ExprResult RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
                                               SourceLocation OpLoc,
-                                              ExprArg Callee,
-                                              ExprArg First,
-                                              ExprArg Second);
+                                              Expr *Callee,
+                                              Expr *First,
+                                              Expr *Second);
 
   /// \brief Build a new C++ "named" cast expression, such as static_cast or
   /// reinterpret_cast.
@@ -1393,57 +1382,57 @@ public:
   /// By default, this routine dispatches to one of the more-specific routines
   /// for a particular named case, e.g., RebuildCXXStaticCastExpr().
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXNamedCastExpr(SourceLocation OpLoc,
+  ExprResult RebuildCXXNamedCastExpr(SourceLocation OpLoc,
                                            Stmt::StmtClass Class,
                                            SourceLocation LAngleLoc,
                                            TypeSourceInfo *TInfo,
                                            SourceLocation RAngleLoc,
                                            SourceLocation LParenLoc,
-                                           ExprArg SubExpr,
+                                           Expr *SubExpr,
                                            SourceLocation RParenLoc) {
     switch (Class) {
     case Stmt::CXXStaticCastExprClass:
       return getDerived().RebuildCXXStaticCastExpr(OpLoc, LAngleLoc, TInfo,
                                                    RAngleLoc, LParenLoc,
-                                                   move(SubExpr), RParenLoc);
+                                                   SubExpr, RParenLoc);
 
     case Stmt::CXXDynamicCastExprClass:
       return getDerived().RebuildCXXDynamicCastExpr(OpLoc, LAngleLoc, TInfo,
                                                     RAngleLoc, LParenLoc,
-                                                    move(SubExpr), RParenLoc);
+                                                    SubExpr, RParenLoc);
 
     case Stmt::CXXReinterpretCastExprClass:
       return getDerived().RebuildCXXReinterpretCastExpr(OpLoc, LAngleLoc, TInfo,
                                                         RAngleLoc, LParenLoc,
-                                                        move(SubExpr),
+                                                        SubExpr,
                                                         RParenLoc);
 
     case Stmt::CXXConstCastExprClass:
       return getDerived().RebuildCXXConstCastExpr(OpLoc, LAngleLoc, TInfo,
                                                    RAngleLoc, LParenLoc,
-                                                   move(SubExpr), RParenLoc);
+                                                   SubExpr, RParenLoc);
 
     default:
       assert(false && "Invalid C++ named cast");
       break;
     }
 
-    return getSema().ExprError();
+    return ExprError();
   }
 
   /// \brief Build a new C++ static_cast expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXStaticCastExpr(SourceLocation OpLoc,
+  ExprResult RebuildCXXStaticCastExpr(SourceLocation OpLoc,
                                             SourceLocation LAngleLoc,
                                             TypeSourceInfo *TInfo,
                                             SourceLocation RAngleLoc,
                                             SourceLocation LParenLoc,
-                                            ExprArg SubExpr,
+                                            Expr *SubExpr,
                                             SourceLocation RParenLoc) {
     return getSema().BuildCXXNamedCast(OpLoc, tok::kw_static_cast,
-                                       TInfo, move(SubExpr),
+                                       TInfo, SubExpr,
                                        SourceRange(LAngleLoc, RAngleLoc),
                                        SourceRange(LParenLoc, RParenLoc));
   }
@@ -1452,15 +1441,15 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXDynamicCastExpr(SourceLocation OpLoc,
+  ExprResult RebuildCXXDynamicCastExpr(SourceLocation OpLoc,
                                              SourceLocation LAngleLoc,
                                              TypeSourceInfo *TInfo,
                                              SourceLocation RAngleLoc,
                                              SourceLocation LParenLoc,
-                                             ExprArg SubExpr,
+                                             Expr *SubExpr,
                                              SourceLocation RParenLoc) {
     return getSema().BuildCXXNamedCast(OpLoc, tok::kw_dynamic_cast,
-                                       TInfo, move(SubExpr),
+                                       TInfo, SubExpr,
                                        SourceRange(LAngleLoc, RAngleLoc),
                                        SourceRange(LParenLoc, RParenLoc));
   }
@@ -1469,15 +1458,15 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXReinterpretCastExpr(SourceLocation OpLoc,
+  ExprResult RebuildCXXReinterpretCastExpr(SourceLocation OpLoc,
                                                  SourceLocation LAngleLoc,
                                                  TypeSourceInfo *TInfo,
                                                  SourceLocation RAngleLoc,
                                                  SourceLocation LParenLoc,
-                                                 ExprArg SubExpr,
+                                                 Expr *SubExpr,
                                                  SourceLocation RParenLoc) {
     return getSema().BuildCXXNamedCast(OpLoc, tok::kw_reinterpret_cast,
-                                       TInfo, move(SubExpr),
+                                       TInfo, SubExpr,
                                        SourceRange(LAngleLoc, RAngleLoc),
                                        SourceRange(LParenLoc, RParenLoc));
   }
@@ -1486,15 +1475,15 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXConstCastExpr(SourceLocation OpLoc,
+  ExprResult RebuildCXXConstCastExpr(SourceLocation OpLoc,
                                            SourceLocation LAngleLoc,
                                            TypeSourceInfo *TInfo,
                                            SourceLocation RAngleLoc,
                                            SourceLocation LParenLoc,
-                                           ExprArg SubExpr,
+                                           Expr *SubExpr,
                                            SourceLocation RParenLoc) {
     return getSema().BuildCXXNamedCast(OpLoc, tok::kw_const_cast,
-                                       TInfo, move(SubExpr),
+                                       TInfo, SubExpr,
                                        SourceRange(LAngleLoc, RAngleLoc),
                                        SourceRange(LParenLoc, RParenLoc));
   }
@@ -1503,16 +1492,15 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXFunctionalCastExpr(SourceRange TypeRange,
+  ExprResult RebuildCXXFunctionalCastExpr(SourceRange TypeRange,
                                                 TypeSourceInfo *TInfo,
                                                 SourceLocation LParenLoc,
-                                                ExprArg SubExpr,
+                                                Expr *Sub,
                                                 SourceLocation RParenLoc) {
-    void *Sub = SubExpr.takeAs<Expr>();
     return getSema().ActOnCXXTypeConstructExpr(TypeRange,
-                                               TInfo->getType().getAsOpaquePtr(),
+                                             ParsedType::make(TInfo->getType()),
                                                LParenLoc,
-                                         Sema::MultiExprArg(getSema(), &Sub, 1),
+                                               MultiExprArg(&Sub, 1),
                                                /*CommaLocs=*/0,
                                                RParenLoc);
   }
@@ -1521,7 +1509,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXTypeidExpr(QualType TypeInfoType,
+  ExprResult RebuildCXXTypeidExpr(QualType TypeInfoType,
                                         SourceLocation TypeidLoc,
                                         TypeSourceInfo *Operand,
                                         SourceLocation RParenLoc) {
@@ -1533,11 +1521,11 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXTypeidExpr(QualType TypeInfoType,
+  ExprResult RebuildCXXTypeidExpr(QualType TypeInfoType,
                                         SourceLocation TypeidLoc,
-                                        ExprArg Operand,
+                                        Expr *Operand,
                                         SourceLocation RParenLoc) {
-    return getSema().BuildCXXTypeId(TypeInfoType, TypeidLoc, move(Operand),
+    return getSema().BuildCXXTypeId(TypeInfoType, TypeidLoc, Operand,
                                     RParenLoc);
   }
 
@@ -1546,7 +1534,7 @@ public:
   /// By default, builds a new "this" expression without performing any
   /// semantic analysis. Subclasses may override this routine to provide
   /// different behavior.
-  OwningExprResult RebuildCXXThisExpr(SourceLocation ThisLoc,
+  ExprResult RebuildCXXThisExpr(SourceLocation ThisLoc,
                                       QualType ThisType,
                                       bool isImplicit) {
     return getSema().Owned(
@@ -1558,8 +1546,8 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXThrowExpr(SourceLocation ThrowLoc, ExprArg Sub) {
-    return getSema().ActOnCXXThrow(ThrowLoc, move(Sub));
+  ExprResult RebuildCXXThrowExpr(SourceLocation ThrowLoc, Expr *Sub) {
+    return getSema().ActOnCXXThrow(ThrowLoc, Sub);
   }
 
   /// \brief Build a new C++ default-argument expression.
@@ -1567,7 +1555,7 @@ public:
   /// By default, builds a new default-argument expression, which does not
   /// require any semantic analysis. Subclasses may override this routine to
   /// provide different behavior.
-  OwningExprResult RebuildCXXDefaultArgExpr(SourceLocation Loc, 
+  ExprResult RebuildCXXDefaultArgExpr(SourceLocation Loc, 
                                             ParmVarDecl *Param) {
     return getSema().Owned(CXXDefaultArgExpr::Create(getSema().Context, Loc,
                                                      Param));
@@ -1577,12 +1565,12 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXScalarValueInitExpr(SourceLocation TypeStartLoc,
+  ExprResult RebuildCXXScalarValueInitExpr(SourceLocation TypeStartLoc,
                                                SourceLocation LParenLoc,
                                                QualType T,
                                                SourceLocation RParenLoc) {
     return getSema().ActOnCXXTypeConstructExpr(SourceRange(TypeStartLoc),
-                                               T.getAsOpaquePtr(), LParenLoc,
+                                               ParsedType::make(T), LParenLoc,
                                                MultiExprArg(getSema(), 0, 0),
                                                0, RParenLoc);
   }
@@ -1591,7 +1579,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXNewExpr(SourceLocation StartLoc,
+  ExprResult RebuildCXXNewExpr(SourceLocation StartLoc,
                                      bool UseGlobal,
                                      SourceLocation PlacementLParen,
                                      MultiExprArg PlacementArgs,
@@ -1600,7 +1588,7 @@ public:
                                      QualType AllocType,
                                      SourceLocation TypeLoc,
                                      SourceRange TypeRange,
-                                     ExprArg ArraySize,
+                                     Expr *ArraySize,
                                      SourceLocation ConstructorLParen,
                                      MultiExprArg ConstructorArgs,
                                      SourceLocation ConstructorRParen) {
@@ -1612,7 +1600,7 @@ public:
                                  AllocType,
                                  TypeLoc,
                                  TypeRange,
-                                 move(ArraySize),
+                                 ArraySize,
                                  ConstructorLParen,
                                  move(ConstructorArgs),
                                  ConstructorRParen);
@@ -1622,25 +1610,25 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXDeleteExpr(SourceLocation StartLoc,
+  ExprResult RebuildCXXDeleteExpr(SourceLocation StartLoc,
                                         bool IsGlobalDelete,
                                         bool IsArrayForm,
-                                        ExprArg Operand) {
+                                        Expr *Operand) {
     return getSema().ActOnCXXDelete(StartLoc, IsGlobalDelete, IsArrayForm,
-                                    move(Operand));
+                                    Operand);
   }
 
   /// \brief Build a new unary type trait expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildUnaryTypeTrait(UnaryTypeTrait Trait,
+  ExprResult RebuildUnaryTypeTrait(UnaryTypeTrait Trait,
                                          SourceLocation StartLoc,
                                          SourceLocation LParenLoc,
                                          QualType T,
                                          SourceLocation RParenLoc) {
     return getSema().ActOnUnaryTypeTrait(Trait, StartLoc, LParenLoc,
-                                         T.getAsOpaquePtr(), RParenLoc);
+                                         ParsedType::make(T), RParenLoc);
   }
 
   /// \brief Build a new (previously unresolved) declaration reference
@@ -1648,27 +1636,26 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildDependentScopeDeclRefExpr(NestedNameSpecifier *NNS,
+  ExprResult RebuildDependentScopeDeclRefExpr(NestedNameSpecifier *NNS,
                                                 SourceRange QualifierRange,
-                                                DeclarationName Name,
-                                                SourceLocation Location,
+                                       const DeclarationNameInfo &NameInfo,
                               const TemplateArgumentListInfo *TemplateArgs) {
     CXXScopeSpec SS;
     SS.setRange(QualifierRange);
     SS.setScopeRep(NNS);
 
     if (TemplateArgs)
-      return getSema().BuildQualifiedTemplateIdExpr(SS, Name, Location,
+      return getSema().BuildQualifiedTemplateIdExpr(SS, NameInfo,
                                                     *TemplateArgs);
 
-    return getSema().BuildQualifiedDeclarationNameExpr(SS, Name, Location);
+    return getSema().BuildQualifiedDeclarationNameExpr(SS, NameInfo);
   }
 
   /// \brief Build a new template-id expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildTemplateIdExpr(const CXXScopeSpec &SS,
+  ExprResult RebuildTemplateIdExpr(const CXXScopeSpec &SS,
                                          LookupResult &R,
                                          bool RequiresADL,
                               const TemplateArgumentListInfo &TemplateArgs) {
@@ -1679,32 +1666,35 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXConstructExpr(QualType T,
+  ExprResult RebuildCXXConstructExpr(QualType T,
                                            SourceLocation Loc,
                                            CXXConstructorDecl *Constructor,
                                            bool IsElidable,
-                                           MultiExprArg Args) {
-    ASTOwningVector<&ActionBase::DeleteExpr> ConvertedArgs(SemaRef);
+                                           MultiExprArg Args,
+                                           bool RequiresZeroInit,
+                             CXXConstructExpr::ConstructionKind ConstructKind) {
+    ASTOwningVector<Expr*> ConvertedArgs(SemaRef);
     if (getSema().CompleteConstructorCall(Constructor, move(Args), Loc, 
                                           ConvertedArgs))
-      return getSema().ExprError();
+      return ExprError();
     
     return getSema().BuildCXXConstructExpr(Loc, T, Constructor, IsElidable,
-                                           move_arg(ConvertedArgs));
+                                           move_arg(ConvertedArgs),
+                                           RequiresZeroInit, ConstructKind);
   }
 
   /// \brief Build a new object-construction expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXTemporaryObjectExpr(SourceLocation TypeBeginLoc,
+  ExprResult RebuildCXXTemporaryObjectExpr(SourceLocation TypeBeginLoc,
                                                  QualType T,
                                                  SourceLocation LParenLoc,
                                                  MultiExprArg Args,
                                                  SourceLocation *Commas,
                                                  SourceLocation RParenLoc) {
     return getSema().ActOnCXXTypeConstructExpr(SourceRange(TypeBeginLoc),
-                                               T.getAsOpaquePtr(),
+                                               ParsedType::make(T),
                                                LParenLoc,
                                                move(Args),
                                                Commas,
@@ -1715,7 +1705,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXUnresolvedConstructExpr(SourceLocation TypeBeginLoc,
+  ExprResult RebuildCXXUnresolvedConstructExpr(SourceLocation TypeBeginLoc,
                                                      QualType T,
                                                      SourceLocation LParenLoc,
                                                      MultiExprArg Args,
@@ -1723,7 +1713,7 @@ public:
                                                      SourceLocation RParenLoc) {
     return getSema().ActOnCXXTypeConstructExpr(SourceRange(TypeBeginLoc,
                                                            /*FIXME*/LParenLoc),
-                                               T.getAsOpaquePtr(),
+                                               ParsedType::make(T),
                                                LParenLoc,
                                                move(Args),
                                                Commas,
@@ -1734,31 +1724,31 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildCXXDependentScopeMemberExpr(ExprArg BaseE,
+  ExprResult RebuildCXXDependentScopeMemberExpr(Expr *BaseE,
                                                   QualType BaseType,
                                                   bool IsArrow,
                                                   SourceLocation OperatorLoc,
                                               NestedNameSpecifier *Qualifier,
                                                   SourceRange QualifierRange,
                                             NamedDecl *FirstQualifierInScope,
-                                                  DeclarationName Name,
-                                                  SourceLocation MemberLoc,
+                                   const DeclarationNameInfo &MemberNameInfo,
                               const TemplateArgumentListInfo *TemplateArgs) {
     CXXScopeSpec SS;
     SS.setRange(QualifierRange);
     SS.setScopeRep(Qualifier);
 
-    return SemaRef.BuildMemberReferenceExpr(move(BaseE), BaseType,
+    return SemaRef.BuildMemberReferenceExpr(BaseE, BaseType,
                                             OperatorLoc, IsArrow,
                                             SS, FirstQualifierInScope,
-                                            Name, MemberLoc, TemplateArgs);
+                                            MemberNameInfo,
+                                            TemplateArgs);
   }
 
   /// \brief Build a new member reference expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildUnresolvedMemberExpr(ExprArg BaseE,
+  ExprResult RebuildUnresolvedMemberExpr(Expr *BaseE,
                                                QualType BaseType,
                                                SourceLocation OperatorLoc,
                                                bool IsArrow,
@@ -1771,7 +1761,7 @@ public:
     SS.setRange(QualifierRange);
     SS.setScopeRep(Qualifier);
 
-    return SemaRef.BuildMemberReferenceExpr(move(BaseE), BaseType,
+    return SemaRef.BuildMemberReferenceExpr(BaseE, BaseType,
                                             OperatorLoc, IsArrow,
                                             SS, FirstQualifierInScope,
                                             R, TemplateArgs);
@@ -1781,7 +1771,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildObjCEncodeExpr(SourceLocation AtLoc,
+  ExprResult RebuildObjCEncodeExpr(SourceLocation AtLoc,
                                          TypeSourceInfo *EncodeTypeInfo,
                                          SourceLocation RParenLoc) {
     return SemaRef.Owned(SemaRef.BuildObjCEncodeExpression(AtLoc, EncodeTypeInfo,
@@ -1789,7 +1779,7 @@ public:
   }
 
   /// \brief Build a new Objective-C class message.
-  OwningExprResult RebuildObjCMessageExpr(TypeSourceInfo *ReceiverTypeInfo,
+  ExprResult RebuildObjCMessageExpr(TypeSourceInfo *ReceiverTypeInfo,
                                           Selector Sel,
                                           ObjCMethodDecl *Method,
                                           SourceLocation LBracLoc, 
@@ -1803,15 +1793,14 @@ public:
   }
 
   /// \brief Build a new Objective-C instance message.
-  OwningExprResult RebuildObjCMessageExpr(ExprArg Receiver,
+  ExprResult RebuildObjCMessageExpr(Expr *Receiver,
                                           Selector Sel,
                                           ObjCMethodDecl *Method,
                                           SourceLocation LBracLoc, 
                                           MultiExprArg Args,
                                           SourceLocation RBracLoc) {
-    QualType ReceiverType = static_cast<Expr *>(Receiver.get())->getType();
-    return SemaRef.BuildInstanceMessage(move(Receiver),
-                                        ReceiverType,
+    return SemaRef.BuildInstanceMessage(Receiver,
+                                        Receiver->getType(),
                                         /*SuperLoc=*/SourceLocation(),
                                         Sel, Method, LBracLoc, RBracLoc,
                                         move(Args));
@@ -1821,26 +1810,25 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildObjCIvarRefExpr(ExprArg BaseArg, ObjCIvarDecl *Ivar,
+  ExprResult RebuildObjCIvarRefExpr(Expr *BaseArg, ObjCIvarDecl *Ivar,
                                           SourceLocation IvarLoc,
                                           bool IsArrow, bool IsFreeIvar) {
     // FIXME: We lose track of the IsFreeIvar bit.
     CXXScopeSpec SS;
-    Expr *Base = BaseArg.takeAs<Expr>();
+    Expr *Base = BaseArg;
     LookupResult R(getSema(), Ivar->getDeclName(), IvarLoc,
                    Sema::LookupMemberName);
-    OwningExprResult Result = getSema().LookupMemberExpr(R, Base, IsArrow,
+    ExprResult Result = getSema().LookupMemberExpr(R, Base, IsArrow,
                                                          /*FIME:*/IvarLoc,
-                                                         SS, DeclPtrTy(),
+                                                         SS, 0,
                                                          false);
     if (Result.isInvalid())
-      return getSema().ExprError();
+      return ExprError();
     
     if (Result.get())
       return move(Result);
     
-    return getSema().BuildMemberReferenceExpr(getSema().Owned(Base), 
-                                              Base->getType(),
+    return getSema().BuildMemberReferenceExpr(Base, Base->getType(),
                                               /*FIXME:*/IvarLoc, IsArrow, SS, 
                                               /*FirstQualifierInScope=*/0,
                                               R, 
@@ -1851,26 +1839,24 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildObjCPropertyRefExpr(ExprArg BaseArg, 
+  ExprResult RebuildObjCPropertyRefExpr(Expr *BaseArg, 
                                               ObjCPropertyDecl *Property,
                                               SourceLocation PropertyLoc) {
     CXXScopeSpec SS;
-    Expr *Base = BaseArg.takeAs<Expr>();
+    Expr *Base = BaseArg;
     LookupResult R(getSema(), Property->getDeclName(), PropertyLoc,
                    Sema::LookupMemberName);
     bool IsArrow = false;
-    OwningExprResult Result = getSema().LookupMemberExpr(R, Base, IsArrow,
+    ExprResult Result = getSema().LookupMemberExpr(R, Base, IsArrow,
                                                          /*FIME:*/PropertyLoc,
-                                                         SS, DeclPtrTy(),
-                                                         false);
+                                                         SS, 0, false);
     if (Result.isInvalid())
-      return getSema().ExprError();
+      return ExprError();
     
     if (Result.get())
       return move(Result);
     
-    return getSema().BuildMemberReferenceExpr(getSema().Owned(Base), 
-                                              Base->getType(),
+    return getSema().BuildMemberReferenceExpr(Base, Base->getType(),
                                               /*FIXME:*/PropertyLoc, IsArrow, 
                                               SS, 
                                               /*FirstQualifierInScope=*/0,
@@ -1883,43 +1869,41 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.  
-  OwningExprResult RebuildObjCImplicitSetterGetterRefExpr(
+  ExprResult RebuildObjCImplicitSetterGetterRefExpr(
                                                         ObjCMethodDecl *Getter,
                                                           QualType T,
                                                         ObjCMethodDecl *Setter,
                                                         SourceLocation NameLoc,
-                                                          ExprArg Base) {
+                                                          Expr *Base) {
     // Since these expressions can only be value-dependent, we do not need to
     // perform semantic analysis again.
-    return getSema().Owned(
+    return Owned(
              new (getSema().Context) ObjCImplicitSetterGetterRefExpr(Getter, T,
                                                                      Setter,
                                                                      NameLoc,
-                                                          Base.takeAs<Expr>()));
+                                                                     Base));
   }
 
   /// \brief Build a new Objective-C "isa" expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildObjCIsaExpr(ExprArg BaseArg, SourceLocation IsaLoc,
+  ExprResult RebuildObjCIsaExpr(Expr *BaseArg, SourceLocation IsaLoc,
                                       bool IsArrow) {
     CXXScopeSpec SS;
-    Expr *Base = BaseArg.takeAs<Expr>();
+    Expr *Base = BaseArg;
     LookupResult R(getSema(), &getSema().Context.Idents.get("isa"), IsaLoc,
                    Sema::LookupMemberName);
-    OwningExprResult Result = getSema().LookupMemberExpr(R, Base, IsArrow,
+    ExprResult Result = getSema().LookupMemberExpr(R, Base, IsArrow,
                                                          /*FIME:*/IsaLoc,
-                                                         SS, DeclPtrTy(),
-                                                         false);
+                                                         SS, 0, false);
     if (Result.isInvalid())
-      return getSema().ExprError();
+      return ExprError();
     
     if (Result.get())
       return move(Result);
     
-    return getSema().BuildMemberReferenceExpr(getSema().Owned(Base), 
-                                              Base->getType(),
+    return getSema().BuildMemberReferenceExpr(Base, Base->getType(),
                                               /*FIXME:*/IsaLoc, IsArrow, SS, 
                                               /*FirstQualifierInScope=*/0,
                                               R, 
@@ -1930,7 +1914,7 @@ public:
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  OwningExprResult RebuildShuffleVectorExpr(SourceLocation BuiltinLoc,
+  ExprResult RebuildShuffleVectorExpr(SourceLocation BuiltinLoc,
                                             MultiExprArg SubExprs,
                                             SourceLocation RParenLoc) {
     // Find the declaration for __builtin_shufflevector
@@ -1954,12 +1938,12 @@ public:
                                                        Subs, NumSubExprs,
                                                    Builtin->getCallResultType(),
                                                        RParenLoc);
-    OwningExprResult OwnedCall(SemaRef.Owned(TheCall));
+    ExprResult OwnedCall(SemaRef.Owned(TheCall));
 
     // Type-check the __builtin_shufflevector expression.
-    OwningExprResult Result = SemaRef.SemaBuiltinShuffleVector(TheCall);
+    ExprResult Result = SemaRef.SemaBuiltinShuffleVector(TheCall);
     if (Result.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     OwnedCall.release();
     return move(Result);
@@ -1967,7 +1951,7 @@ public:
 };
 
 template<typename Derived>
-Sema::OwningStmtResult TreeTransform<Derived>::TransformStmt(Stmt *S) {
+StmtResult TreeTransform<Derived>::TransformStmt(Stmt *S) {
   if (!S)
     return SemaRef.Owned(S);
 
@@ -1986,11 +1970,11 @@ Sema::OwningStmtResult TreeTransform<Derived>::TransformStmt(Stmt *S) {
 #define EXPR(Node, Parent) case Stmt::Node##Class:
 #include "clang/AST/StmtNodes.inc"
     {
-      Sema::OwningExprResult E = getDerived().TransformExpr(cast<Expr>(S));
+      ExprResult E = getDerived().TransformExpr(cast<Expr>(S));
       if (E.isInvalid())
-        return getSema().StmtError();
+        return StmtError();
 
-      return getSema().ActOnExprStmt(getSema().MakeFullExpr(E));
+      return getSema().ActOnExprStmt(getSema().MakeFullExpr(E.take()));
     }
   }
 
@@ -1999,7 +1983,7 @@ Sema::OwningStmtResult TreeTransform<Derived>::TransformStmt(Stmt *S) {
 
 
 template<typename Derived>
-Sema::OwningExprResult TreeTransform<Derived>::TransformExpr(Expr *E) {
+ExprResult TreeTransform<Derived>::TransformExpr(Expr *E) {
   if (!E)
     return SemaRef.Owned(E);
 
@@ -2094,12 +2078,13 @@ TreeTransform<Derived>::TransformNestedNameSpecifier(NestedNameSpecifier *NNS,
 }
 
 template<typename Derived>
-DeclarationName
-TreeTransform<Derived>::TransformDeclarationName(DeclarationName Name,
-                                                 SourceLocation Loc,
-                                                 QualType ObjectType) {
+DeclarationNameInfo
+TreeTransform<Derived>
+::TransformDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
+                               QualType ObjectType) {
+  DeclarationName Name = NameInfo.getName();
   if (!Name)
-    return Name;
+    return DeclarationNameInfo();
 
   switch (Name.getNameKind()) {
   case DeclarationName::Identifier:
@@ -2109,24 +2094,41 @@ TreeTransform<Derived>::TransformDeclarationName(DeclarationName Name,
   case DeclarationName::CXXOperatorName:
   case DeclarationName::CXXLiteralOperatorName:
   case DeclarationName::CXXUsingDirective:
-    return Name;
+    return NameInfo;
 
   case DeclarationName::CXXConstructorName:
   case DeclarationName::CXXDestructorName:
   case DeclarationName::CXXConversionFunctionName: {
-    TemporaryBase Rebase(*this, Loc, Name);
-    QualType T = getDerived().TransformType(Name.getCXXNameType(), 
-                                            ObjectType);
-    if (T.isNull())
-      return DeclarationName();
+    TypeSourceInfo *NewTInfo;
+    CanQualType NewCanTy;
+    if (TypeSourceInfo *OldTInfo = NameInfo.getNamedTypeInfo()) {
+       NewTInfo = getDerived().TransformType(OldTInfo, ObjectType);
+       if (!NewTInfo)
+         return DeclarationNameInfo();
+       NewCanTy = SemaRef.Context.getCanonicalType(NewTInfo->getType());
+    }
+    else {
+      NewTInfo = 0;
+      TemporaryBase Rebase(*this, NameInfo.getLoc(), Name);
+      QualType NewT = getDerived().TransformType(Name.getCXXNameType(),
+                                                 ObjectType);
+      if (NewT.isNull())
+        return DeclarationNameInfo();
+      NewCanTy = SemaRef.Context.getCanonicalType(NewT);
+    }
 
-    return SemaRef.Context.DeclarationNames.getCXXSpecialName(
-                                                           Name.getNameKind(),
-                                          SemaRef.Context.getCanonicalType(T));
+    DeclarationName NewName
+      = SemaRef.Context.DeclarationNames.getCXXSpecialName(Name.getNameKind(),
+                                                           NewCanTy);
+    DeclarationNameInfo NewNameInfo(NameInfo);
+    NewNameInfo.setName(NewName);
+    NewNameInfo.setNamedTypeInfo(NewTInfo);
+    return NewNameInfo;
   }
   }
 
-  return DeclarationName();
+  assert(0 && "Unknown name kind.");
+  return DeclarationNameInfo();
 }
 
 template<typename Derived>
@@ -2268,14 +2270,9 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
     Expr *SourceExpr = Input.getSourceDeclExpression();
     if (SourceExpr) {
       EnterExpressionEvaluationContext Unevaluated(getSema(),
-                                                   Action::Unevaluated);
-      Sema::OwningExprResult E = getDerived().TransformExpr(SourceExpr);
-      if (E.isInvalid())
-        SourceExpr = NULL;
-      else {
-        SourceExpr = E.takeAs<Expr>();
-        SourceExpr->Retain();
-      }
+                                                   Sema::Unevaluated);
+      ExprResult E = getDerived().TransformExpr(SourceExpr);
+      SourceExpr = (E.isInvalid() ? 0 : E.take());
     }
 
     Output = TemplateArgumentLoc(TemplateArgument(D), SourceExpr);
@@ -2298,18 +2295,15 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
   case TemplateArgument::Expression: {
     // Template argument expressions are not potentially evaluated.
     EnterExpressionEvaluationContext Unevaluated(getSema(),
-                                                 Action::Unevaluated);
+                                                 Sema::Unevaluated);
 
     Expr *InputExpr = Input.getSourceExpression();
     if (!InputExpr) InputExpr = Input.getArgument().getAsExpr();
 
-    Sema::OwningExprResult E
+    ExprResult E
       = getDerived().TransformExpr(InputExpr);
     if (E.isInvalid()) return true;
-
-    Expr *ETaken = E.takeAs<Expr>();
-    ETaken->Retain();
-    Output = TemplateArgumentLoc(TemplateArgument(ETaken), ETaken);
+    Output = TemplateArgumentLoc(TemplateArgument(E.take()), E.take());
     return false;
   }
 
@@ -2631,7 +2625,7 @@ TreeTransform<Derived>::TransformConstantArrayType(TypeLocBuilder &TLB,
 
   Expr *Size = TL.getSizeExpr();
   if (Size) {
-    EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+    EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
     Size = getDerived().TransformExpr(Size).template takeAs<Expr>();
   }
   NewTL.setSizeExpr(Size);
@@ -2679,14 +2673,14 @@ TreeTransform<Derived>::TransformVariableArrayType(TypeLocBuilder &TLB,
     return QualType();
 
   // Array bounds are not potentially evaluated contexts
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  Sema::OwningExprResult SizeResult
+  ExprResult SizeResult
     = getDerived().TransformExpr(T->getSizeExpr());
   if (SizeResult.isInvalid())
     return QualType();
 
-  Expr *Size = static_cast<Expr*>(SizeResult.get());
+  Expr *Size = SizeResult.take();
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
@@ -2694,13 +2688,12 @@ TreeTransform<Derived>::TransformVariableArrayType(TypeLocBuilder &TLB,
       Size != T->getSizeExpr()) {
     Result = getDerived().RebuildVariableArrayType(ElementType,
                                                    T->getSizeModifier(),
-                                                   move(SizeResult),
+                                                   Size,
                                              T->getIndexTypeCVRQualifiers(),
                                                    TL.getBracketsRange());
     if (Result.isNull())
       return QualType();
   }
-  else SizeResult.take();
   
   VariableArrayTypeLoc NewTL = TLB.push<VariableArrayTypeLoc>(Result);
   NewTL.setLBracketLoc(TL.getLBracketLoc());
@@ -2721,9 +2714,9 @@ TreeTransform<Derived>::TransformDependentSizedArrayType(TypeLocBuilder &TLB,
     return QualType();
 
   // Array bounds are not potentially evaluated contexts
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  Sema::OwningExprResult SizeResult
+  ExprResult SizeResult
     = getDerived().TransformExpr(T->getSizeExpr());
   if (SizeResult.isInvalid())
     return QualType();
@@ -2736,7 +2729,7 @@ TreeTransform<Derived>::TransformDependentSizedArrayType(TypeLocBuilder &TLB,
       Size != T->getSizeExpr()) {
     Result = getDerived().RebuildDependentSizedArrayType(ElementType,
                                                          T->getSizeModifier(),
-                                                         move(SizeResult),
+                                                         Size,
                                                 T->getIndexTypeCVRQualifiers(),
                                                         TL.getBracketsRange());
     if (Result.isNull())
@@ -2767,9 +2760,9 @@ QualType TreeTransform<Derived>::TransformDependentSizedExtVectorType(
     return QualType();
 
   // Vector sizes are not potentially evaluated contexts
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  Sema::OwningExprResult Size = getDerived().TransformExpr(T->getSizeExpr());
+  ExprResult Size = getDerived().TransformExpr(T->getSizeExpr());
   if (Size.isInvalid())
     return QualType();
 
@@ -2778,12 +2771,11 @@ QualType TreeTransform<Derived>::TransformDependentSizedExtVectorType(
       ElementType != T->getElementType() ||
       Size.get() != T->getSizeExpr()) {
     Result = getDerived().RebuildDependentSizedExtVectorType(ElementType,
-                                                         move(Size),
+                                                             Size.take(),
                                                          T->getAttributeLoc());
     if (Result.isNull())
       return QualType();
   }
-  else Size.take();
 
   // Result might be dependent or not.
   if (isa<DependentSizedExtVectorType>(Result)) {
@@ -2911,18 +2903,25 @@ QualType
 TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
                                                    FunctionProtoTypeLoc TL,
                                                    QualType ObjectType) {
-  // Transform the parameters. We do this first for the benefit of template
-  // instantiations, so that the ParmVarDecls get/ placed into the template
-  // instantiation scope before we transform the function type.
+  // Transform the parameters and return type.
+  //
+  // We instantiate in source order, with the return type first followed by
+  // the parameters, because users tend to expect this (even if they shouldn't
+  // rely on it!).
+  //
+  // FIXME: When we implement late-specified return types, we'll need to
+  // instantiate the return tpe *after* the parameter types in that case,
+  // since the return type can then refer to the parameters themselves (via
+  // decltype, sizeof, etc.).
   llvm::SmallVector<QualType, 4> ParamTypes;
   llvm::SmallVector<ParmVarDecl*, 4> ParamDecls;
-  if (getDerived().TransformFunctionTypeParams(TL, ParamTypes, ParamDecls))
-    return QualType();
-  
   FunctionProtoType *T = TL.getTypePtr();
   QualType ResultType = getDerived().TransformType(TLB, TL.getResultLoc());
   if (ResultType.isNull())
     return QualType();
+
+  if (getDerived().TransformFunctionTypeParams(TL, ParamTypes, ParamDecls))
+    return QualType();  
   
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
@@ -2932,7 +2931,8 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
                                                    ParamTypes.data(),
                                                    ParamTypes.size(),
                                                    T->isVariadic(),
-                                                   T->getTypeQuals());
+                                                   T->getTypeQuals(),
+                                                   T->getExtInfo());
     if (Result.isNull())
       return QualType();
   }
@@ -3022,16 +3022,16 @@ QualType TreeTransform<Derived>::TransformTypeOfExprType(TypeLocBuilder &TLB,
                                                       TypeOfExprTypeLoc TL,
                                                        QualType ObjectType) {
   // typeof expressions are not potentially evaluated contexts
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  Sema::OwningExprResult E = getDerived().TransformExpr(TL.getUnderlyingExpr());
+  ExprResult E = getDerived().TransformExpr(TL.getUnderlyingExpr());
   if (E.isInvalid())
     return QualType();
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       E.get() != TL.getUnderlyingExpr()) {
-    Result = getDerived().RebuildTypeOfExprType(move(E));
+    Result = getDerived().RebuildTypeOfExprType(E.get());
     if (Result.isNull())
       return QualType();
   }
@@ -3077,16 +3077,16 @@ QualType TreeTransform<Derived>::TransformDecltypeType(TypeLocBuilder &TLB,
   DecltypeType *T = TL.getTypePtr();
 
   // decltype expressions are not potentially evaluated contexts
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
-  Sema::OwningExprResult E = getDerived().TransformExpr(T->getUnderlyingExpr());
+  ExprResult E = getDerived().TransformExpr(T->getUnderlyingExpr());
   if (E.isInvalid())
     return QualType();
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() ||
       E.get() != T->getUnderlyingExpr()) {
-    Result = getDerived().RebuildDecltypeType(move(E));
+    Result = getDerived().RebuildDecltypeType(E.get());
     if (Result.isNull())
       return QualType();
   }
@@ -3432,33 +3432,45 @@ TreeTransform<Derived>::TransformObjCObjectPointerType(TypeLocBuilder &TLB,
 // Statement transformation
 //===----------------------------------------------------------------------===//
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformNullStmt(NullStmt *S) {
   return SemaRef.Owned(S->Retain());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformCompoundStmt(CompoundStmt *S) {
   return getDerived().TransformCompoundStmt(S, false);
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformCompoundStmt(CompoundStmt *S,
                                               bool IsStmtExpr) {
+  bool SubStmtInvalid = false;
   bool SubStmtChanged = false;
-  ASTOwningVector<&ActionBase::DeleteStmt> Statements(getSema());
+  ASTOwningVector<Stmt*> Statements(getSema());
   for (CompoundStmt::body_iterator B = S->body_begin(), BEnd = S->body_end();
        B != BEnd; ++B) {
-    OwningStmtResult Result = getDerived().TransformStmt(*B);
-    if (Result.isInvalid())
-      return getSema().StmtError();
+    StmtResult Result = getDerived().TransformStmt(*B);
+    if (Result.isInvalid()) {
+      // Immediately fail if this was a DeclStmt, since it's very
+      // likely that this will cause problems for future statements.
+      if (isa<DeclStmt>(*B))
+        return StmtError();
+
+      // Otherwise, just keep processing substatements and fail later.
+      SubStmtInvalid = true;
+      continue;
+    }
 
     SubStmtChanged = SubStmtChanged || Result.get() != *B;
     Statements.push_back(Result.takeAs<Stmt>());
   }
 
+  if (SubStmtInvalid)
+    return StmtError();
+
   if (!getDerived().AlwaysRebuild() &&
       !SubStmtChanged)
     return SemaRef.Owned(S->Retain());
@@ -3470,75 +3482,75 @@ TreeTransform<Derived>::TransformCompoundStmt(CompoundStmt *S,
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformCaseStmt(CaseStmt *S) {
-  OwningExprResult LHS(SemaRef), RHS(SemaRef);
+  ExprResult LHS, RHS;
   {
     // The case value expressions are not potentially evaluated.
-    EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+    EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
     // Transform the left-hand case value.
     LHS = getDerived().TransformExpr(S->getLHS());
     if (LHS.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
 
     // Transform the right-hand case value (for the GNU case-range extension).
     RHS = getDerived().TransformExpr(S->getRHS());
     if (RHS.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
   }
 
   // Build the case statement.
   // Case statements are always rebuilt so that they will attached to their
   // transformed switch statement.
-  OwningStmtResult Case = getDerived().RebuildCaseStmt(S->getCaseLoc(),
-                                                       move(LHS),
+  StmtResult Case = getDerived().RebuildCaseStmt(S->getCaseLoc(),
+                                                       LHS.get(),
                                                        S->getEllipsisLoc(),
-                                                       move(RHS),
+                                                       RHS.get(),
                                                        S->getColonLoc());
   if (Case.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Transform the statement following the case
-  OwningStmtResult SubStmt = getDerived().TransformStmt(S->getSubStmt());
+  StmtResult SubStmt = getDerived().TransformStmt(S->getSubStmt());
   if (SubStmt.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Attach the body to the case statement
-  return getDerived().RebuildCaseStmtBody(move(Case), move(SubStmt));
+  return getDerived().RebuildCaseStmtBody(Case.get(), SubStmt.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformDefaultStmt(DefaultStmt *S) {
   // Transform the statement following the default case
-  OwningStmtResult SubStmt = getDerived().TransformStmt(S->getSubStmt());
+  StmtResult SubStmt = getDerived().TransformStmt(S->getSubStmt());
   if (SubStmt.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Default statements are always rebuilt
   return getDerived().RebuildDefaultStmt(S->getDefaultLoc(), S->getColonLoc(),
-                                         move(SubStmt));
+                                         SubStmt.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformLabelStmt(LabelStmt *S) {
-  OwningStmtResult SubStmt = getDerived().TransformStmt(S->getSubStmt());
+  StmtResult SubStmt = getDerived().TransformStmt(S->getSubStmt());
   if (SubStmt.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // FIXME: Pass the real colon location in.
   SourceLocation ColonLoc = SemaRef.PP.getLocForEndOfToken(S->getIdentLoc());
   return getDerived().RebuildLabelStmt(S->getIdentLoc(), S->getID(), ColonLoc,
-                                       move(SubStmt));
+                                       SubStmt.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformIfStmt(IfStmt *S) {
   // Transform the condition
-  OwningExprResult Cond(SemaRef);
+  ExprResult Cond;
   VarDecl *ConditionVar = 0;
   if (S->getConditionVariable()) {
     ConditionVar 
@@ -3547,56 +3559,56 @@ TreeTransform<Derived>::TransformIfStmt(IfStmt *S) {
                                       S->getConditionVariable()->getLocation(),
                                                     S->getConditionVariable()));
     if (!ConditionVar)
-      return SemaRef.StmtError();
+      return StmtError();
   } else {
     Cond = getDerived().TransformExpr(S->getCond());
   
     if (Cond.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
     
     // Convert the condition to a boolean value.
     if (S->getCond()) {
-      OwningExprResult CondE = getSema().ActOnBooleanCondition(0, 
+      ExprResult CondE = getSema().ActOnBooleanCondition(0, 
                                                                S->getIfLoc(), 
-                                                               move(Cond));
+                                                               Cond.get());
       if (CondE.isInvalid())
-        return getSema().StmtError();
+        return StmtError();
     
-      Cond = move(CondE);
+      Cond = CondE.get();
     }
   }
   
-  Sema::FullExprArg FullCond(getSema().MakeFullExpr(Cond));
-  if (!S->getConditionVariable() && S->getCond() && !FullCond->get())
-    return SemaRef.StmtError();
+  Sema::FullExprArg FullCond(getSema().MakeFullExpr(Cond.take()));
+  if (!S->getConditionVariable() && S->getCond() && !FullCond.get())
+    return StmtError();
   
   // Transform the "then" branch.
-  OwningStmtResult Then = getDerived().TransformStmt(S->getThen());
+  StmtResult Then = getDerived().TransformStmt(S->getThen());
   if (Then.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Transform the "else" branch.
-  OwningStmtResult Else = getDerived().TransformStmt(S->getElse());
+  StmtResult Else = getDerived().TransformStmt(S->getElse());
   if (Else.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   if (!getDerived().AlwaysRebuild() &&
-      FullCond->get() == S->getCond() &&
+      FullCond.get() == S->getCond() &&
       ConditionVar == S->getConditionVariable() &&
       Then.get() == S->getThen() &&
       Else.get() == S->getElse())
     return SemaRef.Owned(S->Retain());
 
   return getDerived().RebuildIfStmt(S->getIfLoc(), FullCond, ConditionVar,
-                                    move(Then),
-                                    S->getElseLoc(), move(Else));
+                                    Then.get(),
+                                    S->getElseLoc(), Else.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformSwitchStmt(SwitchStmt *S) {
   // Transform the condition.
-  OwningExprResult Cond(SemaRef);
+  ExprResult Cond;
   VarDecl *ConditionVar = 0;
   if (S->getConditionVariable()) {
     ConditionVar 
@@ -3605,36 +3617,36 @@ TreeTransform<Derived>::TransformSwitchStmt(SwitchStmt *S) {
                                       S->getConditionVariable()->getLocation(),
                                                     S->getConditionVariable()));
     if (!ConditionVar)
-      return SemaRef.StmtError();
+      return StmtError();
   } else {
     Cond = getDerived().TransformExpr(S->getCond());
     
     if (Cond.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
   }
 
   // Rebuild the switch statement.
-  OwningStmtResult Switch
-    = getDerived().RebuildSwitchStmtStart(S->getSwitchLoc(), move(Cond),
+  StmtResult Switch
+    = getDerived().RebuildSwitchStmtStart(S->getSwitchLoc(), Cond.get(),
                                           ConditionVar);
   if (Switch.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Transform the body of the switch statement.
-  OwningStmtResult Body = getDerived().TransformStmt(S->getBody());
+  StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Complete the switch statement.
-  return getDerived().RebuildSwitchStmtBody(S->getSwitchLoc(), move(Switch),
-                                            move(Body));
+  return getDerived().RebuildSwitchStmtBody(S->getSwitchLoc(), Switch.get(),
+                                            Body.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformWhileStmt(WhileStmt *S) {
   // Transform the condition
-  OwningExprResult Cond(SemaRef);
+  ExprResult Cond;
   VarDecl *ConditionVar = 0;
   if (S->getConditionVariable()) {
     ConditionVar 
@@ -3643,76 +3655,76 @@ TreeTransform<Derived>::TransformWhileStmt(WhileStmt *S) {
                                       S->getConditionVariable()->getLocation(),
                                                     S->getConditionVariable()));
     if (!ConditionVar)
-      return SemaRef.StmtError();
+      return StmtError();
   } else {
     Cond = getDerived().TransformExpr(S->getCond());
     
     if (Cond.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
 
     if (S->getCond()) {
       // Convert the condition to a boolean value.
-      OwningExprResult CondE = getSema().ActOnBooleanCondition(0, 
+      ExprResult CondE = getSema().ActOnBooleanCondition(0, 
                                                              S->getWhileLoc(), 
-                                                               move(Cond));
+                                                               Cond.get());
       if (CondE.isInvalid())
-        return getSema().StmtError();
-      Cond = move(CondE);
+        return StmtError();
+      Cond = CondE;
     }
   }
 
-  Sema::FullExprArg FullCond(getSema().MakeFullExpr(Cond));
-  if (!S->getConditionVariable() && S->getCond() && !FullCond->get())
-    return SemaRef.StmtError();
+  Sema::FullExprArg FullCond(getSema().MakeFullExpr(Cond.take()));
+  if (!S->getConditionVariable() && S->getCond() && !FullCond.get())
+    return StmtError();
 
   // Transform the body
-  OwningStmtResult Body = getDerived().TransformStmt(S->getBody());
+  StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   if (!getDerived().AlwaysRebuild() &&
-      FullCond->get() == S->getCond() &&
+      FullCond.get() == S->getCond() &&
       ConditionVar == S->getConditionVariable() &&
       Body.get() == S->getBody())
-    return SemaRef.Owned(S->Retain());
+    return Owned(S);
 
   return getDerived().RebuildWhileStmt(S->getWhileLoc(), FullCond,
-                                       ConditionVar, move(Body));
+                                       ConditionVar, Body.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformDoStmt(DoStmt *S) {
   // Transform the body
-  OwningStmtResult Body = getDerived().TransformStmt(S->getBody());
+  StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Transform the condition
-  OwningExprResult Cond = getDerived().TransformExpr(S->getCond());
+  ExprResult Cond = getDerived().TransformExpr(S->getCond());
   if (Cond.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   if (!getDerived().AlwaysRebuild() &&
       Cond.get() == S->getCond() &&
       Body.get() == S->getBody())
     return SemaRef.Owned(S->Retain());
 
-  return getDerived().RebuildDoStmt(S->getDoLoc(), move(Body), S->getWhileLoc(),
-                                    /*FIXME:*/S->getWhileLoc(), move(Cond),
+  return getDerived().RebuildDoStmt(S->getDoLoc(), Body.get(), S->getWhileLoc(),
+                                    /*FIXME:*/S->getWhileLoc(), Cond.get(),
                                     S->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
   // Transform the initialization statement
-  OwningStmtResult Init = getDerived().TransformStmt(S->getInit());
+  StmtResult Init = getDerived().TransformStmt(S->getInit());
   if (Init.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Transform the condition
-  OwningExprResult Cond(SemaRef);
+  ExprResult Cond;
   VarDecl *ConditionVar = 0;
   if (S->getConditionVariable()) {
     ConditionVar 
@@ -3721,57 +3733,57 @@ TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
                                       S->getConditionVariable()->getLocation(),
                                                     S->getConditionVariable()));
     if (!ConditionVar)
-      return SemaRef.StmtError();
+      return StmtError();
   } else {
     Cond = getDerived().TransformExpr(S->getCond());
     
     if (Cond.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
 
     if (S->getCond()) {
       // Convert the condition to a boolean value.
-      OwningExprResult CondE = getSema().ActOnBooleanCondition(0, 
+      ExprResult CondE = getSema().ActOnBooleanCondition(0, 
                                                                S->getForLoc(), 
-                                                               move(Cond));
+                                                               Cond.get());
       if (CondE.isInvalid())
-        return getSema().StmtError();
+        return StmtError();
 
-      Cond = move(CondE);
+      Cond = CondE.get();
     }
   }
 
-  Sema::FullExprArg FullCond(getSema().MakeFullExpr(Cond));  
-  if (!S->getConditionVariable() && S->getCond() && !FullCond->get())
-    return SemaRef.StmtError();
+  Sema::FullExprArg FullCond(getSema().MakeFullExpr(Cond.take()));  
+  if (!S->getConditionVariable() && S->getCond() && !FullCond.get())
+    return StmtError();
 
   // Transform the increment
-  OwningExprResult Inc = getDerived().TransformExpr(S->getInc());
+  ExprResult Inc = getDerived().TransformExpr(S->getInc());
   if (Inc.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
-  Sema::FullExprArg FullInc(getSema().MakeFullExpr(Inc));
-  if (S->getInc() && !FullInc->get())
-    return SemaRef.StmtError();
+  Sema::FullExprArg FullInc(getSema().MakeFullExpr(Inc.get()));
+  if (S->getInc() && !FullInc.get())
+    return StmtError();
 
   // Transform the body
-  OwningStmtResult Body = getDerived().TransformStmt(S->getBody());
+  StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   if (!getDerived().AlwaysRebuild() &&
       Init.get() == S->getInit() &&
-      FullCond->get() == S->getCond() &&
+      FullCond.get() == S->getCond() &&
       Inc.get() == S->getInc() &&
       Body.get() == S->getBody())
     return SemaRef.Owned(S->Retain());
 
   return getDerived().RebuildForStmt(S->getForLoc(), S->getLParenLoc(),
-                                     move(Init), FullCond, ConditionVar,
-                                     FullInc, S->getRParenLoc(), move(Body));
+                                     Init.get(), FullCond, ConditionVar,
+                                     FullInc, S->getRParenLoc(), Body.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformGotoStmt(GotoStmt *S) {
   // Goto statements must always be rebuilt, to resolve the label.
   return getDerived().RebuildGotoStmt(S->getGotoLoc(), S->getLabelLoc(),
@@ -3779,46 +3791,46 @@ TreeTransform<Derived>::TransformGotoStmt(GotoStmt *S) {
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformIndirectGotoStmt(IndirectGotoStmt *S) {
-  OwningExprResult Target = getDerived().TransformExpr(S->getTarget());
+  ExprResult Target = getDerived().TransformExpr(S->getTarget());
   if (Target.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   if (!getDerived().AlwaysRebuild() &&
       Target.get() == S->getTarget())
     return SemaRef.Owned(S->Retain());
 
   return getDerived().RebuildIndirectGotoStmt(S->getGotoLoc(), S->getStarLoc(),
-                                              move(Target));
+                                              Target.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformContinueStmt(ContinueStmt *S) {
   return SemaRef.Owned(S->Retain());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformBreakStmt(BreakStmt *S) {
   return SemaRef.Owned(S->Retain());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformReturnStmt(ReturnStmt *S) {
-  Sema::OwningExprResult Result = getDerived().TransformExpr(S->getRetValue());
+  ExprResult Result = getDerived().TransformExpr(S->getRetValue());
   if (Result.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // FIXME: We always rebuild the return statement because there is no way
   // to tell whether the return type of the function has changed.
-  return getDerived().RebuildReturnStmt(S->getReturnLoc(), move(Result));
+  return getDerived().RebuildReturnStmt(S->getReturnLoc(), Result.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
   bool DeclChanged = false;
   llvm::SmallVector<Decl *, 4> Decls;
@@ -3827,7 +3839,7 @@ TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
     Decl *Transformed = getDerived().TransformDefinition((*D)->getLocation(),
                                                          *D);
     if (!Transformed)
-      return SemaRef.StmtError();
+      return StmtError();
 
     if (Transformed != *D)
       DeclChanged = true;
@@ -3843,22 +3855,22 @@ TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformSwitchCase(SwitchCase *S) {
   assert(false && "SwitchCase is abstract and cannot be transformed");
   return SemaRef.Owned(S->Retain());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
   
-  ASTOwningVector<&ActionBase::DeleteExpr> Constraints(getSema());
-  ASTOwningVector<&ActionBase::DeleteExpr> Exprs(getSema());
+  ASTOwningVector<Expr*> Constraints(getSema());
+  ASTOwningVector<Expr*> Exprs(getSema());
   llvm::SmallVector<IdentifierInfo *, 4> Names;
 
-  OwningExprResult AsmString(SemaRef);
-  ASTOwningVector<&ActionBase::DeleteExpr> Clobbers(getSema());
+  ExprResult AsmString;
+  ASTOwningVector<Expr*> Clobbers(getSema());
 
   bool ExprsChanged = false;
   
@@ -3871,13 +3883,13 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
     
     // Transform the output expr.
     Expr *OutputExpr = S->getOutputExpr(I);
-    OwningExprResult Result = getDerived().TransformExpr(OutputExpr);
+    ExprResult Result = getDerived().TransformExpr(OutputExpr);
     if (Result.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
     
     ExprsChanged |= Result.get() != OutputExpr;
     
-    Exprs.push_back(Result.takeAs<Expr>());
+    Exprs.push_back(Result.get());
   }
   
   // Go through the inputs.
@@ -3889,13 +3901,13 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
     
     // Transform the input expr.
     Expr *InputExpr = S->getInputExpr(I);
-    OwningExprResult Result = getDerived().TransformExpr(InputExpr);
+    ExprResult Result = getDerived().TransformExpr(InputExpr);
     if (Result.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
     
     ExprsChanged |= Result.get() != InputExpr;
     
-    Exprs.push_back(Result.takeAs<Expr>());
+    Exprs.push_back(Result.get());
   }
   
   if (!getDerived().AlwaysRebuild() && !ExprsChanged)
@@ -3916,7 +3928,7 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
                                      Names.data(),
                                      move_arg(Constraints),
                                      move_arg(Exprs),
-                                     move(AsmString),
+                                     AsmString.get(),
                                      move_arg(Clobbers),
                                      S->getRParenLoc(),
                                      S->isMSAsm());
@@ -3924,31 +3936,31 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
 
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformObjCAtTryStmt(ObjCAtTryStmt *S) {
   // Transform the body of the @try.
-  OwningStmtResult TryBody = getDerived().TransformStmt(S->getTryBody());
+  StmtResult TryBody = getDerived().TransformStmt(S->getTryBody());
   if (TryBody.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // Transform the @catch statements (if present).
   bool AnyCatchChanged = false;
-  ASTOwningVector<&ActionBase::DeleteStmt> CatchStmts(SemaRef);
+  ASTOwningVector<Stmt*> CatchStmts(SemaRef);
   for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
-    OwningStmtResult Catch = getDerived().TransformStmt(S->getCatchStmt(I));
+    StmtResult Catch = getDerived().TransformStmt(S->getCatchStmt(I));
     if (Catch.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
     if (Catch.get() != S->getCatchStmt(I))
       AnyCatchChanged = true;
     CatchStmts.push_back(Catch.release());
   }
   
   // Transform the @finally statement (if present).
-  OwningStmtResult Finally(SemaRef);
+  StmtResult Finally;
   if (S->getFinallyStmt()) {
     Finally = getDerived().TransformStmt(S->getFinallyStmt());
     if (Finally.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
   }
 
   // If nothing changed, just retain this statement.
@@ -3959,12 +3971,12 @@ TreeTransform<Derived>::TransformObjCAtTryStmt(ObjCAtTryStmt *S) {
     return SemaRef.Owned(S->Retain());
   
   // Build a new statement.
-  return getDerived().RebuildObjCAtTryStmt(S->getAtTryLoc(), move(TryBody),
-                                           move_arg(CatchStmts), move(Finally));
+  return getDerived().RebuildObjCAtTryStmt(S->getAtTryLoc(), TryBody.get(),
+                                           move_arg(CatchStmts), Finally.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   // Transform the @catch parameter, if there is one.
   VarDecl *Var = 0;
@@ -3973,7 +3985,7 @@ TreeTransform<Derived>::TransformObjCAtCatchStmt(ObjCAtCatchStmt *S) {
     if (FromVar->getTypeSourceInfo()) {
       TSInfo = getDerived().TransformType(FromVar->getTypeSourceInfo());
       if (!TSInfo)
-        return SemaRef.StmtError();
+        return StmtError();
     }
     
     QualType T;
@@ -3982,30 +3994,30 @@ TreeTransform<Derived>::TransformObjCAtCatchStmt(ObjCAtCatchStmt *S) {
     else {
       T = getDerived().TransformType(FromVar->getType());
       if (T.isNull())
-        return SemaRef.StmtError();        
+        return StmtError();        
     }
     
     Var = getDerived().RebuildObjCExceptionDecl(FromVar, TSInfo, T);
     if (!Var)
-      return SemaRef.StmtError();
+      return StmtError();
   }
   
-  OwningStmtResult Body = getDerived().TransformStmt(S->getCatchBody());
+  StmtResult Body = getDerived().TransformStmt(S->getCatchBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   return getDerived().RebuildObjCAtCatchStmt(S->getAtCatchLoc(), 
                                              S->getRParenLoc(),
-                                             Var, move(Body));
+                                             Var, Body.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
   // Transform the body.
-  OwningStmtResult Body = getDerived().TransformStmt(S->getFinallyBody());
+  StmtResult Body = getDerived().TransformStmt(S->getFinallyBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // If nothing changed, just retain this statement.
   if (!getDerived().AlwaysRebuild() &&
@@ -4014,39 +4026,39 @@ TreeTransform<Derived>::TransformObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
 
   // Build a new statement.
   return getDerived().RebuildObjCAtFinallyStmt(S->getAtFinallyLoc(),
-                                               move(Body));
+                                               Body.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformObjCAtThrowStmt(ObjCAtThrowStmt *S) {
-  OwningExprResult Operand(SemaRef);
+  ExprResult Operand;
   if (S->getThrowExpr()) {
     Operand = getDerived().TransformExpr(S->getThrowExpr());
     if (Operand.isInvalid())
-      return getSema().StmtError();
+      return StmtError();
   }
   
   if (!getDerived().AlwaysRebuild() &&
       Operand.get() == S->getThrowExpr())
     return getSema().Owned(S->Retain());
     
-  return getDerived().RebuildObjCAtThrowStmt(S->getThrowLoc(), move(Operand));
+  return getDerived().RebuildObjCAtThrowStmt(S->getThrowLoc(), Operand.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformObjCAtSynchronizedStmt(
                                                   ObjCAtSynchronizedStmt *S) {
   // Transform the object we are locking.
-  OwningExprResult Object = getDerived().TransformExpr(S->getSynchExpr());
+  ExprResult Object = getDerived().TransformExpr(S->getSynchExpr());
   if (Object.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // Transform the body.
-  OwningStmtResult Body = getDerived().TransformStmt(S->getSynchBody());
+  StmtResult Body = getDerived().TransformStmt(S->getSynchBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // If nothing change, just retain the current statement.
   if (!getDerived().AlwaysRebuild() &&
@@ -4056,27 +4068,27 @@ TreeTransform<Derived>::TransformObjCAtSynchronizedStmt(
 
   // Build a new statement.
   return getDerived().RebuildObjCAtSynchronizedStmt(S->getAtSynchronizedLoc(),
-                                                    move(Object), move(Body));
+                                                    Object.get(), Body.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformObjCForCollectionStmt(
                                                   ObjCForCollectionStmt *S) {
   // Transform the element statement.
-  OwningStmtResult Element = getDerived().TransformStmt(S->getElement());
+  StmtResult Element = getDerived().TransformStmt(S->getElement());
   if (Element.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // Transform the collection expression.
-  OwningExprResult Collection = getDerived().TransformExpr(S->getCollection());
+  ExprResult Collection = getDerived().TransformExpr(S->getCollection());
   if (Collection.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // Transform the body.
-  OwningStmtResult Body = getDerived().TransformStmt(S->getBody());
+  StmtResult Body = getDerived().TransformStmt(S->getBody());
   if (Body.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
   
   // If nothing changed, just retain this statement.
   if (!getDerived().AlwaysRebuild() &&
@@ -4088,15 +4100,15 @@ TreeTransform<Derived>::TransformObjCForCollectionStmt(
   // Build a new statement.
   return getDerived().RebuildObjCForCollectionStmt(S->getForLoc(),
                                                    /*FIXME:*/S->getForLoc(),
-                                                   move(Element),
-                                                   move(Collection),
+                                                   Element.get(),
+                                                   Collection.get(),
                                                    S->getRParenLoc(),
-                                                   move(Body));
+                                                   Body.get());
 }
 
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformCXXCatchStmt(CXXCatchStmt *S) {
   // Transform the exception declaration, if any.
   VarDecl *Var = 0;
@@ -4107,7 +4119,7 @@ TreeTransform<Derived>::TransformCXXCatchStmt(CXXCatchStmt *S) {
 
     QualType T = getDerived().TransformType(ExceptionDecl->getType());
     if (T.isNull())
-      return SemaRef.StmtError();
+      return StmtError();
 
     Var = getDerived().RebuildExceptionDecl(ExceptionDecl,
                                             T,
@@ -4116,20 +4128,14 @@ TreeTransform<Derived>::TransformCXXCatchStmt(CXXCatchStmt *S) {
                                             ExceptionDecl->getLocation(),
                                             /*FIXME: Inaccurate*/
                                     SourceRange(ExceptionDecl->getLocation()));
-    if (!Var || Var->isInvalidDecl()) {
-      if (Var)
-        Var->Destroy(SemaRef.Context);
-      return SemaRef.StmtError();
-    }
+    if (!Var || Var->isInvalidDecl())
+      return StmtError();
   }
 
   // Transform the actual exception handler.
-  OwningStmtResult Handler = getDerived().TransformStmt(S->getHandlerBlock());
-  if (Handler.isInvalid()) {
-    if (Var)
-      Var->Destroy(SemaRef.Context);
-    return SemaRef.StmtError();
-  }
+  StmtResult Handler = getDerived().TransformStmt(S->getHandlerBlock());
+  if (Handler.isInvalid())
+    return StmtError();
 
   if (!getDerived().AlwaysRebuild() &&
       !Var &&
@@ -4138,26 +4144,26 @@ TreeTransform<Derived>::TransformCXXCatchStmt(CXXCatchStmt *S) {
 
   return getDerived().RebuildCXXCatchStmt(S->getCatchLoc(),
                                           Var,
-                                          move(Handler));
+                                          Handler.get());
 }
 
 template<typename Derived>
-Sema::OwningStmtResult
+StmtResult
 TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
   // Transform the try block itself.
-  OwningStmtResult TryBlock
+  StmtResult TryBlock
     = getDerived().TransformCompoundStmt(S->getTryBlock());
   if (TryBlock.isInvalid())
-    return SemaRef.StmtError();
+    return StmtError();
 
   // Transform the handlers.
   bool HandlerChanged = false;
-  ASTOwningVector<&ActionBase::DeleteStmt> Handlers(SemaRef);
+  ASTOwningVector<Stmt*> Handlers(SemaRef);
   for (unsigned I = 0, N = S->getNumHandlers(); I != N; ++I) {
-    OwningStmtResult Handler
+    StmtResult Handler
       = getDerived().TransformCXXCatchStmt(S->getHandler(I));
     if (Handler.isInvalid())
-      return SemaRef.StmtError();
+      return StmtError();
 
     HandlerChanged = HandlerChanged || Handler.get() != S->getHandler(I);
     Handlers.push_back(Handler.takeAs<Stmt>());
@@ -4168,7 +4174,7 @@ TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
       !HandlerChanged)
     return SemaRef.Owned(S->Retain());
 
-  return getDerived().RebuildCXXTryStmt(S->getTryLoc(), move(TryBlock),
+  return getDerived().RebuildCXXTryStmt(S->getTryLoc(), TryBlock.get(),
                                         move_arg(Handlers));
 }
 
@@ -4176,32 +4182,40 @@ TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
 // Expression transformation
 //===----------------------------------------------------------------------===//
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformPredefinedExpr(PredefinedExpr *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
   NestedNameSpecifier *Qualifier = 0;
   if (E->getQualifier()) {
     Qualifier = getDerived().TransformNestedNameSpecifier(E->getQualifier(),
                                                        E->getQualifierRange());
     if (!Qualifier)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
   ValueDecl *ND
     = cast_or_null<ValueDecl>(getDerived().TransformDecl(E->getLocation(),
                                                          E->getDecl()));
   if (!ND)
-    return SemaRef.ExprError();
+    return ExprError();
 
-  if (!getDerived().AlwaysRebuild() && 
+  DeclarationNameInfo NameInfo = E->getNameInfo();
+  if (NameInfo.getName()) {
+    NameInfo = getDerived().TransformDeclarationNameInfo(NameInfo);
+    if (!NameInfo.getName())
+      return ExprError();
+  }
+
+  if (!getDerived().AlwaysRebuild() &&
       Qualifier == E->getQualifier() &&
       ND == E->getDecl() &&
-      !E->hasExplicitTemplateArgumentList()) {
+      NameInfo.getName() == E->getDecl()->getDeclName() &&
+      !E->hasExplicitTemplateArgs()) {
 
     // Mark it referenced in the new context regardless.
     // FIXME: this is a bit instantiation-specific.
@@ -4211,88 +4225,88 @@ TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
   }
 
   TemplateArgumentListInfo TransArgs, *TemplateArgs = 0;
-  if (E->hasExplicitTemplateArgumentList()) {
+  if (E->hasExplicitTemplateArgs()) {
     TemplateArgs = &TransArgs;
     TransArgs.setLAngleLoc(E->getLAngleLoc());
     TransArgs.setRAngleLoc(E->getRAngleLoc());
     for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
       TemplateArgumentLoc Loc;
       if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-        return SemaRef.ExprError();
+        return ExprError();
       TransArgs.addArgument(Loc);
     }
   }
 
   return getDerived().RebuildDeclRefExpr(Qualifier, E->getQualifierRange(),
-                                         ND, E->getLocation(), TemplateArgs);
+                                         ND, NameInfo, TemplateArgs);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformIntegerLiteral(IntegerLiteral *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformFloatingLiteral(FloatingLiteral *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformImaginaryLiteral(ImaginaryLiteral *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformStringLiteral(StringLiteral *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCharacterLiteral(CharacterLiteral *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformParenExpr(ParenExpr *E) {
-  OwningExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
+  ExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getSubExpr())
     return SemaRef.Owned(E->Retain());
 
-  return getDerived().RebuildParenExpr(move(SubExpr), E->getLParen(),
+  return getDerived().RebuildParenExpr(SubExpr.get(), E->getLParen(),
                                        E->getRParen());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformUnaryOperator(UnaryOperator *E) {
-  OwningExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
+  ExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getSubExpr())
     return SemaRef.Owned(E->Retain());
 
   return getDerived().RebuildUnaryOperator(E->getOperatorLoc(),
                                            E->getOpcode(),
-                                           move(SubExpr));
+                                           SubExpr.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
   // Transform the type.
   TypeSourceInfo *Type = getDerived().TransformType(E->getTypeSourceInfo());
   if (!Type)
-    return getSema().ExprError();
+    return ExprError();
   
   // Transform all of the components into components similar to what the
   // parser uses.
@@ -4301,7 +4315,7 @@ TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
   // the fields again. However, __builtin_offsetof is rare enough in 
   // template code that we don't care.
   bool ExprChanged = false;
-  typedef Action::OffsetOfComponent Component;
+  typedef Sema::OffsetOfComponent Component;
   typedef OffsetOfExpr::OffsetOfNode Node;
   llvm::SmallVector<Component, 4> Components;
   for (unsigned I = 0, N = E->getNumComponents(); I != N; ++I) {
@@ -4313,13 +4327,13 @@ TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
     switch (ON.getKind()) {
     case Node::Array: {
       Expr *FromIndex = E->getIndexExpr(ON.getArrayExprIndex());
-      OwningExprResult Index = getDerived().TransformExpr(FromIndex);
+      ExprResult Index = getDerived().TransformExpr(FromIndex);
       if (Index.isInvalid())
-        return getSema().ExprError();
+        return ExprError();
       
       ExprChanged = ExprChanged || Index.get() != FromIndex;
       Comp.isBrackets = true;
-      Comp.U.E = Index.takeAs<Expr>(); // FIXME: leaked
+      Comp.U.E = Index.get();
       break;
     }
         
@@ -4353,14 +4367,14 @@ TreeTransform<Derived>::TransformOffsetOfExpr(OffsetOfExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
   if (E->isArgumentType()) {
     TypeSourceInfo *OldT = E->getArgumentTypeInfo();
 
     TypeSourceInfo *NewT = getDerived().TransformType(OldT);
     if (!NewT)
-      return SemaRef.ExprError();
+      return ExprError();
 
     if (!getDerived().AlwaysRebuild() && OldT == NewT)
       return SemaRef.Owned(E->Retain());
@@ -4370,36 +4384,36 @@ TreeTransform<Derived>::TransformSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
                                              E->getSourceRange());
   }
 
-  Sema::OwningExprResult SubExpr(SemaRef);
+  ExprResult SubExpr;
   {
     // C++0x [expr.sizeof]p1:
     //   The operand is either an expression, which is an unevaluated operand
     //   [...]
-    EnterExpressionEvaluationContext Unevaluated(SemaRef, Action::Unevaluated);
+    EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
 
     SubExpr = getDerived().TransformExpr(E->getArgumentExpr());
     if (SubExpr.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     if (!getDerived().AlwaysRebuild() && SubExpr.get() == E->getArgumentExpr())
       return SemaRef.Owned(E->Retain());
   }
 
-  return getDerived().RebuildSizeOfAlignOf(move(SubExpr), E->getOperatorLoc(),
+  return getDerived().RebuildSizeOfAlignOf(SubExpr.get(), E->getOperatorLoc(),
                                            E->isSizeOf(),
                                            E->getSourceRange());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformArraySubscriptExpr(ArraySubscriptExpr *E) {
-  OwningExprResult LHS = getDerived().TransformExpr(E->getLHS());
+  ExprResult LHS = getDerived().TransformExpr(E->getLHS());
   if (LHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult RHS = getDerived().TransformExpr(E->getRHS());
+  ExprResult RHS = getDerived().TransformExpr(E->getRHS());
   if (RHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
 
   if (!getDerived().AlwaysRebuild() &&
@@ -4407,35 +4421,35 @@ TreeTransform<Derived>::TransformArraySubscriptExpr(ArraySubscriptExpr *E) {
       RHS.get() == E->getRHS())
     return SemaRef.Owned(E->Retain());
 
-  return getDerived().RebuildArraySubscriptExpr(move(LHS),
+  return getDerived().RebuildArraySubscriptExpr(LHS.get(),
                                            /*FIXME:*/E->getLHS()->getLocStart(),
-                                                move(RHS),
+                                                RHS.get(),
                                                 E->getRBracketLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
   // Transform the callee.
-  OwningExprResult Callee = getDerived().TransformExpr(E->getCallee());
+  ExprResult Callee = getDerived().TransformExpr(E->getCallee());
   if (Callee.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // Transform arguments.
   bool ArgChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> Args(SemaRef);
+  ASTOwningVector<Expr*> Args(SemaRef);
   llvm::SmallVector<SourceLocation, 4> FakeCommaLocs;
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I) {
-    OwningExprResult Arg = getDerived().TransformExpr(E->getArg(I));
+    ExprResult Arg = getDerived().TransformExpr(E->getArg(I));
     if (Arg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     // FIXME: Wrong source location information for the ','.
     FakeCommaLocs.push_back(
        SemaRef.PP.getLocForEndOfToken(E->getArg(I)->getSourceRange().getEnd()));
 
     ArgChanged = ArgChanged || Arg.get() != E->getArg(I);
-    Args.push_back(Arg.takeAs<Expr>());
+    Args.push_back(Arg.get());
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -4446,18 +4460,18 @@ TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
   // FIXME: Wrong source location information for the '('.
   SourceLocation FakeLParenLoc
     = ((Expr *)Callee.get())->getSourceRange().getBegin();
-  return getDerived().RebuildCallExpr(move(Callee), FakeLParenLoc,
+  return getDerived().RebuildCallExpr(Callee.get(), FakeLParenLoc,
                                       move_arg(Args),
                                       FakeCommaLocs.data(),
                                       E->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   NestedNameSpecifier *Qualifier = 0;
   if (E->hasQualifier()) {
@@ -4465,14 +4479,14 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
       = getDerived().TransformNestedNameSpecifier(E->getQualifier(),
                                                   E->getQualifierRange());
     if (Qualifier == 0)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
   ValueDecl *Member
     = cast_or_null<ValueDecl>(getDerived().TransformDecl(E->getMemberLoc(),
                                                          E->getMemberDecl()));
   if (!Member)
-    return SemaRef.ExprError();
+    return ExprError();
 
   NamedDecl *FoundDecl = E->getFoundDecl();
   if (FoundDecl == E->getMemberDecl()) {
@@ -4481,7 +4495,7 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
     FoundDecl = cast_or_null<NamedDecl>(
                    getDerived().TransformDecl(E->getMemberLoc(), FoundDecl));
     if (!FoundDecl)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -4489,7 +4503,7 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
       Qualifier == E->getQualifier() &&
       Member == E->getMemberDecl() &&
       FoundDecl == E->getFoundDecl() &&
-      !E->hasExplicitTemplateArgumentList()) {
+      !E->hasExplicitTemplateArgs()) {
     
     // Mark it referenced in the new context regardless.
     // FIXME: this is a bit instantiation-specific.
@@ -4498,13 +4512,13 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
   }
 
   TemplateArgumentListInfo TransArgs;
-  if (E->hasExplicitTemplateArgumentList()) {
+  if (E->hasExplicitTemplateArgs()) {
     TransArgs.setLAngleLoc(E->getLAngleLoc());
     TransArgs.setRAngleLoc(E->getRAngleLoc());
     for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
       TemplateArgumentLoc Loc;
       if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-        return SemaRef.ExprError();
+        return ExprError();
       TransArgs.addArgument(Loc);
     }
   }
@@ -4519,28 +4533,28 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
   // nested-name-qualifier (and therefore could do the lookup).
   NamedDecl *FirstQualifierInScope = 0;
 
-  return getDerived().RebuildMemberExpr(move(Base), FakeOperatorLoc,
+  return getDerived().RebuildMemberExpr(Base.get(), FakeOperatorLoc,
                                         E->isArrow(),
                                         Qualifier,
                                         E->getQualifierRange(),
-                                        E->getMemberLoc(),
+                                        E->getMemberNameInfo(),
                                         Member,
                                         FoundDecl,
-                                        (E->hasExplicitTemplateArgumentList()
+                                        (E->hasExplicitTemplateArgs()
                                            ? &TransArgs : 0),
                                         FirstQualifierInScope);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformBinaryOperator(BinaryOperator *E) {
-  OwningExprResult LHS = getDerived().TransformExpr(E->getLHS());
+  ExprResult LHS = getDerived().TransformExpr(E->getLHS());
   if (LHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult RHS = getDerived().TransformExpr(E->getRHS());
+  ExprResult RHS = getDerived().TransformExpr(E->getRHS());
   if (RHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       LHS.get() == E->getLHS() &&
@@ -4548,30 +4562,30 @@ TreeTransform<Derived>::TransformBinaryOperator(BinaryOperator *E) {
     return SemaRef.Owned(E->Retain());
 
   return getDerived().RebuildBinaryOperator(E->getOperatorLoc(), E->getOpcode(),
-                                            move(LHS), move(RHS));
+                                            LHS.get(), RHS.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCompoundAssignOperator(
                                                       CompoundAssignOperator *E) {
   return getDerived().TransformBinaryOperator(E);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformConditionalOperator(ConditionalOperator *E) {
-  OwningExprResult Cond = getDerived().TransformExpr(E->getCond());
+  ExprResult Cond = getDerived().TransformExpr(E->getCond());
   if (Cond.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult LHS = getDerived().TransformExpr(E->getLHS());
+  ExprResult LHS = getDerived().TransformExpr(E->getLHS());
   if (LHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult RHS = getDerived().TransformExpr(E->getRHS());
+  ExprResult RHS = getDerived().TransformExpr(E->getRHS());
   if (RHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       Cond.get() == E->getCond() &&
@@ -4579,15 +4593,15 @@ TreeTransform<Derived>::TransformConditionalOperator(ConditionalOperator *E) {
       RHS.get() == E->getRHS())
     return SemaRef.Owned(E->Retain());
 
-  return getDerived().RebuildConditionalOperator(move(Cond),
+  return getDerived().RebuildConditionalOperator(Cond.get(),
                                                  E->getQuestionLoc(),
-                                                 move(LHS),
+                                                 LHS.get(),
                                                  E->getColonLoc(),
-                                                 move(RHS));
+                                                 RHS.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformImplicitCastExpr(ImplicitCastExpr *E) {
   // Implicit casts are eliminated during transformation, since they
   // will be recomputed by semantic analysis after transformation.
@@ -4595,7 +4609,7 @@ TreeTransform<Derived>::TransformImplicitCastExpr(ImplicitCastExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCStyleCastExpr(CStyleCastExpr *E) {
   TypeSourceInfo *OldT;
   TypeSourceInfo *NewT;
@@ -4608,13 +4622,13 @@ TreeTransform<Derived>::TransformCStyleCastExpr(CStyleCastExpr *E) {
     OldT = E->getTypeInfoAsWritten();
     NewT = getDerived().TransformType(OldT);
     if (!NewT)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
-  OwningExprResult SubExpr
+  ExprResult SubExpr
     = getDerived().TransformExpr(E->getSubExprAsWritten());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       OldT == NewT &&
@@ -4624,20 +4638,20 @@ TreeTransform<Derived>::TransformCStyleCastExpr(CStyleCastExpr *E) {
   return getDerived().RebuildCStyleCastExpr(E->getLParenLoc(),
                                             NewT,
                                             E->getRParenLoc(),
-                                            move(SubExpr));
+                                            SubExpr.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCompoundLiteralExpr(CompoundLiteralExpr *E) {
   TypeSourceInfo *OldT = E->getTypeSourceInfo();
   TypeSourceInfo *NewT = getDerived().TransformType(OldT);
   if (!NewT)
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult Init = getDerived().TransformExpr(E->getInitializer());
+  ExprResult Init = getDerived().TransformExpr(E->getInitializer());
   if (Init.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       OldT == NewT &&
@@ -4650,15 +4664,15 @@ TreeTransform<Derived>::TransformCompoundLiteralExpr(CompoundLiteralExpr *E) {
 
   return getDerived().RebuildCompoundLiteralExpr(E->getLParenLoc(), NewT,
                                    /*FIXME:*/E->getInitializer()->getLocEnd(),
-                                                 move(Init));
+                                                 Init.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformExtVectorElementExpr(ExtVectorElementExpr *E) {
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       Base.get() == E->getBase())
@@ -4667,24 +4681,24 @@ TreeTransform<Derived>::TransformExtVectorElementExpr(ExtVectorElementExpr *E) {
   // FIXME: Bad source location
   SourceLocation FakeOperatorLoc
     = SemaRef.PP.getLocForEndOfToken(E->getBase()->getLocEnd());
-  return getDerived().RebuildExtVectorElementExpr(move(Base), FakeOperatorLoc,
+  return getDerived().RebuildExtVectorElementExpr(Base.get(), FakeOperatorLoc,
                                                   E->getAccessorLoc(),
                                                   E->getAccessor());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformInitListExpr(InitListExpr *E) {
   bool InitChanged = false;
 
-  ASTOwningVector<&ActionBase::DeleteExpr, 4> Inits(SemaRef);
+  ASTOwningVector<Expr*, 4> Inits(SemaRef);
   for (unsigned I = 0, N = E->getNumInits(); I != N; ++I) {
-    OwningExprResult Init = getDerived().TransformExpr(E->getInit(I));
+    ExprResult Init = getDerived().TransformExpr(E->getInit(I));
     if (Init.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     InitChanged = InitChanged || Init.get() != E->getInit(I);
-    Inits.push_back(Init.takeAs<Expr>());
+    Inits.push_back(Init.get());
   }
 
   if (!getDerived().AlwaysRebuild() && !InitChanged)
@@ -4695,17 +4709,17 @@ TreeTransform<Derived>::TransformInitListExpr(InitListExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
   Designation Desig;
 
   // transform the initializer value
-  OwningExprResult Init = getDerived().TransformExpr(E->getInit());
+  ExprResult Init = getDerived().TransformExpr(E->getInit());
   if (Init.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // transform the designators.
-  ASTOwningVector<&ActionBase::DeleteExpr, 4> ArrayExprs(SemaRef);
+  ASTOwningVector<Expr*, 4> ArrayExprs(SemaRef);
   bool ExprChanged = false;
   for (DesignatedInitExpr::designators_iterator D = E->designators_begin(),
                                              DEnd = E->designators_end();
@@ -4718,9 +4732,9 @@ TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
     }
 
     if (D->isArrayDesignator()) {
-      OwningExprResult Index = getDerived().TransformExpr(E->getArrayIndex(*D));
+      ExprResult Index = getDerived().TransformExpr(E->getArrayIndex(*D));
       if (Index.isInvalid())
-        return SemaRef.ExprError();
+        return ExprError();
 
       Desig.AddDesignator(Designator::getArray(Index.get(),
                                                D->getLBracketLoc()));
@@ -4731,14 +4745,14 @@ TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
     }
 
     assert(D->isArrayRangeDesignator() && "New kind of designator?");
-    OwningExprResult Start
+    ExprResult Start
       = getDerived().TransformExpr(E->getArrayRangeStart(*D));
     if (Start.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
-    OwningExprResult End = getDerived().TransformExpr(E->getArrayRangeEnd(*D));
+    ExprResult End = getDerived().TransformExpr(E->getArrayRangeEnd(*D));
     if (End.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     Desig.AddDesignator(Designator::getArrayRange(Start.get(),
                                                   End.get(),
@@ -4759,11 +4773,11 @@ TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
 
   return getDerived().RebuildDesignatedInitExpr(Desig, move_arg(ArrayExprs),
                                                 E->getEqualOrColonLoc(),
-                                                E->usesGNUSyntax(), move(Init));
+                                                E->usesGNUSyntax(), Init.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformImplicitValueInitExpr(
                                                      ImplicitValueInitExpr *E) {
   TemporaryBase Rebase(*this, E->getLocStart(), DeclarationName());
@@ -4772,7 +4786,7 @@ TreeTransform<Derived>::TransformImplicitValueInitExpr(
   // need to transform the type?
   QualType T = getDerived().TransformType(E->getType());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       T == E->getType())
@@ -4782,44 +4796,37 @@ TreeTransform<Derived>::TransformImplicitValueInitExpr(
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformVAArgExpr(VAArgExpr *E) {
-  // FIXME: Do we want the type as written?
-  QualType T;
+  TypeSourceInfo *TInfo = getDerived().TransformType(E->getWrittenTypeInfo());
+  if (!TInfo)
+    return ExprError();
 
-  {
-    // FIXME: Source location isn't quite accurate.
-    TemporaryBase Rebase(*this, E->getBuiltinLoc(), DeclarationName());
-    T = getDerived().TransformType(E->getType());
-    if (T.isNull())
-      return SemaRef.ExprError();
-  }
-
-  OwningExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
+  ExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      T == E->getType() &&
+      TInfo == E->getWrittenTypeInfo() &&
       SubExpr.get() == E->getSubExpr())
     return SemaRef.Owned(E->Retain());
 
-  return getDerived().RebuildVAArgExpr(E->getBuiltinLoc(), move(SubExpr),
-                                       T, E->getRParenLoc());
+  return getDerived().RebuildVAArgExpr(E->getBuiltinLoc(), SubExpr.get(),
+                                       TInfo, E->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformParenListExpr(ParenListExpr *E) {
   bool ArgumentChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr, 4> Inits(SemaRef);
+  ASTOwningVector<Expr*, 4> Inits(SemaRef);
   for (unsigned I = 0, N = E->getNumExprs(); I != N; ++I) {
-    OwningExprResult Init = getDerived().TransformExpr(E->getExpr(I));
+    ExprResult Init = getDerived().TransformExpr(E->getExpr(I));
     if (Init.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || Init.get() != E->getExpr(I);
-    Inits.push_back(Init.takeAs<Expr>());
+    Inits.push_back(Init.get());
   }
 
   return getDerived().RebuildParenListExpr(E->getLParenLoc(),
@@ -4833,69 +4840,67 @@ TreeTransform<Derived>::TransformParenListExpr(ParenListExpr *E) {
 /// rebuilds the expression, so that the label identifier can be resolved to
 /// the corresponding label statement by semantic analysis.
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformAddrLabelExpr(AddrLabelExpr *E) {
   return getDerived().RebuildAddrLabelExpr(E->getAmpAmpLoc(), E->getLabelLoc(),
                                            E->getLabel());
 }
 
 template<typename Derived>
-Sema::OwningExprResult 
+ExprResult 
 TreeTransform<Derived>::TransformStmtExpr(StmtExpr *E) {
-  OwningStmtResult SubStmt
+  StmtResult SubStmt
     = getDerived().TransformCompoundStmt(E->getSubStmt(), true);
   if (SubStmt.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       SubStmt.get() == E->getSubStmt())
     return SemaRef.Owned(E->Retain());
 
   return getDerived().RebuildStmtExpr(E->getLParenLoc(),
-                                      move(SubStmt),
+                                      SubStmt.get(),
                                       E->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformTypesCompatibleExpr(TypesCompatibleExpr *E) {
-  QualType T1, T2;
-  {
-    // FIXME: Source location isn't quite accurate.
-    TemporaryBase Rebase(*this, E->getBuiltinLoc(), DeclarationName());
-
-    T1 = getDerived().TransformType(E->getArgType1());
-    if (T1.isNull())
-      return SemaRef.ExprError();
+  TypeSourceInfo *TInfo1;
+  TypeSourceInfo *TInfo2;
+  
+  TInfo1 = getDerived().TransformType(E->getArgTInfo1());
+  if (!TInfo1)
+    return ExprError();
 
-    T2 = getDerived().TransformType(E->getArgType2());
-    if (T2.isNull())
-      return SemaRef.ExprError();
-  }
+  TInfo2 = getDerived().TransformType(E->getArgTInfo2());
+  if (!TInfo2)
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
-      T1 == E->getArgType1() &&
-      T2 == E->getArgType2())
+      TInfo1 == E->getArgTInfo1() &&
+      TInfo2 == E->getArgTInfo2())
     return SemaRef.Owned(E->Retain());
 
   return getDerived().RebuildTypesCompatibleExpr(E->getBuiltinLoc(),
-                                                 T1, T2, E->getRParenLoc());
+                                                 TInfo1, TInfo2,
+                                                 E->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformChooseExpr(ChooseExpr *E) {
-  OwningExprResult Cond = getDerived().TransformExpr(E->getCond());
+  ExprResult Cond = getDerived().TransformExpr(E->getCond());
   if (Cond.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult LHS = getDerived().TransformExpr(E->getLHS());
+  ExprResult LHS = getDerived().TransformExpr(E->getLHS());
   if (LHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult RHS = getDerived().TransformExpr(E->getRHS());
+  ExprResult RHS = getDerived().TransformExpr(E->getRHS());
   if (RHS.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       Cond.get() == E->getCond() &&
@@ -4904,18 +4909,18 @@ TreeTransform<Derived>::TransformChooseExpr(ChooseExpr *E) {
     return SemaRef.Owned(E->Retain());
 
   return getDerived().RebuildChooseExpr(E->getBuiltinLoc(),
-                                        move(Cond), move(LHS), move(RHS),
+                                        Cond.get(), LHS.get(), RHS.get(),
                                         E->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformGNUNullExpr(GNUNullExpr *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   switch (E->getOperator()) {
   case OO_New:
@@ -4923,16 +4928,16 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   case OO_Array_New:
   case OO_Array_Delete:
     llvm_unreachable("new and delete operators cannot use CXXOperatorCallExpr");
-    return SemaRef.ExprError();
+    return ExprError();
     
   case OO_Call: {
     // This is a call to an object's operator().
     assert(E->getNumArgs() >= 1 && "Object call is missing arguments");
 
     // Transform the object itself.
-    OwningExprResult Object = getDerived().TransformExpr(E->getArg(0));
+    ExprResult Object = getDerived().TransformExpr(E->getArg(0));
     if (Object.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     // FIXME: Poor location information
     SourceLocation FakeLParenLoc
@@ -4940,15 +4945,15 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
                               static_cast<Expr *>(Object.get())->getLocEnd());
 
     // Transform the call arguments.
-    ASTOwningVector<&ActionBase::DeleteExpr> Args(SemaRef);
+    ASTOwningVector<Expr*> Args(SemaRef);
     llvm::SmallVector<SourceLocation, 4> FakeCommaLocs;
     for (unsigned I = 1, N = E->getNumArgs(); I != N; ++I) {
       if (getDerived().DropCallArgument(E->getArg(I)))
         break;
       
-      OwningExprResult Arg = getDerived().TransformExpr(E->getArg(I));
+      ExprResult Arg = getDerived().TransformExpr(E->getArg(I));
       if (Arg.isInvalid())
-        return SemaRef.ExprError();
+        return ExprError();
 
       // FIXME: Poor source location information.
       SourceLocation FakeCommaLoc
@@ -4958,7 +4963,7 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
       Args.push_back(Arg.release());
     }
 
-    return getDerived().RebuildCallExpr(move(Object), FakeLParenLoc,
+    return getDerived().RebuildCallExpr(Object.get(), FakeLParenLoc,
                                         move_arg(Args),
                                         FakeCommaLocs.data(),
                                         E->getLocEnd());
@@ -4974,27 +4979,27 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
 
   case OO_Conditional:
     llvm_unreachable("conditional operator is not actually overloadable");
-    return SemaRef.ExprError();
+    return ExprError();
 
   case OO_None:
   case NUM_OVERLOADED_OPERATORS:
     llvm_unreachable("not an overloaded operator?");
-    return SemaRef.ExprError();
+    return ExprError();
   }
 
-  OwningExprResult Callee = getDerived().TransformExpr(E->getCallee());
+  ExprResult Callee = getDerived().TransformExpr(E->getCallee());
   if (Callee.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult First = getDerived().TransformExpr(E->getArg(0));
+  ExprResult First = getDerived().TransformExpr(E->getArg(0));
   if (First.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  OwningExprResult Second(SemaRef);
+  ExprResult Second;
   if (E->getNumArgs() == 2) {
     Second = getDerived().TransformExpr(E->getArg(1));
     if (Second.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -5005,19 +5010,19 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
 
   return getDerived().RebuildCXXOperatorCallExpr(E->getOperator(),
                                                  E->getOperatorLoc(),
-                                                 move(Callee),
-                                                 move(First),
-                                                 move(Second));
+                                                 Callee.get(),
+                                                 First.get(),
+                                                 Second.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXMemberCallExpr(CXXMemberCallExpr *E) {
   return getDerived().TransformCallExpr(E);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
   TypeSourceInfo *OldT;
   TypeSourceInfo *NewT;
@@ -5030,13 +5035,13 @@ TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
     OldT = E->getTypeInfoAsWritten();
     NewT = getDerived().TransformType(OldT);
     if (!NewT)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
-  OwningExprResult SubExpr
+  ExprResult SubExpr
     = getDerived().TransformExpr(E->getSubExprAsWritten());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       OldT == NewT &&
@@ -5056,37 +5061,37 @@ TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
                                               NewT,
                                               FakeRAngleLoc,
                                               FakeRAngleLoc,
-                                              move(SubExpr),
+                                              SubExpr.get(),
                                               FakeRParenLoc);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXStaticCastExpr(CXXStaticCastExpr *E) {
   return getDerived().TransformCXXNamedCastExpr(E);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXDynamicCastExpr(CXXDynamicCastExpr *E) {
   return getDerived().TransformCXXNamedCastExpr(E);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXReinterpretCastExpr(
                                                       CXXReinterpretCastExpr *E) {
   return getDerived().TransformCXXNamedCastExpr(E);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXConstCastExpr(CXXConstCastExpr *E) {
   return getDerived().TransformCXXNamedCastExpr(E);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXFunctionalCastExpr(
                                                      CXXFunctionalCastExpr *E) {
   TypeSourceInfo *OldT;
@@ -5097,13 +5102,13 @@ TreeTransform<Derived>::TransformCXXFunctionalCastExpr(
     OldT = E->getTypeInfoAsWritten();
     NewT = getDerived().TransformType(OldT);
     if (!NewT)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
-  OwningExprResult SubExpr
+  ExprResult SubExpr
     = getDerived().TransformExpr(E->getSubExprAsWritten());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       OldT == NewT &&
@@ -5115,18 +5120,18 @@ TreeTransform<Derived>::TransformCXXFunctionalCastExpr(
                                   /*FIXME:*/SourceRange(E->getTypeBeginLoc()),
                                                    NewT,
                                       /*FIXME:*/E->getSubExpr()->getLocStart(),
-                                                   move(SubExpr),
+                                                   SubExpr.get(),
                                                    E->getRParenLoc());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
   if (E->isTypeOperand()) {
     TypeSourceInfo *TInfo
       = getDerived().TransformType(E->getTypeOperandSourceInfo());
     if (!TInfo)
-      return SemaRef.ExprError();
+      return ExprError();
 
     if (!getDerived().AlwaysRebuild() &&
         TInfo == E->getTypeOperandSourceInfo())
@@ -5142,11 +5147,11 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
   // after we perform semantic analysis, so the expression is potentially
   // potentially evaluated.
   EnterExpressionEvaluationContext Unevaluated(SemaRef,
-                                      Action::PotentiallyPotentiallyEvaluated);
+                                      Sema::PotentiallyPotentiallyEvaluated);
 
-  OwningExprResult SubExpr = getDerived().TransformExpr(E->getExprOperand());
+  ExprResult SubExpr = getDerived().TransformExpr(E->getExprOperand());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       SubExpr.get() == E->getExprOperand())
@@ -5154,31 +5159,31 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
 
   return getDerived().RebuildCXXTypeidExpr(E->getType(),
                                            E->getLocStart(),
-                                           move(SubExpr),
+                                           SubExpr.get(),
                                            E->getLocEnd());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXNullPtrLiteralExpr(
                                                      CXXNullPtrLiteralExpr *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXThisExpr(CXXThisExpr *E) {
   TemporaryBase Rebase(*this, E->getLocStart(), DeclarationName());
 
   QualType T = getDerived().TransformType(E->getType());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       T == E->getType())
@@ -5188,27 +5193,27 @@ TreeTransform<Derived>::TransformCXXThisExpr(CXXThisExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXThrowExpr(CXXThrowExpr *E) {
-  OwningExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
+  ExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
   if (SubExpr.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       SubExpr.get() == E->getSubExpr())
     return SemaRef.Owned(E->Retain());
 
-  return getDerived().RebuildCXXThrowExpr(E->getThrowLoc(), move(SubExpr));
+  return getDerived().RebuildCXXThrowExpr(E->getThrowLoc(), SubExpr.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   ParmVarDecl *Param
     = cast_or_null<ParmVarDecl>(getDerived().TransformDecl(E->getLocStart(),
                                                            E->getParam()));
   if (!Param)
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       Param == E->getParam())
@@ -5218,13 +5223,13 @@ TreeTransform<Derived>::TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
   TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
 
   QualType T = getDerived().TransformType(E->getType());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       T == E->getType())
@@ -5237,40 +5242,40 @@ TreeTransform<Derived>::TransformCXXScalarValueInitExpr(CXXScalarValueInitExpr *
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
   // Transform the type that we're allocating
   TemporaryBase Rebase(*this, E->getLocStart(), DeclarationName());
   QualType AllocType = getDerived().TransformType(E->getAllocatedType());
   if (AllocType.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // Transform the size of the array we're allocating (if any).
-  OwningExprResult ArraySize = getDerived().TransformExpr(E->getArraySize());
+  ExprResult ArraySize = getDerived().TransformExpr(E->getArraySize());
   if (ArraySize.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // Transform the placement arguments (if any).
   bool ArgumentChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> PlacementArgs(SemaRef);
+  ASTOwningVector<Expr*> PlacementArgs(SemaRef);
   for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
-    OwningExprResult Arg = getDerived().TransformExpr(E->getPlacementArg(I));
+    ExprResult Arg = getDerived().TransformExpr(E->getPlacementArg(I));
     if (Arg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || Arg.get() != E->getPlacementArg(I);
     PlacementArgs.push_back(Arg.take());
   }
 
   // transform the constructor arguments (if any).
-  ASTOwningVector<&ActionBase::DeleteExpr> ConstructorArgs(SemaRef);
+  ASTOwningVector<Expr*> ConstructorArgs(SemaRef);
   for (unsigned I = 0, N = E->getNumConstructorArgs(); I != N; ++I) {
     if (getDerived().DropCallArgument(E->getConstructorArg(I)))
       break;
     
-    OwningExprResult Arg = getDerived().TransformExpr(E->getConstructorArg(I));
+    ExprResult Arg = getDerived().TransformExpr(E->getConstructorArg(I));
     if (Arg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || Arg.get() != E->getConstructorArg(I);
     ConstructorArgs.push_back(Arg.take());
@@ -5283,7 +5288,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
                                    getDerived().TransformDecl(E->getLocStart(),
                                                          E->getConstructor()));
     if (!Constructor)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
   FunctionDecl *OperatorNew = 0;
@@ -5292,7 +5297,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
                                  getDerived().TransformDecl(E->getLocStart(),
                                                          E->getOperatorNew()));
     if (!OperatorNew)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
   FunctionDecl *OperatorDelete = 0;
@@ -5301,7 +5306,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
                                    getDerived().TransformDecl(E->getLocStart(),
                                                        E->getOperatorDelete()));
     if (!OperatorDelete)
-      return SemaRef.ExprError();
+      return ExprError();
   }
   
   if (!getDerived().AlwaysRebuild() &&
@@ -5334,10 +5339,10 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
     } else if (const ConstantArrayType *ConsArrayT
                                      = dyn_cast<ConstantArrayType>(ArrayT)) {
       ArraySize 
-        = SemaRef.Owned(new (SemaRef.Context) IntegerLiteral(
-                                                  ConsArrayT->getSize(), 
-                                                  SemaRef.Context.getSizeType(),
-                                                  /*FIXME:*/E->getLocStart()));
+        = SemaRef.Owned(IntegerLiteral::Create(SemaRef.Context,
+                                               ConsArrayT->getSize(), 
+                                               SemaRef.Context.getSizeType(),
+                                               /*FIXME:*/E->getLocStart()));
       AllocType = ConsArrayT->getElementType();
     } else if (const DependentSizedArrayType *DepArrayT
                               = dyn_cast<DependentSizedArrayType>(ArrayT)) {
@@ -5356,18 +5361,18 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
                                         AllocType,
                                         /*FIXME:*/E->getLocStart(),
                                         /*FIXME:*/SourceRange(),
-                                        move(ArraySize),
+                                        ArraySize.get(),
                                         /*FIXME:*/E->getLocStart(),
                                         move_arg(ConstructorArgs),
                                         E->getLocEnd());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXDeleteExpr(CXXDeleteExpr *E) {
-  OwningExprResult Operand = getDerived().TransformExpr(E->getArgument());
+  ExprResult Operand = getDerived().TransformExpr(E->getArgument());
   if (Operand.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // Transform the delete operator, if known.
   FunctionDecl *OperatorDelete = 0;
@@ -5376,7 +5381,7 @@ TreeTransform<Derived>::TransformCXXDeleteExpr(CXXDeleteExpr *E) {
                                    getDerived().TransformDecl(E->getLocStart(),
                                                        E->getOperatorDelete()));
     if (!OperatorDelete)
-      return SemaRef.ExprError();
+      return ExprError();
   }
   
   if (!getDerived().AlwaysRebuild() &&
@@ -5392,41 +5397,41 @@ TreeTransform<Derived>::TransformCXXDeleteExpr(CXXDeleteExpr *E) {
   return getDerived().RebuildCXXDeleteExpr(E->getLocStart(),
                                            E->isGlobalDelete(),
                                            E->isArrayForm(),
-                                           move(Operand));
+                                           Operand.get());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
                                                      CXXPseudoDestructorExpr *E) {
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  Sema::TypeTy *ObjectTypePtr = 0;
+  ParsedType ObjectTypePtr;
   bool MayBePseudoDestructor = false;
-  Base = SemaRef.ActOnStartCXXMemberReference(0, move(Base), 
+  Base = SemaRef.ActOnStartCXXMemberReference(0, Base.get(), 
                                               E->getOperatorLoc(),
                                         E->isArrow()? tok::arrow : tok::period,
                                               ObjectTypePtr,
                                               MayBePseudoDestructor);
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
                                               
-  QualType ObjectType = QualType::getFromOpaquePtr(ObjectTypePtr);
+  QualType ObjectType = ObjectTypePtr.get();
   NestedNameSpecifier *Qualifier
     = getDerived().TransformNestedNameSpecifier(E->getQualifier(),
                                                 E->getQualifierRange(),
                                                 ObjectType);
   if (E->getQualifier() && !Qualifier)
-    return SemaRef.ExprError();
+    return ExprError();
 
   PseudoDestructorTypeStorage Destroyed;
   if (E->getDestroyedTypeInfo()) {
     TypeSourceInfo *DestroyedTypeInfo
       = getDerived().TransformType(E->getDestroyedTypeInfo(), ObjectType);
     if (!DestroyedTypeInfo)
-      return SemaRef.ExprError();
+      return ExprError();
     Destroyed = DestroyedTypeInfo;
   } else if (ObjectType->isDependentType()) {
     // We aren't likely to be able to resolve the identifier down to a type
@@ -5441,14 +5446,14 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
       SS.setRange(E->getQualifierRange());
     }
     
-    Sema::TypeTy *T = SemaRef.getDestructorName(E->getTildeLoc(),
+    ParsedType T = SemaRef.getDestructorName(E->getTildeLoc(),
                                               *E->getDestroyedTypeIdentifier(),
                                                 E->getDestroyedTypeLoc(),
                                                 /*Scope=*/0,
                                                 SS, ObjectTypePtr,
                                                 false);
     if (!T)
-      return SemaRef.ExprError();
+      return ExprError();
     
     Destroyed
       = SemaRef.Context.getTrivialTypeSourceInfo(SemaRef.GetTypeFromParser(T),
@@ -5460,10 +5465,10 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
     ScopeTypeInfo = getDerived().TransformType(E->getScopeTypeInfo(), 
                                                ObjectType);
     if (!ScopeTypeInfo)
-      return SemaRef.ExprError();
+      return ExprError();
   }
   
-  return getDerived().RebuildCXXPseudoDestructorExpr(move(Base),
+  return getDerived().RebuildCXXPseudoDestructorExpr(Base.get(),
                                                      E->getOperatorLoc(),
                                                      E->isArrow(),
                                                      Qualifier,
@@ -5475,7 +5480,7 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformUnresolvedLookupExpr(
                                                   UnresolvedLookupExpr *Old) {
   TemporaryBase Rebase(*this, Old->getNameLoc(), DeclarationName());
@@ -5495,7 +5500,7 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
       if (isa<UsingShadowDecl>(*I))
         continue;
       else
-        return SemaRef.ExprError();
+        return ExprError();
     }
 
     // Expand using declarations.
@@ -5521,7 +5526,7 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
     Qualifier = getDerived().TransformNestedNameSpecifier(Old->getQualifier(),
                                                     Old->getQualifierRange());
     if (!Qualifier)
-      return SemaRef.ExprError();
+      return ExprError();
     
     SS.setScopeRep(Qualifier);
     SS.setRange(Old->getQualifierRange());
@@ -5533,7 +5538,7 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
                                                             Old->getNameLoc(),
                                                         Old->getNamingClass()));
     if (!NamingClass)
-      return SemaRef.ExprError();
+      return ExprError();
     
     R.setNamingClass(NamingClass);
   }
@@ -5548,7 +5553,7 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
   for (unsigned I = 0, N = Old->getNumTemplateArgs(); I != N; ++I) {
     TemplateArgumentLoc Loc;
     if (getDerived().TransformTemplateArgument(Old->getTemplateArgs()[I], Loc))
-      return SemaRef.ExprError();
+      return ExprError();
     TransArgs.addArgument(Loc);
   }
 
@@ -5557,13 +5562,13 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
   TemporaryBase Rebase(*this, /*FIXME*/E->getLocStart(), DeclarationName());
 
   QualType T = getDerived().TransformType(E->getQueriedType());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       T == E->getQueriedType())
@@ -5581,29 +5586,31 @@ TreeTransform<Derived>::TransformUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
-                                                  DependentScopeDeclRefExpr *E) {
+                                               DependentScopeDeclRefExpr *E) {
   NestedNameSpecifier *NNS
     = getDerived().TransformNestedNameSpecifier(E->getQualifier(),
                                                 E->getQualifierRange());
   if (!NNS)
-    return SemaRef.ExprError();
+    return ExprError();
 
-  DeclarationName Name
-    = getDerived().TransformDeclarationName(E->getDeclName(), E->getLocation());
-  if (!Name)
-    return SemaRef.ExprError();
+  DeclarationNameInfo NameInfo
+    = getDerived().TransformDeclarationNameInfo(E->getNameInfo());
+  if (!NameInfo.getName())
+    return ExprError();
 
   if (!E->hasExplicitTemplateArgs()) {
     if (!getDerived().AlwaysRebuild() &&
         NNS == E->getQualifier() &&
-        Name == E->getDeclName())
+        // Note: it is sufficient to compare the Name component of NameInfo:
+        // if name has not changed, DNLoc has not changed either.
+        NameInfo.getName() == E->getDeclName())
       return SemaRef.Owned(E->Retain());
 
     return getDerived().RebuildDependentScopeDeclRefExpr(NNS,
                                                          E->getQualifierRange(),
-                                                         Name, E->getLocation(),
+                                                         NameInfo,
                                                          /*TemplateArgs*/ 0);
   }
 
@@ -5611,18 +5618,18 @@ TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
   for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
     TemplateArgumentLoc Loc;
     if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-      return SemaRef.ExprError();
+      return ExprError();
     TransArgs.addArgument(Loc);
   }
 
   return getDerived().RebuildDependentScopeDeclRefExpr(NNS,
                                                        E->getQualifierRange(),
-                                                       Name, E->getLocation(),
+                                                       NameInfo,
                                                        &TransArgs);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
   // CXXConstructExprs are always implicit, so when we have a
   // 1-argument construction we just transform that argument.
@@ -5634,17 +5641,17 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
 
   QualType T = getDerived().TransformType(E->getType());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   CXXConstructorDecl *Constructor
     = cast_or_null<CXXConstructorDecl>(
                                 getDerived().TransformDecl(E->getLocStart(),
                                                          E->getConstructor()));
   if (!Constructor)
-    return SemaRef.ExprError();
+    return ExprError();
 
   bool ArgumentChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> Args(SemaRef);
+  ASTOwningVector<Expr*> Args(SemaRef);
   for (CXXConstructExpr::arg_iterator Arg = E->arg_begin(),
        ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg) {
@@ -5653,12 +5660,12 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
       break;
     }
 
-    OwningExprResult TransArg = getDerived().TransformExpr(*Arg);
+    ExprResult TransArg = getDerived().TransformExpr(*Arg);
     if (TransArg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || TransArg.get() != *Arg;
-    Args.push_back(TransArg.takeAs<Expr>());
+    Args.push_back(TransArg.get());
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -5673,7 +5680,9 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
 
   return getDerived().RebuildCXXConstructExpr(T, /*FIXME:*/E->getLocStart(),
                                               Constructor, E->isElidable(),
-                                              move_arg(Args));
+                                              move_arg(Args),
+                                              E->requiresZeroInitialization(),
+                                              E->getConstructionKind());
 }
 
 /// \brief Transform a C++ temporary-binding expression.
@@ -5681,51 +5690,41 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
 /// Since CXXBindTemporaryExpr nodes are implicitly generated, we just
 /// transform the subexpression and return that.
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
   return getDerived().TransformExpr(E->getSubExpr());
 }
 
-/// \brief Transform a C++ reference-binding expression.
-///
-/// Since CXXBindReferenceExpr nodes are implicitly generated, we just
-/// transform the subexpression and return that.
-template<typename Derived>
-Sema::OwningExprResult
-TreeTransform<Derived>::TransformCXXBindReferenceExpr(CXXBindReferenceExpr *E) {
-  return getDerived().TransformExpr(E->getSubExpr());
-}
-
 /// \brief Transform a C++ expression that contains temporaries that should
 /// be destroyed after the expression is evaluated.
 ///
 /// Since CXXExprWithTemporaries nodes are implicitly generated, we
 /// just transform the subexpression and return that.
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXExprWithTemporaries(
                                                     CXXExprWithTemporaries *E) {
   return getDerived().TransformExpr(E->getSubExpr());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
                                                       CXXTemporaryObjectExpr *E) {
   TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
   QualType T = getDerived().TransformType(E->getType());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   CXXConstructorDecl *Constructor
     = cast_or_null<CXXConstructorDecl>(
                                   getDerived().TransformDecl(E->getLocStart(), 
                                                          E->getConstructor()));
   if (!Constructor)
-    return SemaRef.ExprError();
+    return ExprError();
 
   bool ArgumentChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> Args(SemaRef);
+  ASTOwningVector<Expr*> Args(SemaRef);
   Args.reserve(E->getNumArgs());
   for (CXXTemporaryObjectExpr::arg_iterator Arg = E->arg_begin(),
                                          ArgEnd = E->arg_end();
@@ -5735,9 +5734,9 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
       break;
     }
 
-    OwningExprResult TransArg = getDerived().TransformExpr(*Arg);
+    ExprResult TransArg = getDerived().TransformExpr(*Arg);
     if (TransArg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || TransArg.get() != *Arg;
     Args.push_back((Expr *)TransArg.release());
@@ -5768,28 +5767,28 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXUnresolvedConstructExpr(
                                                   CXXUnresolvedConstructExpr *E) {
   TemporaryBase Rebase(*this, E->getTypeBeginLoc(), DeclarationName());
   QualType T = getDerived().TransformType(E->getTypeAsWritten());
   if (T.isNull())
-    return SemaRef.ExprError();
+    return ExprError();
 
   bool ArgumentChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> Args(SemaRef);
+  ASTOwningVector<Expr*> Args(SemaRef);
   llvm::SmallVector<SourceLocation, 8> FakeCommaLocs;
   for (CXXUnresolvedConstructExpr::arg_iterator Arg = E->arg_begin(),
                                              ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg) {
-    OwningExprResult TransArg = getDerived().TransformExpr(*Arg);
+    ExprResult TransArg = getDerived().TransformExpr(*Arg);
     if (TransArg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || TransArg.get() != *Arg;
     FakeCommaLocs.push_back(
                         SemaRef.PP.getLocForEndOfToken((*Arg)->getLocEnd()));
-    Args.push_back(TransArg.takeAs<Expr>());
+    Args.push_back(TransArg.get());
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -5807,11 +5806,11 @@ TreeTransform<Derived>::TransformCXXUnresolvedConstructExpr(
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
-                                                     CXXDependentScopeMemberExpr *E) {
+                                             CXXDependentScopeMemberExpr *E) {
   // Transform the base of the expression.
-  OwningExprResult Base(SemaRef, (Expr*) 0);
+  ExprResult Base((Expr*) 0);
   Expr *OldBase;
   QualType BaseType;
   QualType ObjectType;
@@ -5819,20 +5818,20 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
     OldBase = E->getBase();
     Base = getDerived().TransformExpr(OldBase);
     if (Base.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     // Start the member reference and compute the object's type.
-    Sema::TypeTy *ObjectTy = 0;
+    ParsedType ObjectTy;
     bool MayBePseudoDestructor = false;
-    Base = SemaRef.ActOnStartCXXMemberReference(0, move(Base),
+    Base = SemaRef.ActOnStartCXXMemberReference(0, Base.get(),
                                                 E->getOperatorLoc(),
                                       E->isArrow()? tok::arrow : tok::period,
                                                 ObjectTy,
                                                 MayBePseudoDestructor);
     if (Base.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
-    ObjectType = QualType::getFromOpaquePtr(ObjectTy);
+    ObjectType = ObjectTy.get();
     BaseType = ((Expr*) Base.get())->getType();
   } else {
     OldBase = 0;
@@ -5854,14 +5853,14 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
                                                       ObjectType,
                                                       FirstQualifierInScope);
     if (!Qualifier)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
-  DeclarationName Name
-    = getDerived().TransformDeclarationName(E->getMember(), E->getMemberLoc(),
-                                            ObjectType);
-  if (!Name)
-    return SemaRef.ExprError();
+  DeclarationNameInfo NameInfo
+    = getDerived().TransformDeclarationNameInfo(E->getMemberNameInfo(),
+                                                ObjectType);
+  if (!NameInfo.getName())
+    return ExprError();
 
   if (!E->hasExplicitTemplateArgs()) {
     // This is a reference to a member without an explicitly-specified
@@ -5870,19 +5869,18 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
         Base.get() == OldBase &&
         BaseType == E->getBaseType() &&
         Qualifier == E->getQualifier() &&
-        Name == E->getMember() &&
+        NameInfo.getName() == E->getMember() &&
         FirstQualifierInScope == E->getFirstQualifierFoundInScope())
       return SemaRef.Owned(E->Retain());
 
-    return getDerived().RebuildCXXDependentScopeMemberExpr(move(Base),
+    return getDerived().RebuildCXXDependentScopeMemberExpr(Base.get(),
                                                        BaseType,
                                                        E->isArrow(),
                                                        E->getOperatorLoc(),
                                                        Qualifier,
                                                        E->getQualifierRange(),
                                                        FirstQualifierInScope,
-                                                       Name,
-                                                       E->getMemberLoc(),
+                                                       NameInfo,
                                                        /*TemplateArgs*/ 0);
   }
 
@@ -5890,32 +5888,31 @@ TreeTransform<Derived>::TransformCXXDependentScopeMemberExpr(
   for (unsigned I = 0, N = E->getNumTemplateArgs(); I != N; ++I) {
     TemplateArgumentLoc Loc;
     if (getDerived().TransformTemplateArgument(E->getTemplateArgs()[I], Loc))
-      return SemaRef.ExprError();
+      return ExprError();
     TransArgs.addArgument(Loc);
   }
 
-  return getDerived().RebuildCXXDependentScopeMemberExpr(move(Base),
+  return getDerived().RebuildCXXDependentScopeMemberExpr(Base.get(),
                                                      BaseType,
                                                      E->isArrow(),
                                                      E->getOperatorLoc(),
                                                      Qualifier,
                                                      E->getQualifierRange(),
                                                      FirstQualifierInScope,
-                                                     Name,
-                                                     E->getMemberLoc(),
+                                                     NameInfo,
                                                      &TransArgs);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old) {
   // Transform the base of the expression.
-  OwningExprResult Base(SemaRef, (Expr*) 0);
+  ExprResult Base((Expr*) 0);
   QualType BaseType;
   if (!Old->isImplicitAccess()) {
     Base = getDerived().TransformExpr(Old->getBase());
     if (Base.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
     BaseType = ((Expr*) Base.get())->getType();
   } else {
     BaseType = getDerived().TransformType(Old->getBaseType());
@@ -5927,10 +5924,10 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
       = getDerived().TransformNestedNameSpecifier(Old->getQualifier(),
                                                   Old->getQualifierRange());
     if (Qualifier == 0)
-      return SemaRef.ExprError();
+      return ExprError();
   }
 
-  LookupResult R(SemaRef, Old->getMemberName(), Old->getMemberLoc(),
+  LookupResult R(SemaRef, Old->getMemberNameInfo(),
                  Sema::LookupOrdinaryName);
 
   // Transform all the decls.
@@ -5945,7 +5942,7 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
       if (isa<UsingShadowDecl>(*I))
         continue;
       else
-        return SemaRef.ExprError();
+        return ExprError();
     }
 
     // Expand using declarations.
@@ -5969,7 +5966,7 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
                                                           Old->getMemberLoc(),
                                                         Old->getNamingClass()));
     if (!NamingClass)
-      return SemaRef.ExprError();
+      return ExprError();
     
     R.setNamingClass(NamingClass);
   }
@@ -5982,7 +5979,7 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
       TemplateArgumentLoc Loc;
       if (getDerived().TransformTemplateArgument(Old->getTemplateArgs()[I],
                                                  Loc))
-        return SemaRef.ExprError();
+        return ExprError();
       TransArgs.addArgument(Loc);
     }
   }
@@ -5993,7 +5990,7 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
   // nested-name-qualifier (and therefore could do the lookup).
   NamedDecl *FirstQualifierInScope = 0;
   
-  return getDerived().RebuildUnresolvedMemberExpr(move(Base),
+  return getDerived().RebuildUnresolvedMemberExpr(Base.get(),
                                                   BaseType,
                                                   Old->getOperatorLoc(),
                                                   Old->isArrow(),
@@ -6006,18 +6003,18 @@ TreeTransform<Derived>::TransformUnresolvedMemberExpr(UnresolvedMemberExpr *Old)
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCStringLiteral(ObjCStringLiteral *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCEncodeExpr(ObjCEncodeExpr *E) {
   TypeSourceInfo *EncodedTypeInfo
     = getDerived().TransformType(E->getEncodedTypeSourceInfo());
   if (!EncodedTypeInfo)
-    return SemaRef.ExprError();
+    return ExprError();
 
   if (!getDerived().AlwaysRebuild() &&
       EncodedTypeInfo == E->getEncodedTypeSourceInfo())
@@ -6029,18 +6026,18 @@ TreeTransform<Derived>::TransformObjCEncodeExpr(ObjCEncodeExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
   // Transform arguments.
   bool ArgChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> Args(SemaRef);
+  ASTOwningVector<Expr*> Args(SemaRef);
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I) {
-    OwningExprResult Arg = getDerived().TransformExpr(E->getArg(I));
+    ExprResult Arg = getDerived().TransformExpr(E->getArg(I));
     if (Arg.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
     
     ArgChanged = ArgChanged || Arg.get() != E->getArg(I);
-    Args.push_back(Arg.takeAs<Expr>());
+    Args.push_back(Arg.get());
   }
 
   if (E->getReceiverKind() == ObjCMessageExpr::Class) {
@@ -6048,7 +6045,7 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
     TypeSourceInfo *ReceiverTypeInfo
       = getDerived().TransformType(E->getClassReceiverTypeInfo());
     if (!ReceiverTypeInfo)
-      return SemaRef.ExprError();
+      return ExprError();
     
     // If nothing changed, just retain the existing message send.
     if (!getDerived().AlwaysRebuild() &&
@@ -6067,10 +6064,10 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
   // Instance message: transform the receiver
   assert(E->getReceiverKind() == ObjCMessageExpr::Instance &&
          "Only class and instance messages may be instantiated");
-  OwningExprResult Receiver
+  ExprResult Receiver
     = getDerived().TransformExpr(E->getInstanceReceiver());
   if (Receiver.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // If nothing changed, just retain the existing message send.
   if (!getDerived().AlwaysRebuild() &&
@@ -6078,7 +6075,7 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
     return SemaRef.Owned(E->Retain());
   
   // Build a new instance message send.
-  return getDerived().RebuildObjCMessageExpr(move(Receiver),
+  return getDerived().RebuildObjCMessageExpr(Receiver.get(),
                                              E->getSelector(),
                                              E->getMethodDecl(),
                                              E->getLeftLoc(),
@@ -6087,24 +6084,24 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCSelectorExpr(ObjCSelectorExpr *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCProtocolExpr(ObjCProtocolExpr *E) {
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   // Transform the base expression.
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
   // We don't need to transform the ivar; it will never change.
   
@@ -6113,18 +6110,18 @@ TreeTransform<Derived>::TransformObjCIvarRefExpr(ObjCIvarRefExpr *E) {
       Base.get() == E->getBase())
     return SemaRef.Owned(E->Retain());
   
-  return getDerived().RebuildObjCIvarRefExpr(move(Base), E->getDecl(),
+  return getDerived().RebuildObjCIvarRefExpr(Base.get(), E->getDecl(),
                                              E->getLocation(),
                                              E->isArrow(), E->isFreeIvar());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   // Transform the base expression.
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
   
   // We don't need to transform the property; it will never change.
   
@@ -6133,12 +6130,12 @@ TreeTransform<Derived>::TransformObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
       Base.get() == E->getBase())
     return SemaRef.Owned(E->Retain());
   
-  return getDerived().RebuildObjCPropertyRefExpr(move(Base), E->getProperty(),
+  return getDerived().RebuildObjCPropertyRefExpr(Base.get(), E->getProperty(),
                                                  E->getLocation());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCImplicitSetterGetterRefExpr(
                                           ObjCImplicitSetterGetterRefExpr *E) {
   // If this implicit setter/getter refers to class methods, it cannot have any
@@ -6147,9 +6144,9 @@ TreeTransform<Derived>::TransformObjCImplicitSetterGetterRefExpr(
     return SemaRef.Owned(E->Retain());
   
   // Transform the base expression.
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
   
   // We don't need to transform the getters/setters; they will never change.
   
@@ -6163,46 +6160,46 @@ TreeTransform<Derived>::TransformObjCImplicitSetterGetterRefExpr(
                                                              E->getType(),
                                                           E->getSetterMethod(),
                                                              E->getLocation(),
-                                                             move(Base));
+                                                             Base.get());
                                                              
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCSuperExpr(ObjCSuperExpr *E) {
   // Can never occur in a dependent context.
   return SemaRef.Owned(E->Retain());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformObjCIsaExpr(ObjCIsaExpr *E) {
   // Transform the base expression.
-  OwningExprResult Base = getDerived().TransformExpr(E->getBase());
+  ExprResult Base = getDerived().TransformExpr(E->getBase());
   if (Base.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
   
   // If nothing changed, just retain the existing expression.
   if (!getDerived().AlwaysRebuild() &&
       Base.get() == E->getBase())
     return SemaRef.Owned(E->Retain());
   
-  return getDerived().RebuildObjCIsaExpr(move(Base), E->getIsaMemberLoc(),
+  return getDerived().RebuildObjCIsaExpr(Base.get(), E->getIsaMemberLoc(),
                                          E->isArrow());
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformShuffleVectorExpr(ShuffleVectorExpr *E) {
   bool ArgumentChanged = false;
-  ASTOwningVector<&ActionBase::DeleteExpr> SubExprs(SemaRef);
+  ASTOwningVector<Expr*> SubExprs(SemaRef);
   for (unsigned I = 0, N = E->getNumSubExprs(); I != N; ++I) {
-    OwningExprResult SubExpr = getDerived().TransformExpr(E->getExpr(I));
+    ExprResult SubExpr = getDerived().TransformExpr(E->getExpr(I));
     if (SubExpr.isInvalid())
-      return SemaRef.ExprError();
+      return ExprError();
 
     ArgumentChanged = ArgumentChanged || SubExpr.get() != E->getExpr(I);
-    SubExprs.push_back(SubExpr.takeAs<Expr>());
+    SubExprs.push_back(SubExpr.get());
   }
 
   if (!getDerived().AlwaysRebuild() &&
@@ -6215,7 +6212,7 @@ TreeTransform<Derived>::TransformShuffleVectorExpr(ShuffleVectorExpr *E) {
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
   SourceLocation CaretLoc(E->getExprLoc());
   
@@ -6246,9 +6243,9 @@ TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
   }
     
   // Transform the body
-  OwningStmtResult Body = getDerived().TransformStmt(E->getBody());
+  StmtResult Body = getDerived().TransformStmt(E->getBody());
   if (Body.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
   // Set the parameters on the block decl.
   if (!Params.empty())
     CurBlock->TheDecl->setParams(Params.data(), Params.size());
@@ -6258,14 +6255,15 @@ TreeTransform<Derived>::TransformBlockExpr(BlockExpr *E) {
                                                         ParamTypes.data(),
                                                         ParamTypes.size(),
                                                         BD->isVariadic(),
-                                                        0);
+                                                        0,
+                                               BExprFunctionType->getExtInfo());
   
   CurBlock->FunctionType = FunctionType;
-  return SemaRef.ActOnBlockStmtExpr(CaretLoc, move(Body), /*Scope=*/0);
+  return SemaRef.ActOnBlockStmtExpr(CaretLoc, Body.get(), /*Scope=*/0);
 }
 
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::TransformBlockDeclRefExpr(BlockDeclRefExpr *E) {
   NestedNameSpecifier *Qualifier = 0;
     
@@ -6273,8 +6271,8 @@ TreeTransform<Derived>::TransformBlockDeclRefExpr(BlockDeclRefExpr *E) {
   = cast_or_null<ValueDecl>(getDerived().TransformDecl(E->getLocation(),
                                                        E->getDecl()));
   if (!ND)
-    return SemaRef.ExprError();
-  
+    return ExprError();
+
   if (!getDerived().AlwaysRebuild() &&
       ND == E->getDecl()) {
     // Mark it referenced in the new context regardless.
@@ -6284,8 +6282,9 @@ TreeTransform<Derived>::TransformBlockDeclRefExpr(BlockDeclRefExpr *E) {
     return SemaRef.Owned(E->Retain());
   }
   
+  DeclarationNameInfo NameInfo(E->getDecl()->getDeclName(), E->getLocation());
   return getDerived().RebuildDeclRefExpr(Qualifier, SourceLocation(),
-                                         ND, E->getLocation(), 0);
+                                         ND, NameInfo, 0);
 }
 
 //===----------------------------------------------------------------------===//
@@ -6350,7 +6349,8 @@ TreeTransform<Derived>::RebuildArrayType(QualType ElementType,
       break;
     }
 
-  IntegerLiteral ArraySize(*Size, SizeType, /*FIXME*/BracketsRange.getBegin());
+  IntegerLiteral ArraySize(SemaRef.Context, *Size, SizeType,
+                           /*FIXME*/BracketsRange.getBegin());
   return SemaRef.BuildArrayType(ElementType, SizeMod, &ArraySize,
                                 IndexTypeQuals, BracketsRange,
                                 getDerived().getBaseEntity());
@@ -6381,11 +6381,11 @@ template<typename Derived>
 QualType
 TreeTransform<Derived>::RebuildVariableArrayType(QualType ElementType,
                                           ArrayType::ArraySizeModifier SizeMod,
-                                                 ExprArg SizeExpr,
+                                                 Expr *SizeExpr,
                                                  unsigned IndexTypeQuals,
                                                  SourceRange BracketsRange) {
   return getDerived().RebuildArrayType(ElementType, SizeMod, 0,
-                                       SizeExpr.takeAs<Expr>(),
+                                       SizeExpr,
                                        IndexTypeQuals, BracketsRange);
 }
 
@@ -6393,11 +6393,11 @@ template<typename Derived>
 QualType
 TreeTransform<Derived>::RebuildDependentSizedArrayType(QualType ElementType,
                                           ArrayType::ArraySizeModifier SizeMod,
-                                                       ExprArg SizeExpr,
+                                                       Expr *SizeExpr,
                                                        unsigned IndexTypeQuals,
                                                    SourceRange BracketsRange) {
   return getDerived().RebuildArrayType(ElementType, SizeMod, 0,
-                                       SizeExpr.takeAs<Expr>(),
+                                       SizeExpr,
                                        IndexTypeQuals, BracketsRange);
 }
 
@@ -6416,18 +6416,17 @@ QualType TreeTransform<Derived>::RebuildExtVectorType(QualType ElementType,
   llvm::APInt numElements(SemaRef.Context.getIntWidth(SemaRef.Context.IntTy),
                           NumElements, true);
   IntegerLiteral *VectorSize
-    = new (SemaRef.Context) IntegerLiteral(numElements, SemaRef.Context.IntTy,
-                                           AttributeLoc);
-  return SemaRef.BuildExtVectorType(ElementType, SemaRef.Owned(VectorSize),
-                                    AttributeLoc);
+    = IntegerLiteral::Create(SemaRef.Context, numElements, SemaRef.Context.IntTy,
+                             AttributeLoc);
+  return SemaRef.BuildExtVectorType(ElementType, VectorSize, AttributeLoc);
 }
 
 template<typename Derived>
 QualType
 TreeTransform<Derived>::RebuildDependentSizedExtVectorType(QualType ElementType,
-                                                           ExprArg SizeExpr,
+                                                           Expr *SizeExpr,
                                                   SourceLocation AttributeLoc) {
-  return SemaRef.BuildExtVectorType(ElementType, move(SizeExpr), AttributeLoc);
+  return SemaRef.BuildExtVectorType(ElementType, SizeExpr, AttributeLoc);
 }
 
 template<typename Derived>
@@ -6435,11 +6434,13 @@ QualType TreeTransform<Derived>::RebuildFunctionProtoType(QualType T,
                                                           QualType *ParamTypes,
                                                         unsigned NumParamTypes,
                                                           bool Variadic,
-                                                          unsigned Quals) {
+                                                          unsigned Quals,
+                                            const FunctionType::ExtInfo &Info) {
   return SemaRef.BuildFunctionType(T, ParamTypes, NumParamTypes, Variadic,
                                    Quals,
                                    getDerived().getBaseLocation(),
-                                   getDerived().getBaseEntity());
+                                   getDerived().getBaseEntity(),
+                                   Info);
 }
 
 template<typename Derived>
@@ -6473,8 +6474,8 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(Decl *D) {
 }
 
 template<typename Derived>
-QualType TreeTransform<Derived>::RebuildTypeOfExprType(ExprArg E) {
-  return SemaRef.BuildTypeofExprType(E.takeAs<Expr>());
+QualType TreeTransform<Derived>::RebuildTypeOfExprType(Expr *E) {
+  return SemaRef.BuildTypeofExprType(E);
 }
 
 template<typename Derived>
@@ -6483,8 +6484,8 @@ QualType TreeTransform<Derived>::RebuildTypeOfType(QualType Underlying) {
 }
 
 template<typename Derived>
-QualType TreeTransform<Derived>::RebuildDecltypeType(ExprArg E) {
-  return SemaRef.BuildDecltypeType(E.takeAs<Expr>());
+QualType TreeTransform<Derived>::RebuildDecltypeType(Expr *E) {
+  return SemaRef.BuildDecltypeType(E);
 }
 
 template<typename Derived>
@@ -6563,7 +6564,7 @@ TreeTransform<Derived>::RebuildTemplateName(NestedNameSpecifier *Qualifier,
                                        /*FIXME:*/getDerived().getBaseLocation(),
                                        SS,
                                        Name,
-                                       ObjectType.getAsOpaquePtr(),
+                                       ParsedType::make(ObjectType),
                                        /*EnteringContext=*/false,
                                        Template);
   return Template.template getAsVal<TemplateName>();
@@ -6586,56 +6587,52 @@ TreeTransform<Derived>::RebuildTemplateName(NestedNameSpecifier *Qualifier,
                                        /*FIXME:*/getDerived().getBaseLocation(),
                                        SS,
                                        Name,
-                                       ObjectType.getAsOpaquePtr(),
+                                       ParsedType::make(ObjectType),
                                        /*EnteringContext=*/false,
                                        Template);
   return Template.template getAsVal<TemplateName>();
 }
   
 template<typename Derived>
-Sema::OwningExprResult
+ExprResult
 TreeTransform<Derived>::RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
                                                    SourceLocation OpLoc,
-                                                   ExprArg Callee,
-                                                   ExprArg First,
-                                                   ExprArg Second) {
-  Expr *FirstExpr = (Expr *)First.get();
-  Expr *SecondExpr = (Expr *)Second.get();
-  Expr *CalleeExpr = ((Expr *)Callee.get())->IgnoreParenCasts();
-  bool isPostIncDec = SecondExpr && (Op == OO_PlusPlus || Op == OO_MinusMinus);
+                                                   Expr *OrigCallee,
+                                                   Expr *First,
+                                                   Expr *Second) {
+  Expr *Callee = OrigCallee->IgnoreParenCasts();
+  bool isPostIncDec = Second && (Op == OO_PlusPlus || Op == OO_MinusMinus);
 
   // Determine whether this should be a builtin operation.
   if (Op == OO_Subscript) {
-    if (!FirstExpr->getType()->isOverloadableType() &&
-        !SecondExpr->getType()->isOverloadableType())
-      return getSema().CreateBuiltinArraySubscriptExpr(move(First),
-                                                 CalleeExpr->getLocStart(),
-                                                       move(Second), OpLoc);
+    if (!First->getType()->isOverloadableType() &&
+        !Second->getType()->isOverloadableType())
+      return getSema().CreateBuiltinArraySubscriptExpr(First,
+                                                       Callee->getLocStart(),
+                                                       Second, OpLoc);
   } else if (Op == OO_Arrow) {
     // -> is never a builtin operation.
-    return SemaRef.BuildOverloadedArrowExpr(0, move(First), OpLoc);
-  } else if (SecondExpr == 0 || isPostIncDec) {
-    if (!FirstExpr->getType()->isOverloadableType()) {
+    return SemaRef.BuildOverloadedArrowExpr(0, First, OpLoc);
+  } else if (Second == 0 || isPostIncDec) {
+    if (!First->getType()->isOverloadableType()) {
       // The argument is not of overloadable type, so try to create a
       // built-in unary operation.
-      UnaryOperator::Opcode Opc
+      UnaryOperatorKind Opc
         = UnaryOperator::getOverloadedOpcode(Op, isPostIncDec);
 
-      return getSema().CreateBuiltinUnaryOp(OpLoc, Opc, move(First));
+      return getSema().CreateBuiltinUnaryOp(OpLoc, Opc, First);
     }
   } else {
-    if (!FirstExpr->getType()->isOverloadableType() &&
-        !SecondExpr->getType()->isOverloadableType()) {
+    if (!First->getType()->isOverloadableType() &&
+        !Second->getType()->isOverloadableType()) {
       // Neither of the arguments is an overloadable type, so try to
       // create a built-in binary operation.
-      BinaryOperator::Opcode Opc = BinaryOperator::getOverloadedOpcode(Op);
-      OwningExprResult Result
-        = SemaRef.CreateBuiltinBinOp(OpLoc, Opc, FirstExpr, SecondExpr);
+      BinaryOperatorKind Opc = BinaryOperator::getOverloadedOpcode(Op);
+      ExprResult Result
+        = SemaRef.CreateBuiltinBinOp(OpLoc, Opc, First, Second);
       if (Result.isInvalid())
-        return SemaRef.ExprError();
+        return ExprError();
 
-      First.release();
-      Second.release();
       return move(Result);
     }
   }
@@ -6644,49 +6641,46 @@ TreeTransform<Derived>::RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
   // used during overload resolution.
   UnresolvedSet<16> Functions;
 
-  if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(CalleeExpr)) {
+  if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(Callee)) {
     assert(ULE->requiresADL());
 
     // FIXME: Do we have to check
     // IsAcceptableNonMemberOperatorCandidate for each of these?
     Functions.append(ULE->decls_begin(), ULE->decls_end());
   } else {
-    Functions.addDecl(cast<DeclRefExpr>(CalleeExpr)->getDecl());
+    Functions.addDecl(cast<DeclRefExpr>(Callee)->getDecl());
   }
 
   // Add any functions found via argument-dependent lookup.
-  Expr *Args[2] = { FirstExpr, SecondExpr };
-  unsigned NumArgs = 1 + (SecondExpr != 0);
+  Expr *Args[2] = { First, Second };
+  unsigned NumArgs = 1 + (Second != 0);
 
   // Create the overloaded operator invocation for unary operators.
   if (NumArgs == 1 || isPostIncDec) {
-    UnaryOperator::Opcode Opc
+    UnaryOperatorKind Opc
       = UnaryOperator::getOverloadedOpcode(Op, isPostIncDec);
-    return SemaRef.CreateOverloadedUnaryOp(OpLoc, Opc, Functions, move(First));
+    return SemaRef.CreateOverloadedUnaryOp(OpLoc, Opc, Functions, First);
   }
 
   if (Op == OO_Subscript)
-    return SemaRef.CreateOverloadedArraySubscriptExpr(CalleeExpr->getLocStart(),
+    return SemaRef.CreateOverloadedArraySubscriptExpr(Callee->getLocStart(),
                                                       OpLoc,
-                                                      move(First),
-                                                      move(Second));
+                                                      First,
+                                                      Second);
 
   // Create the overloaded operator invocation for binary operators.
-  BinaryOperator::Opcode Opc =
-    BinaryOperator::getOverloadedOpcode(Op);
-  OwningExprResult Result
+  BinaryOperatorKind Opc = BinaryOperator::getOverloadedOpcode(Op);
+  ExprResult Result
     = SemaRef.CreateOverloadedBinOp(OpLoc, Opc, Functions, Args[0], Args[1]);
   if (Result.isInvalid())
-    return SemaRef.ExprError();
+    return ExprError();
 
-  First.release();
-  Second.release();
   return move(Result);
 }
 
 template<typename Derived>
-Sema::OwningExprResult 
-TreeTransform<Derived>::RebuildCXXPseudoDestructorExpr(ExprArg Base,
+ExprResult 
+TreeTransform<Derived>::RebuildCXXPseudoDestructorExpr(Expr *Base,
                                                      SourceLocation OperatorLoc,
                                                        bool isArrow,
                                                  NestedNameSpecifier *Qualifier,
@@ -6701,32 +6695,32 @@ TreeTransform<Derived>::RebuildCXXPseudoDestructorExpr(ExprArg Base,
     SS.setScopeRep(Qualifier);
   }
 
-  Expr *BaseE = (Expr *)Base.get();
-  QualType BaseType = BaseE->getType();
-  if (BaseE->isTypeDependent() || Destroyed.getIdentifier() ||
+  QualType BaseType = Base->getType();
+  if (Base->isTypeDependent() || Destroyed.getIdentifier() ||
       (!isArrow && !BaseType->getAs<RecordType>()) ||
       (isArrow && BaseType->getAs<PointerType>() && 
        !BaseType->getAs<PointerType>()->getPointeeType()
                                               ->template getAs<RecordType>())){
     // This pseudo-destructor expression is still a pseudo-destructor.
-    return SemaRef.BuildPseudoDestructorExpr(move(Base), OperatorLoc,
+    return SemaRef.BuildPseudoDestructorExpr(Base, OperatorLoc,
                                              isArrow? tok::arrow : tok::period,
                                              SS, ScopeType, CCLoc, TildeLoc,
                                              Destroyed,
                                              /*FIXME?*/true);
   }
-  
+
   TypeSourceInfo *DestroyedType = Destroyed.getTypeSourceInfo();
-  DeclarationName Name
-    = SemaRef.Context.DeclarationNames.getCXXDestructorName(
-                SemaRef.Context.getCanonicalType(DestroyedType->getType()));
-  
+  DeclarationName Name(SemaRef.Context.DeclarationNames.getCXXDestructorName(
+                 SemaRef.Context.getCanonicalType(DestroyedType->getType())));
+  DeclarationNameInfo NameInfo(Name, Destroyed.getLocation());
+  NameInfo.setNamedTypeInfo(DestroyedType);
+
   // FIXME: the ScopeType should be tacked onto SS.
-  
-  return getSema().BuildMemberReferenceExpr(move(Base), BaseType,
+
+  return getSema().BuildMemberReferenceExpr(Base, BaseType,
                                             OperatorLoc, isArrow,
                                             SS, /*FIXME: FirstQualifier*/ 0,
-                                            Name, Destroyed.getLocation(),
+                                            NameInfo,
                                             /*TemplateArgs*/ 0);
 }
 
diff --git a/lib/Serialization/ASTCommon.cpp b/lib/Serialization/ASTCommon.cpp
new file mode 100644
index 000000000000..77c1aff44b39
--- /dev/null
+++ b/lib/Serialization/ASTCommon.cpp
@@ -0,0 +1,69 @@
+//===--- ASTCommon.cpp - Common stuff for ASTReader/ASTWriter----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines common functions that both ASTReader and ASTWriter use.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTCommon.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "llvm/ADT/StringExtras.h"
+
+using namespace clang;
+
+serialization::TypeIdx
+serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
+  unsigned ID = 0;
+  switch (BT->getKind()) {
+  case BuiltinType::Void:       ID = PREDEF_TYPE_VOID_ID;       break;
+  case BuiltinType::Bool:       ID = PREDEF_TYPE_BOOL_ID;       break;
+  case BuiltinType::Char_U:     ID = PREDEF_TYPE_CHAR_U_ID;     break;
+  case BuiltinType::UChar:      ID = PREDEF_TYPE_UCHAR_ID;      break;
+  case BuiltinType::UShort:     ID = PREDEF_TYPE_USHORT_ID;     break;
+  case BuiltinType::UInt:       ID = PREDEF_TYPE_UINT_ID;       break;
+  case BuiltinType::ULong:      ID = PREDEF_TYPE_ULONG_ID;      break;
+  case BuiltinType::ULongLong:  ID = PREDEF_TYPE_ULONGLONG_ID;  break;
+  case BuiltinType::UInt128:    ID = PREDEF_TYPE_UINT128_ID;    break;
+  case BuiltinType::Char_S:     ID = PREDEF_TYPE_CHAR_S_ID;     break;
+  case BuiltinType::SChar:      ID = PREDEF_TYPE_SCHAR_ID;      break;
+  case BuiltinType::WChar:      ID = PREDEF_TYPE_WCHAR_ID;      break;
+  case BuiltinType::Short:      ID = PREDEF_TYPE_SHORT_ID;      break;
+  case BuiltinType::Int:        ID = PREDEF_TYPE_INT_ID;        break;
+  case BuiltinType::Long:       ID = PREDEF_TYPE_LONG_ID;       break;
+  case BuiltinType::LongLong:   ID = PREDEF_TYPE_LONGLONG_ID;   break;
+  case BuiltinType::Int128:     ID = PREDEF_TYPE_INT128_ID;     break;
+  case BuiltinType::Float:      ID = PREDEF_TYPE_FLOAT_ID;      break;
+  case BuiltinType::Double:     ID = PREDEF_TYPE_DOUBLE_ID;     break;
+  case BuiltinType::LongDouble: ID = PREDEF_TYPE_LONGDOUBLE_ID; break;
+  case BuiltinType::NullPtr:    ID = PREDEF_TYPE_NULLPTR_ID;    break;
+  case BuiltinType::Char16:     ID = PREDEF_TYPE_CHAR16_ID;     break;
+  case BuiltinType::Char32:     ID = PREDEF_TYPE_CHAR32_ID;     break;
+  case BuiltinType::Overload:   ID = PREDEF_TYPE_OVERLOAD_ID;   break;
+  case BuiltinType::Dependent:  ID = PREDEF_TYPE_DEPENDENT_ID;  break;
+  case BuiltinType::ObjCId:     ID = PREDEF_TYPE_OBJC_ID;       break;
+  case BuiltinType::ObjCClass:  ID = PREDEF_TYPE_OBJC_CLASS;    break;
+  case BuiltinType::ObjCSel:    ID = PREDEF_TYPE_OBJC_SEL;      break;
+  case BuiltinType::UndeducedAuto:
+    assert(0 && "Should not see undeduced auto here");
+    break;
+  }
+
+  return TypeIdx(ID);
+}
+
+unsigned serialization::ComputeHash(Selector Sel) {
+  unsigned N = Sel.getNumArgs();
+  if (N == 0)
+    ++N;
+  unsigned R = 5381;
+  for (unsigned I = 0; I != N; ++I)
+    if (IdentifierInfo *II = Sel.getIdentifierInfoForSlot(I))
+      R = llvm::HashString(II->getName(), R);
+  return R;
+}
diff --git a/lib/Serialization/ASTCommon.h b/lib/Serialization/ASTCommon.h
new file mode 100644
index 000000000000..a0e2ecd8aa1b
--- /dev/null
+++ b/lib/Serialization/ASTCommon.h
@@ -0,0 +1,50 @@
+//===- ASTCommon.h - Common stuff for ASTReader/ASTWriter -*- C++ -*-=========//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines common functions that both ASTReader and ASTWriter use.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SERIALIZATION_LIB_AST_COMMON_H
+#define LLVM_CLANG_SERIALIZATION_LIB_AST_COMMON_H
+
+#include "clang/Serialization/ASTBitCodes.h"
+
+namespace clang {
+
+namespace serialization {
+
+TypeIdx TypeIdxFromBuiltin(const BuiltinType *BT);
+
+template <typename IdxForTypeTy>
+TypeID MakeTypeID(QualType T, IdxForTypeTy IdxForType) {
+  if (T.isNull())
+    return PREDEF_TYPE_NULL_ID;
+
+  unsigned FastQuals = T.getLocalFastQualifiers();
+  T.removeFastQualifiers();
+
+  if (T.hasLocalNonFastQualifiers())
+    return IdxForType(T).asTypeID(FastQuals);
+
+  assert(!T.hasLocalQualifiers());
+
+  if (const BuiltinType *BT = dyn_cast<BuiltinType>(T.getTypePtr()))
+    return TypeIdxFromBuiltin(BT).asTypeID(FastQuals);
+
+  return IdxForType(T).asTypeID(FastQuals);
+}
+
+unsigned ComputeHash(Selector Sel);
+
+} // namespace serialization
+
+} // namespace clang
+
+#endif
diff --git a/lib/Frontend/PCHReader.cpp b/lib/Serialization/ASTReader.cpp
index 00aee491d644..f07215cb8f51 100644
--- a/lib/Frontend/PCHReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -1,4 +1,4 @@
-//===--- PCHReader.cpp - Precompiled Headers Reader -------------*- C++ -*-===//
+//===--- ASTReader.cpp - AST File Reader ------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,18 +7,22 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines the PCHReader class, which reads a precompiled header.
+//  This file defines the ASTReader class, which reads AST files.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/PCHReader.h"
+#include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/ASTDeserializationListener.h"
+#include "ASTCommon.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
-#include "clang/Frontend/PCHDeserializationListener.h"
 #include "clang/Frontend/Utils.h"
-#include "../Sema/Sema.h" // FIXME: move Sema headers elsewhere
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/Scope.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/Lex/MacroInfo.h"
@@ -41,12 +45,13 @@
 #include <cstdio>
 #include <sys/stat.h>
 using namespace clang;
+using namespace clang::serialization;
 
 //===----------------------------------------------------------------------===//
-// PCH reader validator implementation
+// PCH validator implementation
 //===----------------------------------------------------------------------===//
 
-PCHReaderListener::~PCHReaderListener() {}
+ASTReaderListener::~ASTReaderListener() {}
 
 bool
 PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
@@ -170,6 +175,7 @@ static bool EqualConcatenations(llvm::SmallVector<llvm::StringRef, 2> L,
   // Do it the hard way. At this point, both vectors must be non-empty.
   llvm::StringRef LR = L[0], RR = R[0].Data;
   unsigned LI = 0, RI = 0, LN = L.size(), RN = R.size();
+  (void) RN;
   for (;;) {
     // Compare the current pieces.
     if (LR.size() == RR.size()) {
@@ -241,9 +247,6 @@ bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
 
   // If the concatenation of all the PCH buffers is equal to the adjusted
   // command line, we're done.
-  // We build a SmallVector of the command line here, because we'll eventually
-  // need to support an arbitrary amount of pieces anyway (when we have chained
-  // PCH reading).
   llvm::SmallVector<llvm::StringRef, 2> CommandLine;
   CommandLine.push_back(Left);
   CommandLine.push_back(Right);
@@ -409,60 +412,31 @@ void PCHValidator::ReadCounter(unsigned Value) {
 }
 
 //===----------------------------------------------------------------------===//
-// PCH reader implementation
+// AST reader implementation
 //===----------------------------------------------------------------------===//
 
-PCHReader::PCHReader(Preprocessor &PP, ASTContext *Context,
-                     const char *isysroot)
-  : Listener(new PCHValidator(PP, *this)), DeserializationListener(0),
-    SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
-    Diags(PP.getDiagnostics()), SemaObj(0), PP(&PP), Context(Context),
-    StatCache(0), Consumer(0), IdentifierTableData(0), IdentifierLookupTable(0),
-    IdentifierOffsets(0),
-    MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
-    TotalSelectorsInMethodPool(0), SelectorOffsets(0),
-    TotalNumSelectors(0), MacroDefinitionOffsets(0), 
-    NumPreallocatedPreprocessingEntities(0),  
-    isysroot(isysroot), NumStatHits(0), NumStatMisses(0),
-    NumSLocEntriesRead(0), NumStatementsRead(0),
-    NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
-    NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0),
-    CurrentlyLoadingTypeOrDecl(0) {
-  RelocatablePCH = false;
-}
-
-PCHReader::PCHReader(SourceManager &SourceMgr, FileManager &FileMgr,
-                     Diagnostic &Diags, const char *isysroot)
-  : DeserializationListener(0), SourceMgr(SourceMgr), FileMgr(FileMgr),
-    Diags(Diags), SemaObj(0), PP(0), Context(0), StatCache(0), Consumer(0),
-    IdentifierTableData(0), IdentifierLookupTable(0),
-    IdentifierOffsets(0),
-    MethodPoolLookupTable(0), MethodPoolLookupTableData(0),
-    TotalSelectorsInMethodPool(0), SelectorOffsets(0),
-    TotalNumSelectors(0), MacroDefinitionOffsets(0), 
-    NumPreallocatedPreprocessingEntities(0),  
-    isysroot(isysroot), NumStatHits(0), NumStatMisses(0),
-    NumSLocEntriesRead(0), NumStatementsRead(0),
-    NumMacrosRead(0), NumMethodPoolSelectorsRead(0), NumMethodPoolMisses(0),
-    NumLexicalDeclContextsRead(0), NumVisibleDeclContextsRead(0),
-    CurrentlyLoadingTypeOrDecl(0) {
-  RelocatablePCH = false;
+void
+ASTReader::setDeserializationListener(ASTDeserializationListener *Listener) {
+  DeserializationListener = Listener;
+  if (DeserializationListener)
+    DeserializationListener->SetReader(this);
 }
 
-PCHReader::~PCHReader() {}
-
 
 namespace {
-class PCHMethodPoolLookupTrait {
-  PCHReader &Reader;
+class ASTSelectorLookupTrait {
+  ASTReader &Reader;
 
 public:
-  typedef std::pair<ObjCMethodList, ObjCMethodList> data_type;
+  struct data_type {
+    SelectorID ID;
+    ObjCMethodList Instance, Factory;
+  };
 
   typedef Selector external_key_type;
   typedef external_key_type internal_key_type;
 
-  explicit PCHMethodPoolLookupTrait(PCHReader &Reader) : Reader(Reader) { }
+  explicit ASTSelectorLookupTrait(ASTReader &Reader) : Reader(Reader) { }
 
   static bool EqualKey(const internal_key_type& a,
                        const internal_key_type& b) {
@@ -470,14 +444,7 @@ public:
   }
 
   static unsigned ComputeHash(Selector Sel) {
-    unsigned N = Sel.getNumArgs();
-    if (N == 0)
-      ++N;
-    unsigned R = 5381;
-    for (unsigned I = 0; I != N; ++I)
-      if (IdentifierInfo *II = Sel.getIdentifierInfoForSlot(I))
-        R = llvm::HashString(II->getName(), R);
-    return R;
+    return serialization::ComputeHash(Sel);
   }
 
   // This hopefully will just get inlined and removed by the optimizer.
@@ -513,20 +480,22 @@ public:
 
   data_type ReadData(Selector, const unsigned char* d, unsigned DataLen) {
     using namespace clang::io;
-    unsigned NumInstanceMethods = ReadUnalignedLE16(d);
-    unsigned NumFactoryMethods = ReadUnalignedLE16(d);
 
     data_type Result;
 
+    Result.ID = ReadUnalignedLE32(d);
+    unsigned NumInstanceMethods = ReadUnalignedLE16(d);
+    unsigned NumFactoryMethods = ReadUnalignedLE16(d);
+
     // Load instance methods
     ObjCMethodList *Prev = 0;
     for (unsigned I = 0; I != NumInstanceMethods; ++I) {
       ObjCMethodDecl *Method
         = cast<ObjCMethodDecl>(Reader.GetDecl(ReadUnalignedLE32(d)));
-      if (!Result.first.Method) {
+      if (!Result.Instance.Method) {
         // This is the first method, which is the easy case.
-        Result.first.Method = Method;
-        Prev = &Result.first;
+        Result.Instance.Method = Method;
+        Prev = &Result.Instance;
         continue;
       }
 
@@ -541,10 +510,10 @@ public:
     for (unsigned I = 0; I != NumFactoryMethods; ++I) {
       ObjCMethodDecl *Method
         = cast<ObjCMethodDecl>(Reader.GetDecl(ReadUnalignedLE32(d)));
-      if (!Result.second.Method) {
+      if (!Result.Factory.Method) {
         // This is the first method, which is the easy case.
-        Result.second.Method = Method;
-        Prev = &Result.second;
+        Result.Factory.Method = Method;
+        Prev = &Result.Factory;
         continue;
       }
 
@@ -561,16 +530,17 @@ public:
 } // end anonymous namespace
 
 /// \brief The on-disk hash table used for the global method pool.
-typedef OnDiskChainedHashTable<PCHMethodPoolLookupTrait>
-  PCHMethodPoolLookupTable;
+typedef OnDiskChainedHashTable<ASTSelectorLookupTrait>
+  ASTSelectorLookupTable;
 
 namespace {
-class PCHIdentifierLookupTrait {
-  PCHReader &Reader;
+class ASTIdentifierLookupTrait {
+  ASTReader &Reader;
+  llvm::BitstreamCursor &Stream;
 
   // If we know the IdentifierInfo in advance, it is here and we will
   // not build a new one. Used when deserializing information about an
-  // identifier that was constructed before the PCH file was read.
+  // identifier that was constructed before the AST file was read.
   IdentifierInfo *KnownII;
 
 public:
@@ -580,8 +550,9 @@ public:
 
   typedef external_key_type internal_key_type;
 
-  explicit PCHIdentifierLookupTrait(PCHReader &Reader, IdentifierInfo *II = 0)
-    : Reader(Reader), KnownII(II) { }
+  ASTIdentifierLookupTrait(ASTReader &Reader, llvm::BitstreamCursor &Stream,
+                           IdentifierInfo *II = 0)
+    : Reader(Reader), Stream(Stream), KnownII(II) { }
 
   static bool EqualKey(const internal_key_type& a,
                        const internal_key_type& b) {
@@ -615,26 +586,28 @@ public:
                            const unsigned char* d,
                            unsigned DataLen) {
     using namespace clang::io;
-    pch::IdentID ID = ReadUnalignedLE32(d);
+    IdentID ID = ReadUnalignedLE32(d);
     bool IsInteresting = ID & 0x01;
 
     // Wipe out the "is interesting" bit.
     ID = ID >> 1;
 
     if (!IsInteresting) {
-      // For unintersting identifiers, just build the IdentifierInfo
+      // For uninteresting identifiers, just build the IdentifierInfo
       // and associate it with the persistent ID.
       IdentifierInfo *II = KnownII;
       if (!II)
-        II = &Reader.getIdentifierTable().CreateIdentifierInfo(
-                                                 k.first, k.first + k.second);
+        II = &Reader.getIdentifierTable().getOwn(k.first, k.first + k.second);
       Reader.SetIdentifierInfo(ID, II);
+      II->setIsFromAST();
       return II;
     }
 
     unsigned Bits = ReadUnalignedLE16(d);
     bool CPlusPlusOperatorKeyword = Bits & 0x01;
     Bits >>= 1;
+    bool HasRevertedTokenIDToIdentifier = Bits & 0x01;
+    Bits >>= 1;
     bool Poisoned = Bits & 0x01;
     Bits >>= 1;
     bool ExtensionToken = Bits & 0x01;
@@ -651,12 +624,13 @@ public:
     // the new IdentifierInfo.
     IdentifierInfo *II = KnownII;
     if (!II)
-      II = &Reader.getIdentifierTable().CreateIdentifierInfo(
-                                                 k.first, k.first + k.second);
+      II = &Reader.getIdentifierTable().getOwn(k.first, k.first + k.second);
     Reader.SetIdentifierInfo(ID, II);
 
     // Set or check the various bits in the IdentifierInfo structure.
-    // FIXME: Load token IDs lazily, too?
+    // Token IDs are read-only.
+    if (HasRevertedTokenIDToIdentifier)
+      II->RevertTokenIDToIdentifier();
     II->setObjCOrBuiltinID(ObjCOrBuiltinID);
     assert(II->isExtensionToken() == ExtensionToken &&
            "Incorrect extension token flag");
@@ -670,7 +644,7 @@ public:
     // definition.
     if (hasMacroDefinition) {
       uint32_t Offset = ReadUnalignedLE32(d);
-      Reader.ReadMacroRecord(Offset);
+      Reader.ReadMacroRecord(Stream, Offset);
       DataLen -= 4;
     }
 
@@ -684,6 +658,7 @@ public:
       Reader.SetGloballyVisibleDecls(II, DeclIDs);
     }
 
+    II->setIsFromAST();
     return II;
   }
 };
@@ -692,23 +667,245 @@ public:
 
 /// \brief The on-disk hash table used to contain information about
 /// all of the identifiers in the program.
-typedef OnDiskChainedHashTable<PCHIdentifierLookupTrait>
-  PCHIdentifierLookupTable;
+typedef OnDiskChainedHashTable<ASTIdentifierLookupTrait>
+  ASTIdentifierLookupTable;
+
+namespace {
+class ASTDeclContextNameLookupTrait {
+  ASTReader &Reader;
+
+public:
+  /// \brief Pair of begin/end iterators for DeclIDs.
+  typedef std::pair<DeclID *, DeclID *> data_type;
+
+  /// \brief Special internal key for declaration names.
+  /// The hash table creates keys for comparison; we do not create
+  /// a DeclarationName for the internal key to avoid deserializing types.
+  struct DeclNameKey {
+    DeclarationName::NameKind Kind;
+    uint64_t Data;
+    DeclNameKey() : Kind((DeclarationName::NameKind)0), Data(0) { }
+  };
+
+  typedef DeclarationName external_key_type;
+  typedef DeclNameKey internal_key_type;
+
+  explicit ASTDeclContextNameLookupTrait(ASTReader &Reader) : Reader(Reader) { }
+
+  static bool EqualKey(const internal_key_type& a,
+                       const internal_key_type& b) {
+    return a.Kind == b.Kind && a.Data == b.Data;
+  }
+
+  unsigned ComputeHash(const DeclNameKey &Key) const {
+    llvm::FoldingSetNodeID ID;
+    ID.AddInteger(Key.Kind);
+
+    switch (Key.Kind) {
+    case DeclarationName::Identifier:
+    case DeclarationName::CXXLiteralOperatorName:
+      ID.AddString(((IdentifierInfo*)Key.Data)->getName());
+      break;
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      ID.AddInteger(serialization::ComputeHash(Selector(Key.Data)));
+      break;
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+      ID.AddInteger((TypeID)Key.Data);
+      break;
+    case DeclarationName::CXXOperatorName:
+      ID.AddInteger((OverloadedOperatorKind)Key.Data);
+      break;
+    case DeclarationName::CXXUsingDirective:
+      break;
+    }
+
+    return ID.ComputeHash();
+  }
+
+  internal_key_type GetInternalKey(const external_key_type& Name) const {
+    DeclNameKey Key;
+    Key.Kind = Name.getNameKind();
+    switch (Name.getNameKind()) {
+    case DeclarationName::Identifier:
+      Key.Data = (uint64_t)Name.getAsIdentifierInfo();
+      break;
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      Key.Data = (uint64_t)Name.getObjCSelector().getAsOpaquePtr();
+      break;
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+      Key.Data = Reader.GetTypeID(Name.getCXXNameType());
+      break;
+    case DeclarationName::CXXOperatorName:
+      Key.Data = Name.getCXXOverloadedOperator();
+      break;
+    case DeclarationName::CXXLiteralOperatorName:
+      Key.Data = (uint64_t)Name.getCXXLiteralIdentifier();
+      break;
+    case DeclarationName::CXXUsingDirective:
+      break;
+    }
+    
+    return Key;
+  }
+
+  external_key_type GetExternalKey(const internal_key_type& Key) const {
+    ASTContext *Context = Reader.getContext();
+    switch (Key.Kind) {
+    case DeclarationName::Identifier:
+      return DeclarationName((IdentifierInfo*)Key.Data);
+
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      return DeclarationName(Selector(Key.Data));
+
+    case DeclarationName::CXXConstructorName:
+      return Context->DeclarationNames.getCXXConstructorName(
+                           Context->getCanonicalType(Reader.GetType(Key.Data)));
+
+    case DeclarationName::CXXDestructorName:
+      return Context->DeclarationNames.getCXXDestructorName(
+                           Context->getCanonicalType(Reader.GetType(Key.Data)));
+
+    case DeclarationName::CXXConversionFunctionName:
+      return Context->DeclarationNames.getCXXConversionFunctionName(
+                           Context->getCanonicalType(Reader.GetType(Key.Data)));
+
+    case DeclarationName::CXXOperatorName:
+      return Context->DeclarationNames.getCXXOperatorName(
+                                         (OverloadedOperatorKind)Key.Data);
+
+    case DeclarationName::CXXLiteralOperatorName:
+      return Context->DeclarationNames.getCXXLiteralOperatorName(
+                                                     (IdentifierInfo*)Key.Data);
+
+    case DeclarationName::CXXUsingDirective:
+      return DeclarationName::getUsingDirectiveName();
+    }
+
+    llvm_unreachable("Invalid Name Kind ?");
+  }
+
+  static std::pair<unsigned, unsigned>
+  ReadKeyDataLength(const unsigned char*& d) {
+    using namespace clang::io;
+    unsigned KeyLen = ReadUnalignedLE16(d);
+    unsigned DataLen = ReadUnalignedLE16(d);
+    return std::make_pair(KeyLen, DataLen);
+  }
+
+  internal_key_type ReadKey(const unsigned char* d, unsigned) {
+    using namespace clang::io;
+
+    DeclNameKey Key;
+    Key.Kind = (DeclarationName::NameKind)*d++;
+    switch (Key.Kind) {
+    case DeclarationName::Identifier:
+      Key.Data = (uint64_t)Reader.DecodeIdentifierInfo(ReadUnalignedLE32(d));
+      break;
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      Key.Data = 
+         (uint64_t)Reader.DecodeSelector(ReadUnalignedLE32(d)).getAsOpaquePtr();
+      break;
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+      Key.Data = ReadUnalignedLE32(d); // TypeID
+      break;
+    case DeclarationName::CXXOperatorName:
+      Key.Data = *d++; // OverloadedOperatorKind
+      break;
+    case DeclarationName::CXXLiteralOperatorName:
+      Key.Data = (uint64_t)Reader.DecodeIdentifierInfo(ReadUnalignedLE32(d));
+      break;
+    case DeclarationName::CXXUsingDirective:
+      break;
+    }
+    
+    return Key;
+  }
+
+  data_type ReadData(internal_key_type, const unsigned char* d,
+                     unsigned DataLen) {
+    using namespace clang::io;
+    unsigned NumDecls = ReadUnalignedLE16(d);
+    DeclID *Start = (DeclID *)d;
+    return std::make_pair(Start, Start + NumDecls);
+  }
+};
+
+} // end anonymous namespace
+
+/// \brief The on-disk hash table used for the DeclContext's Name lookup table.
+typedef OnDiskChainedHashTable<ASTDeclContextNameLookupTrait>
+  ASTDeclContextNameLookupTable;
+
+bool ASTReader::ReadDeclContextStorage(llvm::BitstreamCursor &Cursor,
+                                   const std::pair<uint64_t, uint64_t> &Offsets,
+                                       DeclContextInfo &Info) {
+  SavedStreamPosition SavedPosition(Cursor);
+  // First the lexical decls.
+  if (Offsets.first != 0) {
+    Cursor.JumpToBit(Offsets.first);
+
+    RecordData Record;
+    const char *Blob;
+    unsigned BlobLen;
+    unsigned Code = Cursor.ReadCode();
+    unsigned RecCode = Cursor.ReadRecord(Code, Record, &Blob, &BlobLen);
+    if (RecCode != DECL_CONTEXT_LEXICAL) {
+      Error("Expected lexical block");
+      return true;
+    }
+
+    Info.LexicalDecls = reinterpret_cast<const DeclID*>(Blob);
+    Info.NumLexicalDecls = BlobLen / sizeof(DeclID);
+  } else {
+    Info.LexicalDecls = 0;
+    Info.NumLexicalDecls = 0;
+  }
+
+  // Now the lookup table.
+  if (Offsets.second != 0) {
+    Cursor.JumpToBit(Offsets.second);
+
+    RecordData Record;
+    const char *Blob;
+    unsigned BlobLen;
+    unsigned Code = Cursor.ReadCode();
+    unsigned RecCode = Cursor.ReadRecord(Code, Record, &Blob, &BlobLen);
+    if (RecCode != DECL_CONTEXT_VISIBLE) {
+      Error("Expected visible lookup table block");
+      return true;
+    }
+    Info.NameLookupTableData
+      = ASTDeclContextNameLookupTable::Create(
+                    (const unsigned char *)Blob + Record[0],
+                    (const unsigned char *)Blob,
+                    ASTDeclContextNameLookupTrait(*this));
+  } else {
+    Info.NameLookupTableData = 0;
+  }
 
-void PCHReader::Error(const char *Msg) {
+  return false;
+}
+
+void ASTReader::Error(const char *Msg) {
   Diag(diag::err_fe_pch_malformed) << Msg;
 }
 
-/// \brief Check the contents of the concatenation of all predefines buffers in
-/// the PCH chain against the contents of the predefines buffer of the current
-/// compiler invocation.
-///
-/// The contents should be the same. If not, then some command-line option
-/// changed the preprocessor state and we must probably reject the PCH file.
-///
-/// \returns true if there was a mismatch (in which case the PCH file
-/// should be ignored), or false otherwise.
-bool PCHReader::CheckPredefinesBuffers() {
+/// \brief Tell the AST listener about the predefines buffers in the chain.
+bool ASTReader::CheckPredefinesBuffers() {
   if (Listener)
     return Listener->ReadPredefinesBuffer(PCHPredefinesBuffers,
                                           ActualOriginalFileName,
@@ -722,7 +919,7 @@ bool PCHReader::CheckPredefinesBuffers() {
 
 /// \brief Read the line table in the source manager block.
 /// \returns true if ther was an error.
-bool PCHReader::ParseLineTable(llvm::SmallVectorImpl<uint64_t> &Record) {
+bool ASTReader::ParseLineTable(llvm::SmallVectorImpl<uint64_t> &Record) {
   unsigned Idx = 0;
   LineTableInfo &LineTable = SourceMgr.getLineTable();
 
@@ -766,7 +963,7 @@ bool PCHReader::ParseLineTable(llvm::SmallVectorImpl<uint64_t> &Record) {
 
 namespace {
 
-class PCHStatData {
+class ASTStatData {
 public:
   const bool hasStat;
   const ino_t ino;
@@ -775,19 +972,19 @@ public:
   const time_t mtime;
   const off_t size;
 
-  PCHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
+  ASTStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
   : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}
 
-  PCHStatData()
+  ASTStatData()
     : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
 };
 
-class PCHStatLookupTrait {
+class ASTStatLookupTrait {
  public:
   typedef const char *external_key_type;
   typedef const char *internal_key_type;
 
-  typedef PCHStatData data_type;
+  typedef ASTStatData data_type;
 
   static unsigned ComputeHash(const char *path) {
     return llvm::HashString(path);
@@ -830,13 +1027,13 @@ class PCHStatLookupTrait {
 ///
 /// This cache is very similar to the stat cache used by pretokenized
 /// headers.
-class PCHStatCache : public StatSysCallCache {
-  typedef OnDiskChainedHashTable<PCHStatLookupTrait> CacheTy;
+class ASTStatCache : public StatSysCallCache {
+  typedef OnDiskChainedHashTable<ASTStatLookupTrait> CacheTy;
   CacheTy *Cache;
 
   unsigned &NumStatHits, &NumStatMisses;
 public:
-  PCHStatCache(const unsigned char *Buckets,
+  ASTStatCache(const unsigned char *Buckets,
                const unsigned char *Base,
                unsigned &NumStatHits,
                unsigned &NumStatMisses)
@@ -844,20 +1041,20 @@ public:
     Cache = CacheTy::Create(Buckets, Base);
   }
 
-  ~PCHStatCache() { delete Cache; }
+  ~ASTStatCache() { delete Cache; }
 
   int stat(const char *path, struct stat *buf) {
-    // Do the lookup for the file's data in the PCH file.
+    // Do the lookup for the file's data in the AST file.
     CacheTy::iterator I = Cache->find(path);
 
-    // If we don't get a hit in the PCH file just forward to 'stat'.
+    // If we don't get a hit in the AST file just forward to 'stat'.
     if (I == Cache->end()) {
       ++NumStatMisses;
       return StatSysCallCache::stat(path, buf);
     }
 
     ++NumStatHits;
-    PCHStatData Data = *I;
+    ASTStatData Data = *I;
 
     if (!Data.hasStat)
       return 1;
@@ -873,25 +1070,27 @@ public:
 } // end anonymous namespace
 
 
-/// \brief Read the source manager block
-PCHReader::PCHReadResult PCHReader::ReadSourceManagerBlock() {
+/// \brief Read a source manager block
+ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(PerFileData &F) {
   using namespace SrcMgr;
 
+  llvm::BitstreamCursor &SLocEntryCursor = F.SLocEntryCursor;
+
   // Set the source-location entry cursor to the current position in
   // the stream. This cursor will be used to read the contents of the
   // source manager block initially, and then lazily read
   // source-location entries as needed.
-  SLocEntryCursor = Stream;
+  SLocEntryCursor = F.Stream;
 
   // The stream itself is going to skip over the source manager block.
-  if (Stream.SkipBlock()) {
-    Error("malformed block record in PCH file");
+  if (F.Stream.SkipBlock()) {
+    Error("malformed block record in AST file");
     return Failure;
   }
 
   // Enter the source manager block.
-  if (SLocEntryCursor.EnterSubBlock(pch::SOURCE_MANAGER_BLOCK_ID)) {
-    Error("malformed source manager block record in PCH file");
+  if (SLocEntryCursor.EnterSubBlock(SOURCE_MANAGER_BLOCK_ID)) {
+    Error("malformed source manager block record in AST file");
     return Failure;
   }
 
@@ -900,7 +1099,7 @@ PCHReader::PCHReadResult PCHReader::ReadSourceManagerBlock() {
     unsigned Code = SLocEntryCursor.ReadCode();
     if (Code == llvm::bitc::END_BLOCK) {
       if (SLocEntryCursor.ReadBlockEnd()) {
-        Error("error at end of Source Manager block in PCH file");
+        Error("error at end of Source Manager block in AST file");
         return Failure;
       }
       return Success;
@@ -910,7 +1109,7 @@ PCHReader::PCHReadResult PCHReader::ReadSourceManagerBlock() {
       // No known subblocks, always skip them.
       SLocEntryCursor.ReadSubBlockID();
       if (SLocEntryCursor.SkipBlock()) {
-        Error("malformed block record in PCH file");
+        Error("malformed block record in AST file");
         return Failure;
       }
       continue;
@@ -929,37 +1128,58 @@ PCHReader::PCHReadResult PCHReader::ReadSourceManagerBlock() {
     default:  // Default behavior: ignore.
       break;
 
-    case pch::SM_LINE_TABLE:
+    case SM_LINE_TABLE:
       if (ParseLineTable(Record))
         return Failure;
       break;
 
-    case pch::SM_SLOC_FILE_ENTRY:
-    case pch::SM_SLOC_BUFFER_ENTRY:
-    case pch::SM_SLOC_INSTANTIATION_ENTRY:
+    case SM_SLOC_FILE_ENTRY:
+    case SM_SLOC_BUFFER_ENTRY:
+    case SM_SLOC_INSTANTIATION_ENTRY:
       // Once we hit one of the source location entries, we're done.
       return Success;
     }
   }
 }
 
+/// \brief Get a cursor that's correctly positioned for reading the source
+/// location entry with the given ID.
+llvm::BitstreamCursor &ASTReader::SLocCursorForID(unsigned ID) {
+  assert(ID != 0 && ID <= TotalNumSLocEntries &&
+         "SLocCursorForID should only be called for real IDs.");
+
+  ID -= 1;
+  PerFileData *F = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    F = Chain[N - I - 1];
+    if (ID < F->LocalNumSLocEntries)
+      break;
+    ID -= F->LocalNumSLocEntries;
+  }
+  assert(F && F->LocalNumSLocEntries > ID && "Chain corrupted");
+
+  F->SLocEntryCursor.JumpToBit(F->SLocOffsets[ID]);
+  return F->SLocEntryCursor;
+}
+
 /// \brief Read in the source location entry with the given ID.
-PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
+ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(unsigned ID) {
   if (ID == 0)
     return Success;
 
   if (ID > TotalNumSLocEntries) {
-    Error("source location entry ID out-of-range for PCH file");
+    Error("source location entry ID out-of-range for AST file");
     return Failure;
   }
 
+  llvm::BitstreamCursor &SLocEntryCursor = SLocCursorForID(ID);
+
   ++NumSLocEntriesRead;
-  SLocEntryCursor.JumpToBit(SLocOffsets[ID - 1]);
   unsigned Code = SLocEntryCursor.ReadCode();
   if (Code == llvm::bitc::END_BLOCK ||
       Code == llvm::bitc::ENTER_SUBBLOCK ||
       Code == llvm::bitc::DEFINE_ABBREV) {
-    Error("incorrectly-formatted source location entry in PCH file");
+    Error("incorrectly-formatted source location entry in AST file");
     return Failure;
   }
 
@@ -968,17 +1188,17 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
   unsigned BlobLen;
   switch (SLocEntryCursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
   default:
-    Error("incorrectly-formatted source location entry in PCH file");
+    Error("incorrectly-formatted source location entry in AST file");
     return Failure;
 
-  case pch::SM_SLOC_FILE_ENTRY: {
+  case SM_SLOC_FILE_ENTRY: {
     std::string Filename(BlobStart, BlobStart + BlobLen);
     MaybeAddSystemRootToFilename(Filename);
     const FileEntry *File = FileMgr.getFile(Filename);
     if (File == 0) {
       std::string ErrorStr = "could not find file '";
       ErrorStr += Filename;
-      ErrorStr += "' referenced by PCH file";
+      ErrorStr += "' referenced by AST file";
       Error(ErrorStr.c_str());
       return Failure;
     }
@@ -988,14 +1208,15 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
       return Failure;
     }
 
-    if ((off_t)Record[4] != File->getSize()
+    if (!DisableValidation &&
+        ((off_t)Record[4] != File->getSize()
 #if !defined(LLVM_ON_WIN32)
         // In our regression testing, the Windows file system seems to
         // have inconsistent modification times that sometimes
         // erroneously trigger this error-handling path.
-        || (time_t)Record[5] != File->getModificationTime()
+         || (time_t)Record[5] != File->getModificationTime()
 #endif
-        ) {
+        )) {
       Diag(diag::err_fe_pch_file_modified)
         << Filename;
       return Failure;
@@ -1020,7 +1241,7 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
     break;
   }
 
-  case pch::SM_SLOC_BUFFER_ENTRY: {
+  case SM_SLOC_BUFFER_ENTRY: {
     const char *Name = BlobStart;
     unsigned Offset = Record[0];
     unsigned Code = SLocEntryCursor.ReadCode();
@@ -1028,8 +1249,8 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
     unsigned RecCode
       = SLocEntryCursor.ReadRecord(Code, Record, &BlobStart, &BlobLen);
 
-    if (RecCode != pch::SM_SLOC_BUFFER_BLOB) {
-      Error("PCH record has invalid code");
+    if (RecCode != SM_SLOC_BUFFER_BLOB) {
+      Error("AST record has invalid code");
       return Failure;
     }
 
@@ -1049,7 +1270,7 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
     break;
   }
 
-  case pch::SM_SLOC_INSTANTIATION_ENTRY: {
+  case SM_SLOC_INSTANTIATION_ENTRY: {
     SourceLocation SpellingLoc
       = SourceLocation::getFromRawEncoding(Record[1]);
     SourceMgr.createInstantiationLoc(SpellingLoc,
@@ -1068,10 +1289,10 @@ PCHReader::PCHReadResult PCHReader::ReadSLocEntryRecord(unsigned ID) {
 /// ReadBlockAbbrevs - Enter a subblock of the specified BlockID with the
 /// specified cursor.  Read the abbreviations that are at the top of the block
 /// and then leave the cursor pointing into the block.
-bool PCHReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
+bool ASTReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
                                  unsigned BlockID) {
   if (Cursor.EnterSubBlock(BlockID)) {
-    Error("malformed block record in PCH file");
+    Error("malformed block record in AST file");
     return Failure;
   }
 
@@ -1085,7 +1306,7 @@ bool PCHReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
   }
 }
 
-void PCHReader::ReadMacroRecord(uint64_t Offset) {
+void ASTReader::ReadMacroRecord(llvm::BitstreamCursor &Stream, uint64_t Offset){
   assert(PP && "Forgot to set Preprocessor ?");
 
   // Keep track of where we are in the stream, then jump back there
@@ -1107,7 +1328,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       // No known subblocks, always skip them.
       Stream.ReadSubBlockID();
       if (Stream.SkipBlock()) {
-        Error("malformed block record in PCH file");
+        Error("malformed block record in AST file");
         return;
       }
       continue;
@@ -1120,11 +1341,11 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
 
     // Read a record.
     Record.clear();
-    pch::PreprocessorRecordTypes RecType =
-      (pch::PreprocessorRecordTypes)Stream.ReadRecord(Code, Record);
+    PreprocessorRecordTypes RecType =
+      (PreprocessorRecordTypes)Stream.ReadRecord(Code, Record);
     switch (RecType) {
-    case pch::PP_MACRO_OBJECT_LIKE:
-    case pch::PP_MACRO_FUNCTION_LIKE: {
+    case PP_MACRO_OBJECT_LIKE:
+    case PP_MACRO_FUNCTION_LIKE: {
       // If we already have a macro, that means that we've hit the end
       // of the definition of the macro we were looking for. We're
       // done.
@@ -1133,7 +1354,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
 
       IdentifierInfo *II = DecodeIdentifierInfo(Record[0]);
       if (II == 0) {
-        Error("macro must have a name in PCH file");
+        Error("macro must have a name in AST file");
         return;
       }
       SourceLocation Loc = SourceLocation::getFromRawEncoding(Record[1]);
@@ -1141,9 +1362,10 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
 
       MacroInfo *MI = PP->AllocateMacroInfo(Loc);
       MI->setIsUsed(isUsed);
+      MI->setIsFromAST();
 
       unsigned NextIndex = 3;
-      if (RecType == pch::PP_MACRO_FUNCTION_LIKE) {
+      if (RecType == PP_MACRO_FUNCTION_LIKE) {
         // Decode function-like macro info.
         bool isC99VarArgs = Record[3];
         bool isGNUVarArgs = Record[4];
@@ -1178,7 +1400,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       break;
     }
 
-    case pch::PP_TOKEN: {
+    case PP_TOKEN: {
       // If we see a TOKEN before a PP_MACRO_*, then the file is
       // erroneous, just pretend we didn't see this.
       if (Macro == 0) break;
@@ -1195,7 +1417,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       break;
     }
         
-    case pch::PP_MACRO_INSTANTIATION: {
+    case PP_MACRO_INSTANTIATION: {
       // If we already have a macro, that means that we've hit the end
       // of the definition of the macro we were looking for. We're
       // done.
@@ -1203,7 +1425,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
         return;
       
       if (!PP->getPreprocessingRecord()) {
-        Error("missing preprocessing record in PCH file");
+        Error("missing preprocessing record in AST file");
         return;
       }
         
@@ -1221,7 +1443,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
       return;
     }
 
-    case pch::PP_MACRO_DEFINITION: {
+    case PP_MACRO_DEFINITION: {
       // If we already have a macro, that means that we've hit the end
       // of the definition of the macro we were looking for. We're
       // done.
@@ -1229,7 +1451,7 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
         return;
       
       if (!PP->getPreprocessingRecord()) {
-        Error("missing preprocessing record in PCH file");
+        Error("missing preprocessing record in AST file");
         return;
       }
       
@@ -1256,81 +1478,97 @@ void PCHReader::ReadMacroRecord(uint64_t Offset) {
   }
 }
 
-void PCHReader::ReadDefinedMacros() {
-  // If there was no preprocessor block, do nothing.
-  if (!MacroCursor.getBitStreamReader())
-    return;
+void ASTReader::ReadDefinedMacros() {
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    llvm::BitstreamCursor &MacroCursor = Chain[N - I - 1]->MacroCursor;
 
-  llvm::BitstreamCursor Cursor = MacroCursor;
-  if (Cursor.EnterSubBlock(pch::PREPROCESSOR_BLOCK_ID)) {
-    Error("malformed preprocessor block record in PCH file");
-    return;
-  }
+    // If there was no preprocessor block, skip this file.
+    if (!MacroCursor.getBitStreamReader())
+      continue;
 
-  RecordData Record;
-  while (true) {
-    unsigned Code = Cursor.ReadCode();
-    if (Code == llvm::bitc::END_BLOCK) {
-      if (Cursor.ReadBlockEnd())
-        Error("error at end of preprocessor block in PCH file");
+    llvm::BitstreamCursor Cursor = MacroCursor;
+    if (Cursor.EnterSubBlock(PREPROCESSOR_BLOCK_ID)) {
+      Error("malformed preprocessor block record in AST file");
       return;
     }
 
-    if (Code == llvm::bitc::ENTER_SUBBLOCK) {
-      // No known subblocks, always skip them.
-      Cursor.ReadSubBlockID();
-      if (Cursor.SkipBlock()) {
-        Error("malformed block record in PCH file");
-        return;
+    RecordData Record;
+    while (true) {
+      unsigned Code = Cursor.ReadCode();
+      if (Code == llvm::bitc::END_BLOCK) {
+        if (Cursor.ReadBlockEnd()) {
+          Error("error at end of preprocessor block in AST file");
+          return;
+        }
+        break;
       }
-      continue;
-    }
 
-    if (Code == llvm::bitc::DEFINE_ABBREV) {
-      Cursor.ReadAbbrevRecord();
-      continue;
-    }
+      if (Code == llvm::bitc::ENTER_SUBBLOCK) {
+        // No known subblocks, always skip them.
+        Cursor.ReadSubBlockID();
+        if (Cursor.SkipBlock()) {
+          Error("malformed block record in AST file");
+          return;
+        }
+        continue;
+      }
 
-    // Read a record.
-    const char *BlobStart;
-    unsigned BlobLen;
-    Record.clear();
-    switch (Cursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
-    default:  // Default behavior: ignore.
-      break;
+      if (Code == llvm::bitc::DEFINE_ABBREV) {
+        Cursor.ReadAbbrevRecord();
+        continue;
+      }
 
-    case pch::PP_MACRO_OBJECT_LIKE:
-    case pch::PP_MACRO_FUNCTION_LIKE:
-      DecodeIdentifierInfo(Record[0]);
-      break;
+      // Read a record.
+      const char *BlobStart;
+      unsigned BlobLen;
+      Record.clear();
+      switch (Cursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
+      default:  // Default behavior: ignore.
+        break;
 
-    case pch::PP_TOKEN:
-      // Ignore tokens.
-      break;
+      case PP_MACRO_OBJECT_LIKE:
+      case PP_MACRO_FUNCTION_LIKE:
+        DecodeIdentifierInfo(Record[0]);
+        break;
+
+      case PP_TOKEN:
+        // Ignore tokens.
+        break;
         
-    case pch::PP_MACRO_INSTANTIATION:
-    case pch::PP_MACRO_DEFINITION:
-      // Read the macro record.
-      ReadMacroRecord(Cursor.GetCurrentBitNo());
-      break;
+      case PP_MACRO_INSTANTIATION:
+      case PP_MACRO_DEFINITION:
+        // Read the macro record.
+        ReadMacroRecord(Chain[N - I - 1]->Stream, Cursor.GetCurrentBitNo());
+        break;
+      }
     }
   }
 }
 
-MacroDefinition *PCHReader::getMacroDefinition(pch::IdentID ID) {
+MacroDefinition *ASTReader::getMacroDefinition(IdentID ID) {
   if (ID == 0 || ID >= MacroDefinitionsLoaded.size())
     return 0;
-  
-  if (!MacroDefinitionsLoaded[ID])
-    ReadMacroRecord(MacroDefinitionOffsets[ID]);
-    
+
+  if (!MacroDefinitionsLoaded[ID]) {
+    unsigned Index = ID;
+    for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+      PerFileData &F = *Chain[N - I - 1];
+      if (Index < F.LocalNumMacroDefinitions) {
+        ReadMacroRecord(F.Stream, F.MacroDefinitionOffsets[Index]);
+        break;
+      }
+      Index -= F.LocalNumMacroDefinitions;
+    }
+    assert(MacroDefinitionsLoaded[ID] && "Broken chain");
+  }
+
   return MacroDefinitionsLoaded[ID];
 }
 
 /// \brief If we are loading a relocatable PCH file, and the filename is
 /// not an absolute path, add the system root to the beginning of the file
 /// name.
-void PCHReader::MaybeAddSystemRootToFilename(std::string &Filename) {
+void ASTReader::MaybeAddSystemRootToFilename(std::string &Filename) {
   // If this is not a relocatable PCH file, there's nothing to do.
   if (!RelocatablePCH)
     return;
@@ -1351,20 +1589,23 @@ void PCHReader::MaybeAddSystemRootToFilename(std::string &Filename) {
   Filename.insert(Filename.begin(), isysroot, isysroot + Length);
 }
 
-PCHReader::PCHReadResult
-PCHReader::ReadPCHBlock() {
-  if (Stream.EnterSubBlock(pch::PCH_BLOCK_ID)) {
-    Error("malformed block record in PCH file");
+ASTReader::ASTReadResult
+ASTReader::ReadASTBlock(PerFileData &F) {
+  llvm::BitstreamCursor &Stream = F.Stream;
+
+  if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
+    Error("malformed block record in AST file");
     return Failure;
   }
 
-  // Read all of the records and blocks for the PCH file.
+  // Read all of the records and blocks for the ASt file.
   RecordData Record;
+  bool First = true;
   while (!Stream.AtEndOfStream()) {
     unsigned Code = Stream.ReadCode();
     if (Code == llvm::bitc::END_BLOCK) {
       if (Stream.ReadBlockEnd()) {
-        Error("error at end of module block in PCH file");
+        Error("error at end of module block in AST file");
         return Failure;
       }
 
@@ -1373,38 +1614,38 @@ PCHReader::ReadPCHBlock() {
 
     if (Code == llvm::bitc::ENTER_SUBBLOCK) {
       switch (Stream.ReadSubBlockID()) {
-      case pch::DECLTYPES_BLOCK_ID:
+      case DECLTYPES_BLOCK_ID:
         // We lazily load the decls block, but we want to set up the
         // DeclsCursor cursor to point into it.  Clone our current bitcode
         // cursor to it, enter the block and read the abbrevs in that block.
         // With the main cursor, we just skip over it.
-        DeclsCursor = Stream;
+        F.DeclsCursor = Stream;
         if (Stream.SkipBlock() ||  // Skip with the main cursor.
             // Read the abbrevs.
-            ReadBlockAbbrevs(DeclsCursor, pch::DECLTYPES_BLOCK_ID)) {
-          Error("malformed block record in PCH file");
+            ReadBlockAbbrevs(F.DeclsCursor, DECLTYPES_BLOCK_ID)) {
+          Error("malformed block record in AST file");
           return Failure;
         }
         break;
 
-      case pch::PREPROCESSOR_BLOCK_ID:
-        MacroCursor = Stream;
+      case PREPROCESSOR_BLOCK_ID:
+        F.MacroCursor = Stream;
         if (PP)
           PP->setExternalSource(this);
 
         if (Stream.SkipBlock()) {
-          Error("malformed block record in PCH file");
+          Error("malformed block record in AST file");
           return Failure;
         }
         break;
 
-      case pch::SOURCE_MANAGER_BLOCK_ID:
-        switch (ReadSourceManagerBlock()) {
+      case SOURCE_MANAGER_BLOCK_ID:
+        switch (ReadSourceManagerBlock(F)) {
         case Success:
           break;
 
         case Failure:
-          Error("malformed source manager block in PCH file");
+          Error("malformed source manager block in AST file");
           return Failure;
 
         case IgnorePCH:
@@ -1412,6 +1653,7 @@ PCHReader::ReadPCHBlock() {
         }
         break;
       }
+      First = false;
       continue;
     }
 
@@ -1424,37 +1666,14 @@ PCHReader::ReadPCHBlock() {
     Record.clear();
     const char *BlobStart = 0;
     unsigned BlobLen = 0;
-    switch ((pch::PCHRecordTypes)Stream.ReadRecord(Code, Record,
+    switch ((ASTRecordTypes)Stream.ReadRecord(Code, Record,
                                                    &BlobStart, &BlobLen)) {
     default:  // Default behavior: ignore.
       break;
 
-    case pch::TYPE_OFFSET:
-      if (!TypesLoaded.empty()) {
-        Error("duplicate TYPE_OFFSET record in PCH file");
-        return Failure;
-      }
-      TypeOffsets = (const uint32_t *)BlobStart;
-      TypesLoaded.resize(Record[0]);
-      break;
-
-    case pch::DECL_OFFSET:
-      if (!DeclsLoaded.empty()) {
-        Error("duplicate DECL_OFFSET record in PCH file");
-        return Failure;
-      }
-      DeclOffsets = (const uint32_t *)BlobStart;
-      DeclsLoaded.resize(Record[0]);
-      break;
-
-    case pch::LANGUAGE_OPTIONS:
-      if (ParseLanguageOptions(Record))
-        return IgnorePCH;
-      break;
-
-    case pch::METADATA: {
-      if (Record[0] != pch::VERSION_MAJOR) {
-        Diag(Record[0] < pch::VERSION_MAJOR? diag::warn_pch_version_too_old
+    case METADATA: {
+      if (Record[0] != VERSION_MAJOR && !DisableValidation) {
+        Diag(Record[0] < VERSION_MAJOR? diag::warn_pch_version_too_old
                                            : diag::warn_pch_version_too_new);
         return IgnorePCH;
       }
@@ -1468,195 +1687,419 @@ PCHReader::ReadPCHBlock() {
       break;
     }
 
-    case pch::IDENTIFIER_TABLE:
-      IdentifierTableData = BlobStart;
-      if (Record[0]) {
-        IdentifierLookupTable
-          = PCHIdentifierLookupTable::Create(
-                        (const unsigned char *)IdentifierTableData + Record[0],
-                        (const unsigned char *)IdentifierTableData,
-                        PCHIdentifierLookupTrait(*this));
-        if (PP)
-          PP->getIdentifierTable().setExternalIdentifierLookup(this);
+    case CHAINED_METADATA: {
+      if (!First) {
+        Error("CHAINED_METADATA is not first record in block");
+        return Failure;
+      }
+      if (Record[0] != VERSION_MAJOR && !DisableValidation) {
+        Diag(Record[0] < VERSION_MAJOR? diag::warn_pch_version_too_old
+                                           : diag::warn_pch_version_too_new);
+        return IgnorePCH;
+      }
+
+      // Load the chained file.
+      switch(ReadASTCore(llvm::StringRef(BlobStart, BlobLen))) {
+      case Failure: return Failure;
+        // If we have to ignore the dependency, we'll have to ignore this too.
+      case IgnorePCH: return IgnorePCH;
+      case Success: break;
       }
       break;
+    }
 
-    case pch::IDENTIFIER_OFFSET:
-      if (!IdentifiersLoaded.empty()) {
-        Error("duplicate IDENTIFIER_OFFSET record in PCH file");
+    case TYPE_OFFSET:
+      if (F.LocalNumTypes != 0) {
+        Error("duplicate TYPE_OFFSET record in AST file");
         return Failure;
       }
-      IdentifierOffsets = (const uint32_t *)BlobStart;
-      IdentifiersLoaded.resize(Record[0]);
-      if (PP)
-        PP->getHeaderSearchInfo().SetExternalLookup(this);
+      F.TypeOffsets = (const uint32_t *)BlobStart;
+      F.LocalNumTypes = Record[0];
       break;
 
-    case pch::EXTERNAL_DEFINITIONS:
-      if (!ExternalDefinitions.empty()) {
-        Error("duplicate EXTERNAL_DEFINITIONS record in PCH file");
+    case DECL_OFFSET:
+      if (F.LocalNumDecls != 0) {
+        Error("duplicate DECL_OFFSET record in AST file");
         return Failure;
       }
-      ExternalDefinitions.swap(Record);
+      F.DeclOffsets = (const uint32_t *)BlobStart;
+      F.LocalNumDecls = Record[0];
       break;
 
-    case pch::SPECIAL_TYPES:
-      SpecialTypes.swap(Record);
+    case TU_UPDATE_LEXICAL: {
+      DeclContextInfo Info = {
+        /* No visible information */ 0,
+        reinterpret_cast<const DeclID *>(BlobStart),
+        BlobLen / sizeof(DeclID)
+      };
+      DeclContextOffsets[Context->getTranslationUnitDecl()].push_back(Info);
       break;
+    }
 
-    case pch::STATISTICS:
-      TotalNumStatements = Record[0];
-      TotalNumMacros = Record[1];
-      TotalLexicalDeclContexts = Record[2];
-      TotalVisibleDeclContexts = Record[3];
+    case UPDATE_VISIBLE: {
+      serialization::DeclID ID = Record[0];
+      void *Table = ASTDeclContextNameLookupTable::Create(
+                        (const unsigned char *)BlobStart + Record[1],
+                        (const unsigned char *)BlobStart,
+                        ASTDeclContextNameLookupTrait(*this));
+      if (ID == 1) { // Is it the TU?
+        DeclContextInfo Info = {
+          Table, /* No lexical inforamtion */ 0, 0
+        };
+        DeclContextOffsets[Context->getTranslationUnitDecl()].push_back(Info);
+      } else
+        PendingVisibleUpdates[ID].push_back(Table);
       break;
+    }
 
-    case pch::TENTATIVE_DEFINITIONS:
-      if (!TentativeDefinitions.empty()) {
-        Error("duplicate TENTATIVE_DEFINITIONS record in PCH file");
-        return Failure;
+    case REDECLS_UPDATE_LATEST: {
+      assert(Record.size() % 2 == 0 && "Expected pairs of DeclIDs");
+      for (unsigned i = 0, e = Record.size(); i < e; i += 2) {
+        DeclID First = Record[i], Latest = Record[i+1];
+        assert((FirstLatestDeclIDs.find(First) == FirstLatestDeclIDs.end() ||
+                Latest > FirstLatestDeclIDs[First]) &&
+               "The new latest is supposed to come after the previous latest");
+        FirstLatestDeclIDs[First] = Latest;
       }
-      TentativeDefinitions.swap(Record);
       break;
+    }
 
-    case pch::UNUSED_STATIC_FUNCS:
-      if (!UnusedStaticFuncs.empty()) {
-        Error("duplicate UNUSED_STATIC_FUNCS record in PCH file");
-        return Failure;
+    case LANGUAGE_OPTIONS:
+      if (ParseLanguageOptions(Record) && !DisableValidation)
+        return IgnorePCH;
+      break;
+
+    case IDENTIFIER_TABLE:
+      F.IdentifierTableData = BlobStart;
+      if (Record[0]) {
+        F.IdentifierLookupTable
+          = ASTIdentifierLookupTable::Create(
+                       (const unsigned char *)F.IdentifierTableData + Record[0],
+                       (const unsigned char *)F.IdentifierTableData,
+                       ASTIdentifierLookupTrait(*this, F.Stream));
+        if (PP)
+          PP->getIdentifierTable().setExternalIdentifierLookup(this);
       }
-      UnusedStaticFuncs.swap(Record);
       break;
 
-    case pch::LOCALLY_SCOPED_EXTERNAL_DECLS:
-      if (!LocallyScopedExternalDecls.empty()) {
-        Error("duplicate LOCALLY_SCOPED_EXTERNAL_DECLS record in PCH file");
+    case IDENTIFIER_OFFSET:
+      if (F.LocalNumIdentifiers != 0) {
+        Error("duplicate IDENTIFIER_OFFSET record in AST file");
         return Failure;
       }
-      LocallyScopedExternalDecls.swap(Record);
+      F.IdentifierOffsets = (const uint32_t *)BlobStart;
+      F.LocalNumIdentifiers = Record[0];
+      break;
+
+    case EXTERNAL_DEFINITIONS:
+      // Optimization for the first block.
+      if (ExternalDefinitions.empty())
+        ExternalDefinitions.swap(Record);
+      else
+        ExternalDefinitions.insert(ExternalDefinitions.end(),
+                                   Record.begin(), Record.end());
+      break;
+
+    case SPECIAL_TYPES:
+      // Optimization for the first block
+      if (SpecialTypes.empty())
+        SpecialTypes.swap(Record);
+      else
+        SpecialTypes.insert(SpecialTypes.end(), Record.begin(), Record.end());
+      break;
+
+    case STATISTICS:
+      TotalNumStatements += Record[0];
+      TotalNumMacros += Record[1];
+      TotalLexicalDeclContexts += Record[2];
+      TotalVisibleDeclContexts += Record[3];
+      break;
+
+    case TENTATIVE_DEFINITIONS:
+      // Optimization for the first block.
+      if (TentativeDefinitions.empty())
+        TentativeDefinitions.swap(Record);
+      else
+        TentativeDefinitions.insert(TentativeDefinitions.end(),
+                                    Record.begin(), Record.end());
       break;
 
-    case pch::SELECTOR_OFFSETS:
-      SelectorOffsets = (const uint32_t *)BlobStart;
-      TotalNumSelectors = Record[0];
-      SelectorsLoaded.resize(TotalNumSelectors);
+    case UNUSED_FILESCOPED_DECLS:
+      // Optimization for the first block.
+      if (UnusedFileScopedDecls.empty())
+        UnusedFileScopedDecls.swap(Record);
+      else
+        UnusedFileScopedDecls.insert(UnusedFileScopedDecls.end(),
+                                     Record.begin(), Record.end());
       break;
 
-    case pch::METHOD_POOL:
-      MethodPoolLookupTableData = (const unsigned char *)BlobStart;
+    case WEAK_UNDECLARED_IDENTIFIERS:
+      // Later blocks overwrite earlier ones.
+      WeakUndeclaredIdentifiers.swap(Record);
+      break;
+
+    case LOCALLY_SCOPED_EXTERNAL_DECLS:
+      // Optimization for the first block.
+      if (LocallyScopedExternalDecls.empty())
+        LocallyScopedExternalDecls.swap(Record);
+      else
+        LocallyScopedExternalDecls.insert(LocallyScopedExternalDecls.end(),
+                                          Record.begin(), Record.end());
+      break;
+
+    case SELECTOR_OFFSETS:
+      F.SelectorOffsets = (const uint32_t *)BlobStart;
+      F.LocalNumSelectors = Record[0];
+      break;
+
+    case METHOD_POOL:
+      F.SelectorLookupTableData = (const unsigned char *)BlobStart;
       if (Record[0])
-        MethodPoolLookupTable
-          = PCHMethodPoolLookupTable::Create(
-                        MethodPoolLookupTableData + Record[0],
-                        MethodPoolLookupTableData,
-                        PCHMethodPoolLookupTrait(*this));
-      TotalSelectorsInMethodPool = Record[1];
+        F.SelectorLookupTable
+          = ASTSelectorLookupTable::Create(
+                        F.SelectorLookupTableData + Record[0],
+                        F.SelectorLookupTableData,
+                        ASTSelectorLookupTrait(*this));
+      TotalNumMethodPoolEntries += Record[1];
       break;
 
-    case pch::PP_COUNTER_VALUE:
+    case REFERENCED_SELECTOR_POOL: {
+      ReferencedSelectorsData.insert(ReferencedSelectorsData.end(),
+          Record.begin(), Record.end());
+      break;
+    }
+
+    case PP_COUNTER_VALUE:
       if (!Record.empty() && Listener)
         Listener->ReadCounter(Record[0]);
       break;
 
-    case pch::SOURCE_LOCATION_OFFSETS:
-      SLocOffsets = (const uint32_t *)BlobStart;
-      TotalNumSLocEntries = Record[0];
+    case SOURCE_LOCATION_OFFSETS:
+      F.SLocOffsets = (const uint32_t *)BlobStart;
+      F.LocalNumSLocEntries = Record[0];
+      // We cannot delay this until the entire chain is loaded, because then
+      // source location preloads would also have to be delayed.
+      // FIXME: Is there a reason not to do that?
+      TotalNumSLocEntries += F.LocalNumSLocEntries;
       SourceMgr.PreallocateSLocEntries(this, TotalNumSLocEntries, Record[1]);
       break;
 
-    case pch::SOURCE_LOCATION_PRELOADS:
+    case SOURCE_LOCATION_PRELOADS:
       for (unsigned I = 0, N = Record.size(); I != N; ++I) {
-        PCHReadResult Result = ReadSLocEntryRecord(Record[I]);
+        ASTReadResult Result = ReadSLocEntryRecord(Record[I]);
         if (Result != Success)
           return Result;
       }
       break;
 
-    case pch::STAT_CACHE: {
-      PCHStatCache *MyStatCache =
-        new PCHStatCache((const unsigned char *)BlobStart + Record[0],
+    case STAT_CACHE: {
+      ASTStatCache *MyStatCache =
+        new ASTStatCache((const unsigned char *)BlobStart + Record[0],
                          (const unsigned char *)BlobStart,
                          NumStatHits, NumStatMisses);
       FileMgr.addStatCache(MyStatCache);
-      StatCache = MyStatCache;
+      F.StatCache = MyStatCache;
       break;
     }
 
-    case pch::EXT_VECTOR_DECLS:
-      if (!ExtVectorDecls.empty()) {
-        Error("duplicate EXT_VECTOR_DECLS record in PCH file");
-        return Failure;
-      }
-      ExtVectorDecls.swap(Record);
+    case EXT_VECTOR_DECLS:
+      // Optimization for the first block.
+      if (ExtVectorDecls.empty())
+        ExtVectorDecls.swap(Record);
+      else
+        ExtVectorDecls.insert(ExtVectorDecls.end(),
+                              Record.begin(), Record.end());
       break;
 
-    case pch::VTABLE_USES:
-      if (!VTableUses.empty()) {
-        Error("duplicate VTABLE_USES record in PCH file");
-        return Failure;
-      }
+    case VTABLE_USES:
+      // Later tables overwrite earlier ones.
       VTableUses.swap(Record);
       break;
 
-    case pch::DYNAMIC_CLASSES:
-      if (!DynamicClasses.empty()) {
-        Error("duplicate DYNAMIC_CLASSES record in PCH file");
-        return Failure;
-      }
-      DynamicClasses.swap(Record);
+    case DYNAMIC_CLASSES:
+      // Optimization for the first block.
+      if (DynamicClasses.empty())
+        DynamicClasses.swap(Record);
+      else
+        DynamicClasses.insert(DynamicClasses.end(),
+                              Record.begin(), Record.end());
+      break;
+
+    case PENDING_IMPLICIT_INSTANTIATIONS:
+      // Optimization for the first block.
+      if (PendingInstantiations.empty())
+        PendingInstantiations.swap(Record);
+      else
+        PendingInstantiations.insert(PendingInstantiations.end(),
+                                     Record.begin(), Record.end());
       break;
 
-    case pch::ORIGINAL_FILE_NAME:
+    case SEMA_DECL_REFS:
+      // Later tables overwrite earlier ones.
+      SemaDeclRefs.swap(Record);
+      break;
+
+    case ORIGINAL_FILE_NAME:
+      // The primary AST will be the last to get here, so it will be the one
+      // that's used.
       ActualOriginalFileName.assign(BlobStart, BlobLen);
       OriginalFileName = ActualOriginalFileName;
       MaybeAddSystemRootToFilename(OriginalFileName);
       break;
 
-    case pch::VERSION_CONTROL_BRANCH_REVISION: {
+    case VERSION_CONTROL_BRANCH_REVISION: {
       const std::string &CurBranch = getClangFullRepositoryVersion();
-      llvm::StringRef PCHBranch(BlobStart, BlobLen);
-      if (llvm::StringRef(CurBranch) != PCHBranch) {
-        Diag(diag::warn_pch_different_branch) << PCHBranch << CurBranch;
+      llvm::StringRef ASTBranch(BlobStart, BlobLen);
+      if (llvm::StringRef(CurBranch) != ASTBranch && !DisableValidation) {
+        Diag(diag::warn_pch_different_branch) << ASTBranch << CurBranch;
         return IgnorePCH;
       }
       break;
     }
-        
-    case pch::MACRO_DEFINITION_OFFSETS:
-      MacroDefinitionOffsets = (const uint32_t *)BlobStart;
-      if (PP) {
-        if (!PP->getPreprocessingRecord())
-          PP->createPreprocessingRecord();
-        PP->getPreprocessingRecord()->SetExternalSource(*this, Record[0]);
-      } else {
-        NumPreallocatedPreprocessingEntities = Record[0];
+
+    case MACRO_DEFINITION_OFFSETS:
+      F.MacroDefinitionOffsets = (const uint32_t *)BlobStart;
+      F.NumPreallocatedPreprocessingEntities = Record[0];
+      F.LocalNumMacroDefinitions = Record[1];
+      break;
+
+    case DECL_REPLACEMENTS: {
+      if (Record.size() % 2 != 0) {
+        Error("invalid DECL_REPLACEMENTS block in AST file");
+        return Failure;
       }
-       
-      MacroDefinitionsLoaded.resize(Record[1]);
+      for (unsigned I = 0, N = Record.size(); I != N; I += 2)
+        ReplacedDecls[static_cast<DeclID>(Record[I])] =
+            std::make_pair(&F, Record[I+1]);
       break;
     }
+
+    case ADDITIONAL_TEMPLATE_SPECIALIZATIONS: {
+      AdditionalTemplateSpecializations &ATS =
+          AdditionalTemplateSpecializationsPending[Record[0]];
+      ATS.insert(ATS.end(), Record.begin()+1, Record.end());
+      break;
+    }
+    }
+    First = false;
   }
-  Error("premature end of bitstream in PCH file");
+  Error("premature end of bitstream in AST file");
   return Failure;
 }
 
-PCHReader::PCHReadResult PCHReader::ReadPCH(const std::string &FileName) {
-  // Set the PCH file name.
-  this->FileName = FileName;
+ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName) {
+  switch(ReadASTCore(FileName)) {
+  case Failure: return Failure;
+  case IgnorePCH: return IgnorePCH;
+  case Success: break;
+  }
+
+  // Here comes stuff that we only do once the entire chain is loaded.
+
+  // Allocate space for loaded identifiers, decls and types.
+  unsigned TotalNumIdentifiers = 0, TotalNumTypes = 0, TotalNumDecls = 0,
+           TotalNumPreallocatedPreprocessingEntities = 0, TotalNumMacroDefs = 0,
+           TotalNumSelectors = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    TotalNumIdentifiers += Chain[I]->LocalNumIdentifiers;
+    TotalNumTypes += Chain[I]->LocalNumTypes;
+    TotalNumDecls += Chain[I]->LocalNumDecls;
+    TotalNumPreallocatedPreprocessingEntities +=
+        Chain[I]->NumPreallocatedPreprocessingEntities;
+    TotalNumMacroDefs += Chain[I]->LocalNumMacroDefinitions;
+    TotalNumSelectors += Chain[I]->LocalNumSelectors;
+  }
+  IdentifiersLoaded.resize(TotalNumIdentifiers);
+  TypesLoaded.resize(TotalNumTypes);
+  DeclsLoaded.resize(TotalNumDecls);
+  MacroDefinitionsLoaded.resize(TotalNumMacroDefs);
+  if (PP) {
+    if (TotalNumIdentifiers > 0)
+      PP->getHeaderSearchInfo().SetExternalLookup(this);
+    if (TotalNumPreallocatedPreprocessingEntities > 0) {
+      if (!PP->getPreprocessingRecord())
+        PP->createPreprocessingRecord();
+      PP->getPreprocessingRecord()->SetExternalSource(*this,
+                                     TotalNumPreallocatedPreprocessingEntities);
+    }
+  }
+  SelectorsLoaded.resize(TotalNumSelectors);
+
+  // Check the predefines buffers.
+  if (!DisableValidation && CheckPredefinesBuffers())
+    return IgnorePCH;
+
+  if (PP) {
+    // Initialization of keywords and pragmas occurs before the
+    // AST file is read, so there may be some identifiers that were
+    // loaded into the IdentifierTable before we intercepted the
+    // creation of identifiers. Iterate through the list of known
+    // identifiers and determine whether we have to establish
+    // preprocessor definitions or top-level identifier declaration
+    // chains for those identifiers.
+    //
+    // We copy the IdentifierInfo pointers to a small vector first,
+    // since de-serializing declarations or macro definitions can add
+    // new entries into the identifier table, invalidating the
+    // iterators.
+    llvm::SmallVector<IdentifierInfo *, 128> Identifiers;
+    for (IdentifierTable::iterator Id = PP->getIdentifierTable().begin(),
+                                IdEnd = PP->getIdentifierTable().end();
+         Id != IdEnd; ++Id)
+      Identifiers.push_back(Id->second);
+    // We need to search the tables in all files.
+    for (unsigned J = 0, M = Chain.size(); J != M; ++J) {
+      ASTIdentifierLookupTable *IdTable
+        = (ASTIdentifierLookupTable *)Chain[J]->IdentifierLookupTable;
+      // Not all AST files necessarily have identifier tables, only the useful
+      // ones.
+      if (!IdTable)
+        continue;
+      for (unsigned I = 0, N = Identifiers.size(); I != N; ++I) {
+        IdentifierInfo *II = Identifiers[I];
+        // Look in the on-disk hash tables for an entry for this identifier
+        ASTIdentifierLookupTrait Info(*this, Chain[J]->Stream, II);
+        std::pair<const char*,unsigned> Key(II->getNameStart(),II->getLength());
+        ASTIdentifierLookupTable::iterator Pos = IdTable->find(Key, &Info);
+        if (Pos == IdTable->end())
+          continue;
+
+        // Dereferencing the iterator has the effect of populating the
+        // IdentifierInfo node with the various declarations it needs.
+        (void)*Pos;
+      }
+    }
+  }
+
+  if (Context)
+    InitializeContext(*Context);
+
+  return Success;
+}
+
+ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName) {
+  Chain.push_back(new PerFileData());
+  PerFileData &F = *Chain.back();
+
+  // Set the AST file name.
+  F.FileName = FileName;
 
-  // Open the PCH file.
+  // Open the AST file.
   //
   // FIXME: This shouldn't be here, we should just take a raw_ostream.
   std::string ErrStr;
-  Buffer.reset(llvm::MemoryBuffer::getFileOrSTDIN(FileName, &ErrStr));
-  if (!Buffer) {
+  F.Buffer.reset(llvm::MemoryBuffer::getFileOrSTDIN(FileName, &ErrStr));
+  if (!F.Buffer) {
     Error(ErrStr.c_str());
     return IgnorePCH;
   }
 
   // Initialize the stream
-  StreamFile.init((const unsigned char *)Buffer->getBufferStart(),
-                  (const unsigned char *)Buffer->getBufferEnd());
-  Stream.init(StreamFile);
+  F.StreamFile.init((const unsigned char *)F.Buffer->getBufferStart(),
+                    (const unsigned char *)F.Buffer->getBufferEnd());
+  llvm::BitstreamCursor &Stream = F.Stream;
+  Stream.init(F.StreamFile);
+  F.SizeInBits = F.Buffer->getBufferSize() * 8;
 
   // Sniff for the signature.
   if (Stream.Read(8) != 'C' ||
@@ -1671,22 +2114,22 @@ PCHReader::PCHReadResult PCHReader::ReadPCH(const std::string &FileName) {
     unsigned Code = Stream.ReadCode();
 
     if (Code != llvm::bitc::ENTER_SUBBLOCK) {
-      Error("invalid record at top-level of PCH file");
+      Error("invalid record at top-level of AST file");
       return Failure;
     }
 
     unsigned BlockID = Stream.ReadSubBlockID();
 
-    // We only know the PCH subblock ID.
+    // We only know the AST subblock ID.
     switch (BlockID) {
     case llvm::bitc::BLOCKINFO_BLOCK_ID:
       if (Stream.ReadBlockInfoBlock()) {
-        Error("malformed BlockInfoBlock in PCH file");
+        Error("malformed BlockInfoBlock in AST file");
         return Failure;
       }
       break;
-    case pch::PCH_BLOCK_ID:
-      switch (ReadPCHBlock()) {
+    case AST_BLOCK_ID:
+      switch (ReadASTBlock(F)) {
       case Success:
         break;
 
@@ -1695,87 +2138,46 @@ PCHReader::PCHReadResult PCHReader::ReadPCH(const std::string &FileName) {
 
       case IgnorePCH:
         // FIXME: We could consider reading through to the end of this
-        // PCH block, skipping subblocks, to see if there are other
-        // PCH blocks elsewhere.
+        // AST block, skipping subblocks, to see if there are other
+        // AST blocks elsewhere.
 
         // Clear out any preallocated source location entries, so that
         // the source manager does not try to resolve them later.
         SourceMgr.ClearPreallocatedSLocEntries();
 
         // Remove the stat cache.
-        if (StatCache)
-          FileMgr.removeStatCache((PCHStatCache*)StatCache);
+        if (F.StatCache)
+          FileMgr.removeStatCache((ASTStatCache*)F.StatCache);
 
         return IgnorePCH;
       }
       break;
     default:
       if (Stream.SkipBlock()) {
-        Error("malformed block record in PCH file");
+        Error("malformed block record in AST file");
         return Failure;
       }
       break;
     }
   }
 
-  // Check the predefines buffer.
-  if (CheckPredefinesBuffers())
-    return IgnorePCH;
-
-  if (PP) {
-    // Initialization of keywords and pragmas occurs before the
-    // PCH file is read, so there may be some identifiers that were
-    // loaded into the IdentifierTable before we intercepted the
-    // creation of identifiers. Iterate through the list of known
-    // identifiers and determine whether we have to establish
-    // preprocessor definitions or top-level identifier declaration
-    // chains for those identifiers.
-    //
-    // We copy the IdentifierInfo pointers to a small vector first,
-    // since de-serializing declarations or macro definitions can add
-    // new entries into the identifier table, invalidating the
-    // iterators.
-    llvm::SmallVector<IdentifierInfo *, 128> Identifiers;
-    for (IdentifierTable::iterator Id = PP->getIdentifierTable().begin(),
-                                IdEnd = PP->getIdentifierTable().end();
-         Id != IdEnd; ++Id)
-      Identifiers.push_back(Id->second);
-    PCHIdentifierLookupTable *IdTable
-      = (PCHIdentifierLookupTable *)IdentifierLookupTable;
-    for (unsigned I = 0, N = Identifiers.size(); I != N; ++I) {
-      IdentifierInfo *II = Identifiers[I];
-      // Look in the on-disk hash table for an entry for
-      PCHIdentifierLookupTrait Info(*this, II);
-      std::pair<const char*, unsigned> Key(II->getNameStart(), II->getLength());
-      PCHIdentifierLookupTable::iterator Pos = IdTable->find(Key, &Info);
-      if (Pos == IdTable->end())
-        continue;
-
-      // Dereferencing the iterator has the effect of populating the
-      // IdentifierInfo node with the various declarations it needs.
-      (void)*Pos;
-    }
-  }
-
-  if (Context)
-    InitializeContext(*Context);
-
   return Success;
 }
 
-void PCHReader::setPreprocessor(Preprocessor &pp) {
+void ASTReader::setPreprocessor(Preprocessor &pp) {
   PP = &pp;
-  
-  if (NumPreallocatedPreprocessingEntities) {
+
+  unsigned TotalNum = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I)
+    TotalNum += Chain[I]->NumPreallocatedPreprocessingEntities;
+  if (TotalNum) {
     if (!PP->getPreprocessingRecord())
       PP->createPreprocessingRecord();
-    PP->getPreprocessingRecord()->SetExternalSource(*this, 
-                                          NumPreallocatedPreprocessingEntities);
-    NumPreallocatedPreprocessingEntities = 0;
+    PP->getPreprocessingRecord()->SetExternalSource(*this, TotalNum);
   }
 }
 
-void PCHReader::InitializeContext(ASTContext &Ctx) {
+void ASTReader::InitializeContext(ASTContext &Ctx) {
   Context = &Ctx;
   assert(Context && "Passed null context!");
 
@@ -1789,22 +2191,22 @@ void PCHReader::InitializeContext(ASTContext &Ctx) {
 
   // Load the special types.
   Context->setBuiltinVaListType(
-    GetType(SpecialTypes[pch::SPECIAL_TYPE_BUILTIN_VA_LIST]));
-  if (unsigned Id = SpecialTypes[pch::SPECIAL_TYPE_OBJC_ID])
+    GetType(SpecialTypes[SPECIAL_TYPE_BUILTIN_VA_LIST]));
+  if (unsigned Id = SpecialTypes[SPECIAL_TYPE_OBJC_ID])
     Context->setObjCIdType(GetType(Id));
-  if (unsigned Sel = SpecialTypes[pch::SPECIAL_TYPE_OBJC_SELECTOR])
+  if (unsigned Sel = SpecialTypes[SPECIAL_TYPE_OBJC_SELECTOR])
     Context->setObjCSelType(GetType(Sel));
-  if (unsigned Proto = SpecialTypes[pch::SPECIAL_TYPE_OBJC_PROTOCOL])
+  if (unsigned Proto = SpecialTypes[SPECIAL_TYPE_OBJC_PROTOCOL])
     Context->setObjCProtoType(GetType(Proto));
-  if (unsigned Class = SpecialTypes[pch::SPECIAL_TYPE_OBJC_CLASS])
+  if (unsigned Class = SpecialTypes[SPECIAL_TYPE_OBJC_CLASS])
     Context->setObjCClassType(GetType(Class));
 
-  if (unsigned String = SpecialTypes[pch::SPECIAL_TYPE_CF_CONSTANT_STRING])
+  if (unsigned String = SpecialTypes[SPECIAL_TYPE_CF_CONSTANT_STRING])
     Context->setCFConstantStringType(GetType(String));
   if (unsigned FastEnum
-        = SpecialTypes[pch::SPECIAL_TYPE_OBJC_FAST_ENUMERATION_STATE])
+        = SpecialTypes[SPECIAL_TYPE_OBJC_FAST_ENUMERATION_STATE])
     Context->setObjCFastEnumerationStateType(GetType(FastEnum));
-  if (unsigned File = SpecialTypes[pch::SPECIAL_TYPE_FILE]) {
+  if (unsigned File = SpecialTypes[SPECIAL_TYPE_FILE]) {
     QualType FileType = GetType(File);
     if (FileType.isNull()) {
       Error("FILE type is NULL");
@@ -1815,13 +2217,13 @@ void PCHReader::InitializeContext(ASTContext &Ctx) {
     else {
       const TagType *Tag = FileType->getAs<TagType>();
       if (!Tag) {
-        Error("Invalid FILE type in PCH file");
+        Error("Invalid FILE type in AST file");
         return;
       }
       Context->setFILEDecl(Tag->getDecl());
     }
   }
-  if (unsigned Jmp_buf = SpecialTypes[pch::SPECIAL_TYPE_jmp_buf]) {
+  if (unsigned Jmp_buf = SpecialTypes[SPECIAL_TYPE_jmp_buf]) {
     QualType Jmp_bufType = GetType(Jmp_buf);
     if (Jmp_bufType.isNull()) {
       Error("jmp_bug type is NULL");
@@ -1832,13 +2234,13 @@ void PCHReader::InitializeContext(ASTContext &Ctx) {
     else {
       const TagType *Tag = Jmp_bufType->getAs<TagType>();
       if (!Tag) {
-        Error("Invalid jmp_bug type in PCH file");
+        Error("Invalid jmp_buf type in AST file");
         return;
       }
       Context->setjmp_bufDecl(Tag->getDecl());
     }
   }
-  if (unsigned Sigjmp_buf = SpecialTypes[pch::SPECIAL_TYPE_sigjmp_buf]) {
+  if (unsigned Sigjmp_buf = SpecialTypes[SPECIAL_TYPE_sigjmp_buf]) {
     QualType Sigjmp_bufType = GetType(Sigjmp_buf);
     if (Sigjmp_bufType.isNull()) {
       Error("sigjmp_buf type is NULL");
@@ -1848,40 +2250,40 @@ void PCHReader::InitializeContext(ASTContext &Ctx) {
       Context->setsigjmp_bufDecl(Typedef->getDecl());
     else {
       const TagType *Tag = Sigjmp_bufType->getAs<TagType>();
-      assert(Tag && "Invalid sigjmp_buf type in PCH file");
+      assert(Tag && "Invalid sigjmp_buf type in AST file");
       Context->setsigjmp_bufDecl(Tag->getDecl());
     }
   }
   if (unsigned ObjCIdRedef
-        = SpecialTypes[pch::SPECIAL_TYPE_OBJC_ID_REDEFINITION])
+        = SpecialTypes[SPECIAL_TYPE_OBJC_ID_REDEFINITION])
     Context->ObjCIdRedefinitionType = GetType(ObjCIdRedef);
   if (unsigned ObjCClassRedef
-      = SpecialTypes[pch::SPECIAL_TYPE_OBJC_CLASS_REDEFINITION])
+      = SpecialTypes[SPECIAL_TYPE_OBJC_CLASS_REDEFINITION])
     Context->ObjCClassRedefinitionType = GetType(ObjCClassRedef);
-  if (unsigned String = SpecialTypes[pch::SPECIAL_TYPE_BLOCK_DESCRIPTOR])
+  if (unsigned String = SpecialTypes[SPECIAL_TYPE_BLOCK_DESCRIPTOR])
     Context->setBlockDescriptorType(GetType(String));
   if (unsigned String
-      = SpecialTypes[pch::SPECIAL_TYPE_BLOCK_EXTENDED_DESCRIPTOR])
+      = SpecialTypes[SPECIAL_TYPE_BLOCK_EXTENDED_DESCRIPTOR])
     Context->setBlockDescriptorExtendedType(GetType(String));
   if (unsigned ObjCSelRedef
-      = SpecialTypes[pch::SPECIAL_TYPE_OBJC_SEL_REDEFINITION])
+      = SpecialTypes[SPECIAL_TYPE_OBJC_SEL_REDEFINITION])
     Context->ObjCSelRedefinitionType = GetType(ObjCSelRedef);
-  if (unsigned String = SpecialTypes[pch::SPECIAL_TYPE_NS_CONSTANT_STRING])
+  if (unsigned String = SpecialTypes[SPECIAL_TYPE_NS_CONSTANT_STRING])
     Context->setNSConstantStringType(GetType(String));
 
-  if (SpecialTypes[pch::SPECIAL_TYPE_INT128_INSTALLED])
+  if (SpecialTypes[SPECIAL_TYPE_INT128_INSTALLED])
     Context->setInt128Installed();
 }
 
 /// \brief Retrieve the name of the original source file name
-/// directly from the PCH file, without actually loading the PCH
+/// directly from the AST file, without actually loading the AST
 /// file.
-std::string PCHReader::getOriginalSourceFile(const std::string &PCHFileName,
+std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName,
                                              Diagnostic &Diags) {
-  // Open the PCH file.
+  // Open the AST file.
   std::string ErrStr;
   llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
-  Buffer.reset(llvm::MemoryBuffer::getFile(PCHFileName.c_str(), &ErrStr));
+  Buffer.reset(llvm::MemoryBuffer::getFile(ASTFileName.c_str(), &ErrStr));
   if (!Buffer) {
     Diags.Report(diag::err_fe_unable_to_read_pch_file) << ErrStr;
     return std::string();
@@ -1899,7 +2301,7 @@ std::string PCHReader::getOriginalSourceFile(const std::string &PCHFileName,
       Stream.Read(8) != 'P' ||
       Stream.Read(8) != 'C' ||
       Stream.Read(8) != 'H') {
-    Diags.Report(diag::err_fe_not_a_pch_file) << PCHFileName;
+    Diags.Report(diag::err_fe_not_a_pch_file) << ASTFileName;
     return std::string();
   }
 
@@ -1910,18 +2312,18 @@ std::string PCHReader::getOriginalSourceFile(const std::string &PCHFileName,
     if (Code == llvm::bitc::ENTER_SUBBLOCK) {
       unsigned BlockID = Stream.ReadSubBlockID();
 
-      // We only know the PCH subblock ID.
+      // We only know the AST subblock ID.
       switch (BlockID) {
-      case pch::PCH_BLOCK_ID:
-        if (Stream.EnterSubBlock(pch::PCH_BLOCK_ID)) {
-          Diags.Report(diag::err_fe_pch_malformed_block) << PCHFileName;
+      case AST_BLOCK_ID:
+        if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
+          Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
           return std::string();
         }
         break;
 
       default:
         if (Stream.SkipBlock()) {
-          Diags.Report(diag::err_fe_pch_malformed_block) << PCHFileName;
+          Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
           return std::string();
         }
         break;
@@ -1931,7 +2333,7 @@ std::string PCHReader::getOriginalSourceFile(const std::string &PCHFileName,
 
     if (Code == llvm::bitc::END_BLOCK) {
       if (Stream.ReadBlockEnd()) {
-        Diags.Report(diag::err_fe_pch_error_at_end_block) << PCHFileName;
+        Diags.Report(diag::err_fe_pch_error_at_end_block) << ASTFileName;
         return std::string();
       }
       continue;
@@ -1946,7 +2348,7 @@ std::string PCHReader::getOriginalSourceFile(const std::string &PCHFileName,
     const char *BlobStart = 0;
     unsigned BlobLen = 0;
     if (Stream.ReadRecord(Code, Record, &BlobStart, &BlobLen)
-          == pch::ORIGINAL_FILE_NAME)
+          == ORIGINAL_FILE_NAME)
       return std::string(BlobStart, BlobLen);
   }
 
@@ -1956,18 +2358,11 @@ std::string PCHReader::getOriginalSourceFile(const std::string &PCHFileName,
 /// \brief Parse the record that corresponds to a LangOptions data
 /// structure.
 ///
-/// This routine compares the language options used to generate the
-/// PCH file against the language options set for the current
-/// compilation. For each option, we classify differences between the
-/// two compiler states as either "benign" or "important". Benign
-/// differences don't matter, and we accept them without complaint
-/// (and without modifying the language options). Differences between
-/// the states for important options cause the PCH file to be
-/// unusable, so we emit a warning and return true to indicate that
-/// there was an error.
+/// This routine parses the language options from the AST file and then gives
+/// them to the AST listener if one is set.
 ///
-/// \returns true if the PCH file is unacceptable, false otherwise.
-bool PCHReader::ParseLanguageOptions(
+/// \returns true if the listener deems the file unacceptable, false otherwise.
+bool ASTReader::ParseLanguageOptions(
                              const llvm::SmallVectorImpl<uint64_t> &Record) {
   if (Listener) {
     LangOptions LangOpts;
@@ -2038,30 +2433,47 @@ bool PCHReader::ParseLanguageOptions(
   return false;
 }
 
-void PCHReader::ReadPreprocessedEntities() {
+void ASTReader::ReadPreprocessedEntities() {
   ReadDefinedMacros();
 }
 
-/// \brief Read and return the type at the given offset.
+/// \brief Get the correct cursor and offset for loading a type.
+ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
+  PerFileData *F = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    F = Chain[N - I - 1];
+    if (Index < F->LocalNumTypes)
+      break;
+    Index -= F->LocalNumTypes;
+  }
+  assert(F && F->LocalNumTypes > Index && "Broken chain");
+  return RecordLocation(&F->DeclsCursor, F->TypeOffsets[Index]);
+}
+
+/// \brief Read and return the type with the given index..
 ///
-/// This routine actually reads the record corresponding to the type
-/// at the given offset in the bitstream. It is a helper routine for
-/// GetType, which deals with reading type IDs.
-QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
+/// The index is the type ID, shifted and minus the number of predefs. This
+/// routine actually reads the record corresponding to the type at the given
+/// location. It is a helper routine for GetType, which deals with reading type
+/// IDs.
+QualType ASTReader::ReadTypeRecord(unsigned Index) {
+  RecordLocation Loc = TypeCursorForIndex(Index);
+  llvm::BitstreamCursor &DeclsCursor = *Loc.first;
+
   // Keep track of where we are in the stream, then jump back there
   // after reading this type.
   SavedStreamPosition SavedPosition(DeclsCursor);
 
   ReadingKindTracker ReadingKind(Read_Type, *this);
-  
+
   // Note that we are loading a type record.
-  LoadingTypeOrDecl Loading(*this);
+  Deserializing AType(this);
 
-  DeclsCursor.JumpToBit(Offset);
+  DeclsCursor.JumpToBit(Loc.second);
   RecordData Record;
   unsigned Code = DeclsCursor.ReadCode();
-  switch ((pch::TypeCode)DeclsCursor.ReadRecord(Code, Record)) {
-  case pch::TYPE_EXT_QUAL: {
+  switch ((TypeCode)DeclsCursor.ReadRecord(Code, Record)) {
+  case TYPE_EXT_QUAL: {
     if (Record.size() != 2) {
       Error("Incorrect encoding of extended qualifier type");
       return QualType();
@@ -2071,7 +2483,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getQualifiedType(Base, Quals);
   }
 
-  case pch::TYPE_COMPLEX: {
+  case TYPE_COMPLEX: {
     if (Record.size() != 1) {
       Error("Incorrect encoding of complex type");
       return QualType();
@@ -2080,7 +2492,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getComplexType(ElemType);
   }
 
-  case pch::TYPE_POINTER: {
+  case TYPE_POINTER: {
     if (Record.size() != 1) {
       Error("Incorrect encoding of pointer type");
       return QualType();
@@ -2089,7 +2501,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getPointerType(PointeeType);
   }
 
-  case pch::TYPE_BLOCK_POINTER: {
+  case TYPE_BLOCK_POINTER: {
     if (Record.size() != 1) {
       Error("Incorrect encoding of block pointer type");
       return QualType();
@@ -2098,7 +2510,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getBlockPointerType(PointeeType);
   }
 
-  case pch::TYPE_LVALUE_REFERENCE: {
+  case TYPE_LVALUE_REFERENCE: {
     if (Record.size() != 1) {
       Error("Incorrect encoding of lvalue reference type");
       return QualType();
@@ -2107,7 +2519,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getLValueReferenceType(PointeeType);
   }
 
-  case pch::TYPE_RVALUE_REFERENCE: {
+  case TYPE_RVALUE_REFERENCE: {
     if (Record.size() != 1) {
       Error("Incorrect encoding of rvalue reference type");
       return QualType();
@@ -2116,7 +2528,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getRValueReferenceType(PointeeType);
   }
 
-  case pch::TYPE_MEMBER_POINTER: {
+  case TYPE_MEMBER_POINTER: {
     if (Record.size() != 2) {
       Error("Incorrect encoding of member pointer type");
       return QualType();
@@ -2126,7 +2538,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getMemberPointerType(PointeeType, ClassType.getTypePtr());
   }
 
-  case pch::TYPE_CONSTANT_ARRAY: {
+  case TYPE_CONSTANT_ARRAY: {
     QualType ElementType = GetType(Record[0]);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
@@ -2136,27 +2548,27 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
                                          ASM, IndexTypeQuals);
   }
 
-  case pch::TYPE_INCOMPLETE_ARRAY: {
+  case TYPE_INCOMPLETE_ARRAY: {
     QualType ElementType = GetType(Record[0]);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
     return Context->getIncompleteArrayType(ElementType, ASM, IndexTypeQuals);
   }
 
-  case pch::TYPE_VARIABLE_ARRAY: {
+  case TYPE_VARIABLE_ARRAY: {
     QualType ElementType = GetType(Record[0]);
     ArrayType::ArraySizeModifier ASM = (ArrayType::ArraySizeModifier)Record[1];
     unsigned IndexTypeQuals = Record[2];
     SourceLocation LBLoc = SourceLocation::getFromRawEncoding(Record[3]);
     SourceLocation RBLoc = SourceLocation::getFromRawEncoding(Record[4]);
-    return Context->getVariableArrayType(ElementType, ReadExpr(),
+    return Context->getVariableArrayType(ElementType, ReadExpr(DeclsCursor),
                                          ASM, IndexTypeQuals,
                                          SourceRange(LBLoc, RBLoc));
   }
 
-  case pch::TYPE_VECTOR: {
+  case TYPE_VECTOR: {
     if (Record.size() != 3) {
-      Error("incorrect encoding of vector type in PCH file");
+      Error("incorrect encoding of vector type in AST file");
       return QualType();
     }
 
@@ -2167,9 +2579,9 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
                                   (VectorType::AltiVecSpecific)AltiVecSpec);
   }
 
-  case pch::TYPE_EXT_VECTOR: {
+  case TYPE_EXT_VECTOR: {
     if (Record.size() != 3) {
-      Error("incorrect encoding of extended vector type in PCH file");
+      Error("incorrect encoding of extended vector type in AST file");
       return QualType();
     }
 
@@ -2178,7 +2590,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getExtVectorType(ElementType, NumElements);
   }
 
-  case pch::TYPE_FUNCTION_NO_PROTO: {
+  case TYPE_FUNCTION_NO_PROTO: {
     if (Record.size() != 4) {
       Error("incorrect encoding of no-proto function type");
       return QualType();
@@ -2188,7 +2600,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getFunctionNoProtoType(ResultType, Info);
   }
 
-  case pch::TYPE_FUNCTION_PROTO: {
+  case TYPE_FUNCTION_PROTO: {
     QualType ResultType = GetType(Record[0]);
     bool NoReturn = Record[1];
     unsigned RegParm = Record[2];
@@ -2214,11 +2626,11 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
                                                           CallConv));
   }
 
-  case pch::TYPE_UNRESOLVED_USING:
+  case TYPE_UNRESOLVED_USING:
     return Context->getTypeDeclType(
              cast<UnresolvedUsingTypenameDecl>(GetDecl(Record[0])));
 
-  case pch::TYPE_TYPEDEF: {
+  case TYPE_TYPEDEF: {
     if (Record.size() != 2) {
       Error("incorrect encoding of typedef type");
       return QualType();
@@ -2228,22 +2640,22 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getTypedefType(Decl, Canonical);
   }
 
-  case pch::TYPE_TYPEOF_EXPR:
-    return Context->getTypeOfExprType(ReadExpr());
+  case TYPE_TYPEOF_EXPR:
+    return Context->getTypeOfExprType(ReadExpr(DeclsCursor));
 
-  case pch::TYPE_TYPEOF: {
+  case TYPE_TYPEOF: {
     if (Record.size() != 1) {
-      Error("incorrect encoding of typeof(type) in PCH file");
+      Error("incorrect encoding of typeof(type) in AST file");
       return QualType();
     }
     QualType UnderlyingType = GetType(Record[0]);
     return Context->getTypeOfType(UnderlyingType);
   }
 
-  case pch::TYPE_DECLTYPE:
-    return Context->getDecltypeType(ReadExpr());
+  case TYPE_DECLTYPE:
+    return Context->getDecltypeType(ReadExpr(DeclsCursor));
 
-  case pch::TYPE_RECORD: {
+  case TYPE_RECORD: {
     if (Record.size() != 2) {
       Error("incorrect encoding of record type");
       return QualType();
@@ -2254,7 +2666,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return T;
   }
 
-  case pch::TYPE_ENUM: {
+  case TYPE_ENUM: {
     if (Record.size() != 2) {
       Error("incorrect encoding of enum type");
       return QualType();
@@ -2265,7 +2677,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return T;
   }
 
-  case pch::TYPE_ELABORATED: {
+  case TYPE_ELABORATED: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
     NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
@@ -2273,13 +2685,13 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getElaboratedType(Keyword, NNS, NamedType);
   }
 
-  case pch::TYPE_OBJC_INTERFACE: {
+  case TYPE_OBJC_INTERFACE: {
     unsigned Idx = 0;
     ObjCInterfaceDecl *ItfD = cast<ObjCInterfaceDecl>(GetDecl(Record[Idx++]));
     return Context->getObjCInterfaceType(ItfD);
   }
 
-  case pch::TYPE_OBJC_OBJECT: {
+  case TYPE_OBJC_OBJECT: {
     unsigned Idx = 0;
     QualType Base = GetType(Record[Idx++]);
     unsigned NumProtos = Record[Idx++];
@@ -2289,13 +2701,13 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getObjCObjectType(Base, Protos.data(), NumProtos);    
   }
 
-  case pch::TYPE_OBJC_OBJECT_POINTER: {
+  case TYPE_OBJC_OBJECT_POINTER: {
     unsigned Idx = 0;
     QualType Pointee = GetType(Record[Idx++]);
     return Context->getObjCObjectPointerType(Pointee);
   }
 
-  case pch::TYPE_SUBST_TEMPLATE_TYPE_PARM: {
+  case TYPE_SUBST_TEMPLATE_TYPE_PARM: {
     unsigned Idx = 0;
     QualType Parm = GetType(Record[Idx++]);
     QualType Replacement = GetType(Record[Idx++]);
@@ -2304,16 +2716,16 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
                                             Replacement);
   }
 
-  case pch::TYPE_INJECTED_CLASS_NAME: {
+  case TYPE_INJECTED_CLASS_NAME: {
     CXXRecordDecl *D = cast<CXXRecordDecl>(GetDecl(Record[0]));
     QualType TST = GetType(Record[1]); // probably derivable
     // FIXME: ASTContext::getInjectedClassNameType is not currently suitable
-    // for PCH reading, too much interdependencies.
+    // for AST reading, too much interdependencies.
     return
       QualType(new (*Context, TypeAlignment) InjectedClassNameType(D, TST), 0);
   }
   
-  case pch::TYPE_TEMPLATE_TYPE_PARM: {
+  case TYPE_TEMPLATE_TYPE_PARM: {
     unsigned Idx = 0;
     unsigned Depth = Record[Idx++];
     unsigned Index = Record[Idx++];
@@ -2322,7 +2734,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getTemplateTypeParmType(Depth, Index, Pack, Name);
   }
   
-  case pch::TYPE_DEPENDENT_NAME: {
+  case TYPE_DEPENDENT_NAME: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
     NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
@@ -2331,7 +2743,7 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     return Context->getDependentNameType(Keyword, NNS, Name, Canon);
   }
   
-  case pch::TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION: {
+  case TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION: {
     unsigned Idx = 0;
     ElaboratedTypeKeyword Keyword = (ElaboratedTypeKeyword)Record[Idx++];
     NestedNameSpecifier *NNS = ReadNestedNameSpecifier(Record, Idx);
@@ -2340,12 +2752,12 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     llvm::SmallVector<TemplateArgument, 8> Args;
     Args.reserve(NumArgs);
     while (NumArgs--)
-      Args.push_back(ReadTemplateArgument(Record, Idx));
+      Args.push_back(ReadTemplateArgument(DeclsCursor, Record, Idx));
     return Context->getDependentTemplateSpecializationType(Keyword, NNS, Name,
                                                       Args.size(), Args.data());
   }
   
-  case pch::TYPE_DEPENDENT_SIZED_ARRAY: {
+  case TYPE_DEPENDENT_SIZED_ARRAY: {
     unsigned Idx = 0;
 
     // ArrayType
@@ -2355,19 +2767,19 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
     unsigned IndexTypeQuals = Record[Idx++];
 
     // DependentSizedArrayType
-    Expr *NumElts = ReadExpr();
+    Expr *NumElts = ReadExpr(DeclsCursor);
     SourceRange Brackets = ReadSourceRange(Record, Idx);
 
     return Context->getDependentSizedArrayType(ElementType, NumElts, ASM,
                                                IndexTypeQuals, Brackets);
   }
 
-  case pch::TYPE_TEMPLATE_SPECIALIZATION: {
+  case TYPE_TEMPLATE_SPECIALIZATION: {
     unsigned Idx = 0;
     bool IsDependent = Record[Idx++];
     TemplateName Name = ReadTemplateName(Record, Idx);
     llvm::SmallVector<TemplateArgument, 8> Args;
-    ReadTemplateArgumentList(Args, Record, Idx);
+    ReadTemplateArgumentList(Args, DeclsCursor, Record, Idx);
     QualType Canon = GetType(Record[Idx++]);
     QualType T;
     if (Canon.isNull())
@@ -2387,14 +2799,15 @@ QualType PCHReader::ReadTypeRecord(uint64_t Offset) {
 namespace {
 
 class TypeLocReader : public TypeLocVisitor<TypeLocReader> {
-  PCHReader &Reader;
-  const PCHReader::RecordData &Record;
+  ASTReader &Reader;
+  llvm::BitstreamCursor &DeclsCursor;
+  const ASTReader::RecordData &Record;
   unsigned &Idx;
 
 public:
-  TypeLocReader(PCHReader &Reader, const PCHReader::RecordData &Record,
-                unsigned &Idx)
-    : Reader(Reader), Record(Record), Idx(Idx) { }
+  TypeLocReader(ASTReader &Reader, llvm::BitstreamCursor &Cursor,
+                const ASTReader::RecordData &Record, unsigned &Idx)
+    : Reader(Reader), DeclsCursor(Cursor), Record(Record), Idx(Idx) { }
 
   // We want compile-time assurance that we've enumerated all of
   // these, so unfortunately we have to declare them first, then
@@ -2444,7 +2857,7 @@ void TypeLocReader::VisitArrayTypeLoc(ArrayTypeLoc TL) {
   TL.setLBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   TL.setRBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   if (Record[Idx++])
-    TL.setSizeExpr(Reader.ReadExpr());
+    TL.setSizeExpr(Reader.ReadExpr(DeclsCursor));
   else
     TL.setSizeExpr(0);
 }
@@ -2499,7 +2912,7 @@ void TypeLocReader::VisitTypeOfTypeLoc(TypeOfTypeLoc TL) {
   TL.setTypeofLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   TL.setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   TL.setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  TL.setUnderlyingTInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  TL.setUnderlyingTInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
 }
 void TypeLocReader::VisitDecltypeTypeLoc(DecltypeTypeLoc TL) {
   TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -2525,7 +2938,7 @@ void TypeLocReader::VisitTemplateSpecializationTypeLoc(
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     TL.setArgLocInfo(i,
         Reader.GetTemplateArgumentLocInfo(TL.getTypePtr()->getArg(i).getKind(),
-                                          Record, Idx));
+                                          DeclsCursor, Record, Idx));
 }
 void TypeLocReader::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
   TL.setKeywordLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -2549,7 +2962,7 @@ void TypeLocReader::VisitDependentTemplateSpecializationTypeLoc(
   for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I)
     TL.setArgLocInfo(I,
         Reader.GetTemplateArgumentLocInfo(TL.getTypePtr()->getArg(I).getKind(),
-                                          Record, Idx));
+                                          DeclsCursor, Record, Idx));
 }
 void TypeLocReader::VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc TL) {
   TL.setNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -2565,87 +2978,112 @@ void TypeLocReader::VisitObjCObjectPointerTypeLoc(ObjCObjectPointerTypeLoc TL) {
   TL.setStarLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-TypeSourceInfo *PCHReader::GetTypeSourceInfo(const RecordData &Record,
+TypeSourceInfo *ASTReader::GetTypeSourceInfo(llvm::BitstreamCursor &DeclsCursor,
+                                             const RecordData &Record,
                                              unsigned &Idx) {
   QualType InfoTy = GetType(Record[Idx++]);
   if (InfoTy.isNull())
     return 0;
 
   TypeSourceInfo *TInfo = getContext()->CreateTypeSourceInfo(InfoTy);
-  TypeLocReader TLR(*this, Record, Idx);
+  TypeLocReader TLR(*this, DeclsCursor, Record, Idx);
   for (TypeLoc TL = TInfo->getTypeLoc(); !TL.isNull(); TL = TL.getNextTypeLoc())
     TLR.Visit(TL);
   return TInfo;
 }
 
-QualType PCHReader::GetType(pch::TypeID ID) {
+QualType ASTReader::GetType(TypeID ID) {
   unsigned FastQuals = ID & Qualifiers::FastMask;
   unsigned Index = ID >> Qualifiers::FastWidth;
 
-  if (Index < pch::NUM_PREDEF_TYPE_IDS) {
+  if (Index < NUM_PREDEF_TYPE_IDS) {
     QualType T;
-    switch ((pch::PredefinedTypeIDs)Index) {
-    case pch::PREDEF_TYPE_NULL_ID: return QualType();
-    case pch::PREDEF_TYPE_VOID_ID: T = Context->VoidTy; break;
-    case pch::PREDEF_TYPE_BOOL_ID: T = Context->BoolTy; break;
+    switch ((PredefinedTypeIDs)Index) {
+    case PREDEF_TYPE_NULL_ID: return QualType();
+    case PREDEF_TYPE_VOID_ID: T = Context->VoidTy; break;
+    case PREDEF_TYPE_BOOL_ID: T = Context->BoolTy; break;
 
-    case pch::PREDEF_TYPE_CHAR_U_ID:
-    case pch::PREDEF_TYPE_CHAR_S_ID:
+    case PREDEF_TYPE_CHAR_U_ID:
+    case PREDEF_TYPE_CHAR_S_ID:
       // FIXME: Check that the signedness of CharTy is correct!
       T = Context->CharTy;
       break;
 
-    case pch::PREDEF_TYPE_UCHAR_ID:      T = Context->UnsignedCharTy;     break;
-    case pch::PREDEF_TYPE_USHORT_ID:     T = Context->UnsignedShortTy;    break;
-    case pch::PREDEF_TYPE_UINT_ID:       T = Context->UnsignedIntTy;      break;
-    case pch::PREDEF_TYPE_ULONG_ID:      T = Context->UnsignedLongTy;     break;
-    case pch::PREDEF_TYPE_ULONGLONG_ID:  T = Context->UnsignedLongLongTy; break;
-    case pch::PREDEF_TYPE_UINT128_ID:    T = Context->UnsignedInt128Ty;   break;
-    case pch::PREDEF_TYPE_SCHAR_ID:      T = Context->SignedCharTy;       break;
-    case pch::PREDEF_TYPE_WCHAR_ID:      T = Context->WCharTy;            break;
-    case pch::PREDEF_TYPE_SHORT_ID:      T = Context->ShortTy;            break;
-    case pch::PREDEF_TYPE_INT_ID:        T = Context->IntTy;              break;
-    case pch::PREDEF_TYPE_LONG_ID:       T = Context->LongTy;             break;
-    case pch::PREDEF_TYPE_LONGLONG_ID:   T = Context->LongLongTy;         break;
-    case pch::PREDEF_TYPE_INT128_ID:     T = Context->Int128Ty;           break;
-    case pch::PREDEF_TYPE_FLOAT_ID:      T = Context->FloatTy;            break;
-    case pch::PREDEF_TYPE_DOUBLE_ID:     T = Context->DoubleTy;           break;
-    case pch::PREDEF_TYPE_LONGDOUBLE_ID: T = Context->LongDoubleTy;       break;
-    case pch::PREDEF_TYPE_OVERLOAD_ID:   T = Context->OverloadTy;         break;
-    case pch::PREDEF_TYPE_DEPENDENT_ID:  T = Context->DependentTy;        break;
-    case pch::PREDEF_TYPE_NULLPTR_ID:    T = Context->NullPtrTy;          break;
-    case pch::PREDEF_TYPE_CHAR16_ID:     T = Context->Char16Ty;           break;
-    case pch::PREDEF_TYPE_CHAR32_ID:     T = Context->Char32Ty;           break;
-    case pch::PREDEF_TYPE_OBJC_ID:       T = Context->ObjCBuiltinIdTy;    break;
-    case pch::PREDEF_TYPE_OBJC_CLASS:    T = Context->ObjCBuiltinClassTy; break;
-    case pch::PREDEF_TYPE_OBJC_SEL:      T = Context->ObjCBuiltinSelTy;   break;
+    case PREDEF_TYPE_UCHAR_ID:      T = Context->UnsignedCharTy;     break;
+    case PREDEF_TYPE_USHORT_ID:     T = Context->UnsignedShortTy;    break;
+    case PREDEF_TYPE_UINT_ID:       T = Context->UnsignedIntTy;      break;
+    case PREDEF_TYPE_ULONG_ID:      T = Context->UnsignedLongTy;     break;
+    case PREDEF_TYPE_ULONGLONG_ID:  T = Context->UnsignedLongLongTy; break;
+    case PREDEF_TYPE_UINT128_ID:    T = Context->UnsignedInt128Ty;   break;
+    case PREDEF_TYPE_SCHAR_ID:      T = Context->SignedCharTy;       break;
+    case PREDEF_TYPE_WCHAR_ID:      T = Context->WCharTy;            break;
+    case PREDEF_TYPE_SHORT_ID:      T = Context->ShortTy;            break;
+    case PREDEF_TYPE_INT_ID:        T = Context->IntTy;              break;
+    case PREDEF_TYPE_LONG_ID:       T = Context->LongTy;             break;
+    case PREDEF_TYPE_LONGLONG_ID:   T = Context->LongLongTy;         break;
+    case PREDEF_TYPE_INT128_ID:     T = Context->Int128Ty;           break;
+    case PREDEF_TYPE_FLOAT_ID:      T = Context->FloatTy;            break;
+    case PREDEF_TYPE_DOUBLE_ID:     T = Context->DoubleTy;           break;
+    case PREDEF_TYPE_LONGDOUBLE_ID: T = Context->LongDoubleTy;       break;
+    case PREDEF_TYPE_OVERLOAD_ID:   T = Context->OverloadTy;         break;
+    case PREDEF_TYPE_DEPENDENT_ID:  T = Context->DependentTy;        break;
+    case PREDEF_TYPE_NULLPTR_ID:    T = Context->NullPtrTy;          break;
+    case PREDEF_TYPE_CHAR16_ID:     T = Context->Char16Ty;           break;
+    case PREDEF_TYPE_CHAR32_ID:     T = Context->Char32Ty;           break;
+    case PREDEF_TYPE_OBJC_ID:       T = Context->ObjCBuiltinIdTy;    break;
+    case PREDEF_TYPE_OBJC_CLASS:    T = Context->ObjCBuiltinClassTy; break;
+    case PREDEF_TYPE_OBJC_SEL:      T = Context->ObjCBuiltinSelTy;   break;
     }
 
     assert(!T.isNull() && "Unknown predefined type");
     return T.withFastQualifiers(FastQuals);
   }
 
-  Index -= pch::NUM_PREDEF_TYPE_IDS;
-  //assert(Index < TypesLoaded.size() && "Type index out-of-range");
+  Index -= NUM_PREDEF_TYPE_IDS;
+  assert(Index < TypesLoaded.size() && "Type index out-of-range");
   if (TypesLoaded[Index].isNull()) {
-    TypesLoaded[Index] = ReadTypeRecord(TypeOffsets[Index]);
-    TypesLoaded[Index]->setFromPCH();
+    TypesLoaded[Index] = ReadTypeRecord(Index);
+    TypesLoaded[Index]->setFromAST();
+    TypeIdxs[TypesLoaded[Index]] = TypeIdx::fromTypeID(ID);
     if (DeserializationListener)
-      DeserializationListener->TypeRead(ID, TypesLoaded[Index]);
+      DeserializationListener->TypeRead(TypeIdx::fromTypeID(ID),
+                                        TypesLoaded[Index]);
   }
 
   return TypesLoaded[Index].withFastQualifiers(FastQuals);
 }
 
+TypeID ASTReader::GetTypeID(QualType T) const {
+  return MakeTypeID(T,
+              std::bind1st(std::mem_fun(&ASTReader::GetTypeIdx), this));
+}
+
+TypeIdx ASTReader::GetTypeIdx(QualType T) const {
+  if (T.isNull())
+    return TypeIdx();
+  assert(!T.getLocalFastQualifiers());
+
+  TypeIdxMap::const_iterator I = TypeIdxs.find(T);
+  // GetTypeIdx is mostly used for computing the hash of DeclarationNames and
+  // comparing keys of ASTDeclContextNameLookupTable.
+  // If the type didn't come from the AST file use a specially marked index
+  // so that any hash/key comparison fail since no such index is stored
+  // in a AST file.
+  if (I == TypeIdxs.end())
+    return TypeIdx(-1);
+  return I->second;
+}
+
 TemplateArgumentLocInfo
-PCHReader::GetTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
+ASTReader::GetTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
+                                      llvm::BitstreamCursor &DeclsCursor,
                                       const RecordData &Record,
                                       unsigned &Index) {
   switch (Kind) {
   case TemplateArgument::Expression:
-    return ReadExpr();
+    return ReadExpr(DeclsCursor);
   case TemplateArgument::Type:
-    return GetTypeSourceInfo(Record, Index);
+    return GetTypeSourceInfo(DeclsCursor, Record, Index);
   case TemplateArgument::Template: {
     SourceRange QualifierRange = ReadSourceRange(Record, Index);
     SourceLocation TemplateNameLoc = ReadSourceLocation(Record, Index);
@@ -2662,43 +3100,45 @@ PCHReader::GetTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
 }
 
 TemplateArgumentLoc
-PCHReader::ReadTemplateArgumentLoc(const RecordData &Record, unsigned &Index) {
-  TemplateArgument Arg = ReadTemplateArgument(Record, Index);
+ASTReader::ReadTemplateArgumentLoc(llvm::BitstreamCursor &DeclsCursor,
+                                   const RecordData &Record, unsigned &Index) {
+  TemplateArgument Arg = ReadTemplateArgument(DeclsCursor, Record, Index);
 
   if (Arg.getKind() == TemplateArgument::Expression) {
     if (Record[Index++]) // bool InfoHasSameExpr.
       return TemplateArgumentLoc(Arg, TemplateArgumentLocInfo(Arg.getAsExpr()));
   }
   return TemplateArgumentLoc(Arg, GetTemplateArgumentLocInfo(Arg.getKind(),
+                                                             DeclsCursor,
                                                              Record, Index));
 }
 
-Decl *PCHReader::GetExternalDecl(uint32_t ID) {
+Decl *ASTReader::GetExternalDecl(uint32_t ID) {
   return GetDecl(ID);
 }
 
-TranslationUnitDecl *PCHReader::GetTranslationUnitDecl() {
+TranslationUnitDecl *ASTReader::GetTranslationUnitDecl() {
   if (!DeclsLoaded[0]) {
-    ReadDeclRecord(DeclOffsets[0], 0);
+    ReadDeclRecord(0, 1);
     if (DeserializationListener)
-      DeserializationListener->DeclRead(0, DeclsLoaded[0]);
+      DeserializationListener->DeclRead(1, DeclsLoaded[0]);
   }
 
   return cast<TranslationUnitDecl>(DeclsLoaded[0]);
 }
 
-Decl *PCHReader::GetDecl(pch::DeclID ID) {
+Decl *ASTReader::GetDecl(DeclID ID) {
   if (ID == 0)
     return 0;
 
   if (ID > DeclsLoaded.size()) {
-    Error("declaration ID out-of-range for PCH file");
+    Error("declaration ID out-of-range for AST file");
     return 0;
   }
 
   unsigned Index = ID - 1;
   if (!DeclsLoaded[Index]) {
-    ReadDeclRecord(DeclOffsets[Index], Index);
+    ReadDeclRecord(Index, ID);
     if (DeserializationListener)
       DeserializationListener->DeclRead(ID, DeclsLoaded[Index]);
   }
@@ -2711,109 +3151,122 @@ Decl *PCHReader::GetDecl(pch::DeclID ID) {
 /// This operation will read a new statement from the external
 /// source each time it is called, and is meant to be used via a
 /// LazyOffsetPtr (which is used by Decls for the body of functions, etc).
-Stmt *PCHReader::GetExternalDeclStmt(uint64_t Offset) {
-  // Since we know tha this statement is part of a decl, make sure to use the
-  // decl cursor to read it.
-  DeclsCursor.JumpToBit(Offset);
-  return ReadStmtFromStream(DeclsCursor);
+Stmt *ASTReader::GetExternalDeclStmt(uint64_t Offset) {
+  // Offset here is a global offset across the entire chain.
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    PerFileData &F = *Chain[N - I - 1];
+    if (Offset < F.SizeInBits) {
+      // Since we know that this statement is part of a decl, make sure to use
+      // the decl cursor to read it.
+      F.DeclsCursor.JumpToBit(Offset);
+      return ReadStmtFromStream(F.DeclsCursor);
+    }
+    Offset -= F.SizeInBits;
+  }
+  llvm_unreachable("Broken chain");
 }
 
-bool PCHReader::FindExternalLexicalDecls(const DeclContext *DC,
+bool ASTReader::FindExternalLexicalDecls(const DeclContext *DC,
                                          llvm::SmallVectorImpl<Decl*> &Decls) {
   assert(DC->hasExternalLexicalStorage() &&
          "DeclContext has no lexical decls in storage");
 
-  uint64_t Offset = DeclContextOffsets[DC].first;
-  if (Offset == 0) {
-    Error("DeclContext has no lexical decls in storage");
-    return true;
-  }
-
-  // Keep track of where we are in the stream, then jump back there
-  // after reading this context.
-  SavedStreamPosition SavedPosition(DeclsCursor);
+  // There might be lexical decls in multiple parts of the chain, for the TU
+  // at least.
+  DeclContextInfos &Infos = DeclContextOffsets[DC];
+  for (DeclContextInfos::iterator I = Infos.begin(), E = Infos.end();
+       I != E; ++I) {
+    // IDs can be 0 if this context doesn't contain declarations.
+    if (!I->LexicalDecls)
+      continue;
 
-  // Load the record containing all of the declarations lexically in
-  // this context.
-  DeclsCursor.JumpToBit(Offset);
-  RecordData Record;
-  unsigned Code = DeclsCursor.ReadCode();
-  unsigned RecCode = DeclsCursor.ReadRecord(Code, Record);
-  if (RecCode != pch::DECL_CONTEXT_LEXICAL) {
-    Error("Expected lexical block");
-    return true;
+    // Load all of the declaration IDs
+    for (const DeclID *ID = I->LexicalDecls,
+                           *IDE = ID + I->NumLexicalDecls;
+         ID != IDE; ++ID)
+      Decls.push_back(GetDecl(*ID));
   }
 
-  // Load all of the declaration IDs
-  for (RecordData::iterator I = Record.begin(), E = Record.end(); I != E; ++I)
-    Decls.push_back(GetDecl(*I));
   ++NumLexicalDeclContextsRead;
   return false;
 }
 
 DeclContext::lookup_result
-PCHReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
+ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
                                           DeclarationName Name) {
   assert(DC->hasExternalVisibleStorage() &&
          "DeclContext has no visible decls in storage");
-  uint64_t Offset = DeclContextOffsets[DC].second;
-  if (Offset == 0) {
-    Error("DeclContext has no visible decls in storage");
-    return DeclContext::lookup_result(DeclContext::lookup_iterator(),
-                                      DeclContext::lookup_iterator());
-  }
+  if (!Name)
+    return DeclContext::lookup_result(DeclContext::lookup_iterator(0),
+                                      DeclContext::lookup_iterator(0));
+
+  llvm::SmallVector<NamedDecl *, 64> Decls;
+  // There might be visible decls in multiple parts of the chain, for the TU
+  // and namespaces. For any given name, the last available results replace
+  // all earlier ones. For this reason, we walk in reverse.
+  DeclContextInfos &Infos = DeclContextOffsets[DC];
+  for (DeclContextInfos::reverse_iterator I = Infos.rbegin(), E = Infos.rend();
+       I != E; ++I) {
+    if (!I->NameLookupTableData)
+      continue;
 
-  // Keep track of where we are in the stream, then jump back there
-  // after reading this context.
-  SavedStreamPosition SavedPosition(DeclsCursor);
+    ASTDeclContextNameLookupTable *LookupTable =
+        (ASTDeclContextNameLookupTable*)I->NameLookupTableData;
+    ASTDeclContextNameLookupTable::iterator Pos = LookupTable->find(Name);
+    if (Pos == LookupTable->end())
+      continue;
 
-  // Load the record containing all of the declarations visible in
-  // this context.
-  DeclsCursor.JumpToBit(Offset);
-  RecordData Record;
-  unsigned Code = DeclsCursor.ReadCode();
-  unsigned RecCode = DeclsCursor.ReadRecord(Code, Record);
-  if (RecCode != pch::DECL_CONTEXT_VISIBLE) {
-    Error("Expected visible block");
-    return DeclContext::lookup_result(DeclContext::lookup_iterator(),
-                                      DeclContext::lookup_iterator());
+    ASTDeclContextNameLookupTrait::data_type Data = *Pos;
+    for (; Data.first != Data.second; ++Data.first)
+      Decls.push_back(cast<NamedDecl>(GetDecl(*Data.first)));
+    break;
   }
 
-  llvm::SmallVector<VisibleDeclaration, 64> Decls;
-  if (Record.empty()) {
-    SetExternalVisibleDecls(DC, Decls);
-    return DeclContext::lookup_result(DeclContext::lookup_iterator(),
-                                      DeclContext::lookup_iterator());
-  }
+  ++NumVisibleDeclContextsRead;
 
-  unsigned Idx = 0;
-  while (Idx < Record.size()) {
-    Decls.push_back(VisibleDeclaration());
-    Decls.back().Name = ReadDeclarationName(Record, Idx);
+  SetExternalVisibleDeclsForName(DC, Name, Decls);
+  return const_cast<DeclContext*>(DC)->lookup(Name);
+}
 
-    unsigned Size = Record[Idx++];
-    llvm::SmallVector<unsigned, 4> &LoadedDecls = Decls.back().Declarations;
-    LoadedDecls.reserve(Size);
-    for (unsigned I = 0; I < Size; ++I)
-      LoadedDecls.push_back(Record[Idx++]);
-  }
+void ASTReader::MaterializeVisibleDecls(const DeclContext *DC) {
+  assert(DC->hasExternalVisibleStorage() &&
+         "DeclContext has no visible decls in storage");
 
-  ++NumVisibleDeclContextsRead;
+  llvm::SmallVector<NamedDecl *, 64> Decls;
+  // There might be visible decls in multiple parts of the chain, for the TU
+  // and namespaces.
+  DeclContextInfos &Infos = DeclContextOffsets[DC];
+  for (DeclContextInfos::iterator I = Infos.begin(), E = Infos.end();
+       I != E; ++I) {
+    if (!I->NameLookupTableData)
+      continue;
 
-  SetExternalVisibleDecls(DC, Decls);
-  return const_cast<DeclContext*>(DC)->lookup(Name);
+    ASTDeclContextNameLookupTable *LookupTable =
+        (ASTDeclContextNameLookupTable*)I->NameLookupTableData;
+    for (ASTDeclContextNameLookupTable::item_iterator
+           ItemI = LookupTable->item_begin(),
+           ItemEnd = LookupTable->item_end() ; ItemI != ItemEnd; ++ItemI) {
+      ASTDeclContextNameLookupTable::item_iterator::value_type Val
+          = *ItemI;
+      ASTDeclContextNameLookupTrait::data_type Data = Val.second;
+      Decls.clear();
+      for (; Data.first != Data.second; ++Data.first)
+        Decls.push_back(cast<NamedDecl>(GetDecl(*Data.first)));
+      MaterializeVisibleDeclsForName(DC, Val.first, Decls);
+    }
+  }
 }
 
-void PCHReader::PassInterestingDeclsToConsumer() {
+void ASTReader::PassInterestingDeclsToConsumer() {
   assert(Consumer);
   while (!InterestingDecls.empty()) {
     DeclGroupRef DG(InterestingDecls.front());
     InterestingDecls.pop_front();
-    Consumer->HandleTopLevelDecl(DG);
+    Consumer->HandleInterestingDecl(DG);
   }
 }
 
-void PCHReader::StartTranslationUnit(ASTConsumer *Consumer) {
+void ASTReader::StartTranslationUnit(ASTConsumer *Consumer) {
   this->Consumer = Consumer;
 
   if (!Consumer)
@@ -2828,8 +3281,8 @@ void PCHReader::StartTranslationUnit(ASTConsumer *Consumer) {
   PassInterestingDeclsToConsumer();
 }
 
-void PCHReader::PrintStats() {
-  std::fprintf(stderr, "*** PCH Statistics:\n");
+void ASTReader::PrintStats() {
+  std::fprintf(stderr, "*** AST File Statistics:\n");
 
   unsigned NumTypesLoaded
     = TypesLoaded.size() - std::count(TypesLoaded.begin(), TypesLoaded.end(),
@@ -2864,10 +3317,10 @@ void PCHReader::PrintStats() {
     std::fprintf(stderr, "  %u/%u identifiers read (%f%%)\n",
                  NumIdentifiersLoaded, (unsigned)IdentifiersLoaded.size(),
                  ((float)NumIdentifiersLoaded/IdentifiersLoaded.size() * 100));
-  if (TotalNumSelectors)
+  if (!SelectorsLoaded.empty())
     std::fprintf(stderr, "  %u/%u selectors read (%f%%)\n",
-                 NumSelectorsLoaded, TotalNumSelectors,
-                 ((float)NumSelectorsLoaded/TotalNumSelectors * 100));
+                 NumSelectorsLoaded, (unsigned)SelectorsLoaded.size(),
+                 ((float)NumSelectorsLoaded/SelectorsLoaded.size() * 100));
   if (TotalNumStatements)
     std::fprintf(stderr, "  %u/%u statements read (%f%%)\n",
                  NumStatementsRead, TotalNumStatements,
@@ -2886,25 +3339,27 @@ void PCHReader::PrintStats() {
                  NumVisibleDeclContextsRead, TotalVisibleDeclContexts,
                  ((float)NumVisibleDeclContextsRead/TotalVisibleDeclContexts
                   * 100));
-  if (TotalSelectorsInMethodPool) {
+  if (TotalNumMethodPoolEntries) {
     std::fprintf(stderr, "  %u/%u method pool entries read (%f%%)\n",
-                 NumMethodPoolSelectorsRead, TotalSelectorsInMethodPool,
-                 ((float)NumMethodPoolSelectorsRead/TotalSelectorsInMethodPool
+                 NumMethodPoolEntriesRead, TotalNumMethodPoolEntries,
+                 ((float)NumMethodPoolEntriesRead/TotalNumMethodPoolEntries
                   * 100));
     std::fprintf(stderr, "  %u method pool misses\n", NumMethodPoolMisses);
   }
   std::fprintf(stderr, "\n");
 }
 
-void PCHReader::InitializeSema(Sema &S) {
+void ASTReader::InitializeSema(Sema &S) {
   SemaObj = &S;
   S.ExternalSource = this;
 
   // Makes sure any declarations that were deserialized "too early"
   // still get added to the identifier's declaration chains.
-  for (unsigned I = 0, N = PreloadedDecls.size(); I != N; ++I) {
-    SemaObj->TUScope->AddDecl(Action::DeclPtrTy::make(PreloadedDecls[I]));
-    SemaObj->IdResolver.AddDecl(PreloadedDecls[I]);
+  if (SemaObj->TUScope) {
+    for (unsigned I = 0, N = PreloadedDecls.size(); I != N; ++I) {
+      SemaObj->TUScope->AddDecl(PreloadedDecls[I]);
+      SemaObj->IdResolver.AddDecl(PreloadedDecls[I]);
+    }
   }
   PreloadedDecls.clear();
 
@@ -2915,11 +3370,26 @@ void PCHReader::InitializeSema(Sema &S) {
     SemaObj->TentativeDefinitions.push_back(Var);
   }
 
-  // If there were any unused static functions, deserialize them and add to
-  // Sema's list of unused static functions.
-  for (unsigned I = 0, N = UnusedStaticFuncs.size(); I != N; ++I) {
-    FunctionDecl *FD = cast<FunctionDecl>(GetDecl(UnusedStaticFuncs[I]));
-    SemaObj->UnusedStaticFuncs.push_back(FD);
+  // If there were any unused file scoped decls, deserialize them and add to
+  // Sema's list of unused file scoped decls.
+  for (unsigned I = 0, N = UnusedFileScopedDecls.size(); I != N; ++I) {
+    DeclaratorDecl *D = cast<DeclaratorDecl>(GetDecl(UnusedFileScopedDecls[I]));
+    SemaObj->UnusedFileScopedDecls.push_back(D);
+  }
+
+  // If there were any weak undeclared identifiers, deserialize them and add to
+  // Sema's list of weak undeclared identifiers.
+  if (!WeakUndeclaredIdentifiers.empty()) {
+    unsigned Idx = 0;
+    for (unsigned I = 0, N = WeakUndeclaredIdentifiers[Idx++]; I != N; ++I) {
+      IdentifierInfo *WeakId = GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
+      IdentifierInfo *AliasId=GetIdentifierInfo(WeakUndeclaredIdentifiers,Idx);
+      SourceLocation Loc = ReadSourceLocation(WeakUndeclaredIdentifiers, Idx);
+      bool Used = WeakUndeclaredIdentifiers[Idx++];
+      Sema::WeakInfo WI(AliasId, Loc);
+      WI.setUsed(Used);
+      SemaObj->WeakUndeclaredIdentifiers.insert(std::make_pair(WeakId, WI));
+    }
   }
 
   // If there were any locally-scoped external declarations,
@@ -2941,13 +3411,15 @@ void PCHReader::InitializeSema(Sema &S) {
 
   // If there were any VTable uses, deserialize the information and add it
   // to Sema's vector and map of VTable uses.
-  unsigned Idx = 0;
-  for (unsigned I = 0, N = VTableUses[Idx++]; I != N; ++I) {
-    CXXRecordDecl *Class = cast<CXXRecordDecl>(GetDecl(VTableUses[Idx++]));
-    SourceLocation Loc = ReadSourceLocation(VTableUses, Idx);
-    bool DefinitionRequired = VTableUses[Idx++];
-    SemaObj->VTableUses.push_back(std::make_pair(Class, Loc));
-    SemaObj->VTablesUsed[Class] = DefinitionRequired;
+  if (!VTableUses.empty()) {
+    unsigned Idx = 0;
+    for (unsigned I = 0, N = VTableUses[Idx++]; I != N; ++I) {
+      CXXRecordDecl *Class = cast<CXXRecordDecl>(GetDecl(VTableUses[Idx++]));
+      SourceLocation Loc = ReadSourceLocation(VTableUses, Idx);
+      bool DefinitionRequired = VTableUses[Idx++];
+      SemaObj->VTableUses.push_back(std::make_pair(Class, Loc));
+      SemaObj->VTablesUsed[Class] = DefinitionRequired;
+    }
   }
 
   // If there were any dynamic classes declarations, deserialize them
@@ -2955,51 +3427,104 @@ void PCHReader::InitializeSema(Sema &S) {
   for (unsigned I = 0, N = DynamicClasses.size(); I != N; ++I)
     SemaObj->DynamicClasses.push_back(
                                cast<CXXRecordDecl>(GetDecl(DynamicClasses[I])));
+
+  // If there were any pending implicit instantiations, deserialize them
+  // and add them to Sema's queue of such instantiations.
+  assert(PendingInstantiations.size() % 2 == 0 && "Expected pairs of entries");
+  for (unsigned Idx = 0, N = PendingInstantiations.size(); Idx < N;) {
+    ValueDecl *D=cast<ValueDecl>(GetDecl(PendingInstantiations[Idx++]));
+    SourceLocation Loc = ReadSourceLocation(PendingInstantiations, Idx);
+    SemaObj->PendingInstantiations.push_back(std::make_pair(D, Loc));
+  }
+
+  // Load the offsets of the declarations that Sema references.
+  // They will be lazily deserialized when needed.
+  if (!SemaDeclRefs.empty()) {
+    assert(SemaDeclRefs.size() == 2 && "More decl refs than expected!");
+    SemaObj->StdNamespace = SemaDeclRefs[0];
+    SemaObj->StdBadAlloc = SemaDeclRefs[1];
+  }
+
+  // If there are @selector references added them to its pool. This is for
+  // implementation of -Wselector.
+  if (!ReferencedSelectorsData.empty()) {
+    unsigned int DataSize = ReferencedSelectorsData.size()-1;
+    unsigned I = 0;
+    while (I < DataSize) {
+      Selector Sel = DecodeSelector(ReferencedSelectorsData[I++]);
+      SourceLocation SelLoc = 
+        SourceLocation::getFromRawEncoding(ReferencedSelectorsData[I++]);
+      SemaObj->ReferencedSelectors.insert(std::make_pair(Sel, SelLoc));
+    }
+  }
 }
 
-IdentifierInfo* PCHReader::get(const char *NameStart, const char *NameEnd) {
-  // Try to find this name within our on-disk hash table
-  PCHIdentifierLookupTable *IdTable
-    = (PCHIdentifierLookupTable *)IdentifierLookupTable;
-  std::pair<const char*, unsigned> Key(NameStart, NameEnd - NameStart);
-  PCHIdentifierLookupTable::iterator Pos = IdTable->find(Key);
-  if (Pos == IdTable->end())
-    return 0;
+IdentifierInfo* ASTReader::get(const char *NameStart, const char *NameEnd) {
+  // Try to find this name within our on-disk hash tables. We start with the
+  // most recent one, since that one contains the most up-to-date info.
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    ASTIdentifierLookupTable *IdTable
+        = (ASTIdentifierLookupTable *)Chain[I]->IdentifierLookupTable;
+    if (!IdTable)
+      continue;
+    std::pair<const char*, unsigned> Key(NameStart, NameEnd - NameStart);
+    ASTIdentifierLookupTable::iterator Pos = IdTable->find(Key);
+    if (Pos == IdTable->end())
+      continue;
 
-  // Dereferencing the iterator has the effect of building the
-  // IdentifierInfo node and populating it with the various
-  // declarations it needs.
-  return *Pos;
+    // Dereferencing the iterator has the effect of building the
+    // IdentifierInfo node and populating it with the various
+    // declarations it needs.
+    return *Pos;
+  }
+  return 0;
 }
 
 std::pair<ObjCMethodList, ObjCMethodList>
-PCHReader::ReadMethodPool(Selector Sel) {
-  if (!MethodPoolLookupTable)
-    return std::pair<ObjCMethodList, ObjCMethodList>();
+ASTReader::ReadMethodPool(Selector Sel) {
+  // Find this selector in a hash table. We want to find the most recent entry.
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    PerFileData &F = *Chain[I];
+    if (!F.SelectorLookupTable)
+      continue;
 
-  // Try to find this selector within our on-disk hash table.
-  PCHMethodPoolLookupTable *PoolTable
-    = (PCHMethodPoolLookupTable*)MethodPoolLookupTable;
-  PCHMethodPoolLookupTable::iterator Pos = PoolTable->find(Sel);
-  if (Pos == PoolTable->end()) {
-    ++NumMethodPoolMisses;
-    return std::pair<ObjCMethodList, ObjCMethodList>();;
+    ASTSelectorLookupTable *PoolTable
+      = (ASTSelectorLookupTable*)F.SelectorLookupTable;
+    ASTSelectorLookupTable::iterator Pos = PoolTable->find(Sel);
+    if (Pos != PoolTable->end()) {
+      ++NumSelectorsRead;
+      // FIXME: Not quite happy with the statistics here. We probably should
+      // disable this tracking when called via LoadSelector.
+      // Also, should entries without methods count as misses?
+      ++NumMethodPoolEntriesRead;
+      ASTSelectorLookupTrait::data_type Data = *Pos;
+      if (DeserializationListener)
+        DeserializationListener->SelectorRead(Data.ID, Sel);
+      return std::make_pair(Data.Instance, Data.Factory);
+    }
   }
 
-  ++NumMethodPoolSelectorsRead;
-  return *Pos;
+  ++NumMethodPoolMisses;
+  return std::pair<ObjCMethodList, ObjCMethodList>();
+}
+
+void ASTReader::LoadSelector(Selector Sel) {
+  // It would be complicated to avoid reading the methods anyway. So don't.
+  ReadMethodPool(Sel);
 }
 
-void PCHReader::SetIdentifierInfo(unsigned ID, IdentifierInfo *II) {
+void ASTReader::SetIdentifierInfo(unsigned ID, IdentifierInfo *II) {
   assert(ID && "Non-zero identifier ID required");
   assert(ID <= IdentifiersLoaded.size() && "identifier ID out of range");
   IdentifiersLoaded[ID - 1] = II;
+  if (DeserializationListener)
+    DeserializationListener->IdentifierRead(ID, II);
 }
 
 /// \brief Set the globally-visible declarations associated with the given
 /// identifier.
 ///
-/// If the PCH reader is currently in a state where the given declaration IDs
+/// If the AST reader is currently in a state where the given declaration IDs
 /// cannot safely be resolved, they are queued until it is safe to resolve
 /// them.
 ///
@@ -3013,10 +3538,10 @@ void PCHReader::SetIdentifierInfo(unsigned ID, IdentifierInfo *II) {
 /// this call is non-recursive, and therefore the globally-visible declarations
 /// will not be placed onto the pending queue.
 void
-PCHReader::SetGloballyVisibleDecls(IdentifierInfo *II,
+ASTReader::SetGloballyVisibleDecls(IdentifierInfo *II,
                               const llvm::SmallVectorImpl<uint32_t> &DeclIDs,
                                    bool Nonrecursive) {
-  if (CurrentlyLoadingTypeOrDecl && !Nonrecursive) {
+  if (NumCurrentElementsDeserializing && !Nonrecursive) {
     PendingIdentifierInfos.push_back(PendingIdentifierInfo());
     PendingIdentifierInfo &PII = PendingIdentifierInfos.back();
     PII.II = II;
@@ -3028,11 +3553,13 @@ PCHReader::SetGloballyVisibleDecls(IdentifierInfo *II,
   for (unsigned I = 0, N = DeclIDs.size(); I != N; ++I) {
     NamedDecl *D = cast<NamedDecl>(GetDecl(DeclIDs[I]));
     if (SemaObj) {
-      // Introduce this declaration into the translation-unit scope
-      // and add it to the declaration chain for this identifier, so
-      // that (unqualified) name lookup will find it.
-      SemaObj->TUScope->AddDecl(Action::DeclPtrTy::make(D));
-      SemaObj->IdResolver.AddDeclToIdentifierChain(II, D);
+      if (SemaObj->TUScope) {
+        // Introduce this declaration into the translation-unit scope
+        // and add it to the declaration chain for this identifier, so
+        // that (unqualified) name lookup will find it.
+        SemaObj->TUScope->AddDecl(D);
+        SemaObj->IdResolver.AddDeclToIdentifierChain(II, D);
+      }
     } else {
       // Queue this declaration so that it will be added to the
       // translation unit scope and identifier's declaration chain
@@ -3042,74 +3569,92 @@ PCHReader::SetGloballyVisibleDecls(IdentifierInfo *II,
   }
 }
 
-IdentifierInfo *PCHReader::DecodeIdentifierInfo(unsigned ID) {
+IdentifierInfo *ASTReader::DecodeIdentifierInfo(unsigned ID) {
   if (ID == 0)
     return 0;
 
-  if (!IdentifierTableData || IdentifiersLoaded.empty()) {
-    Error("no identifier table in PCH file");
+  if (IdentifiersLoaded.empty()) {
+    Error("no identifier table in AST file");
     return 0;
   }
 
   assert(PP && "Forgot to set Preprocessor ?");
-  if (!IdentifiersLoaded[ID - 1]) {
-    uint32_t Offset = IdentifierOffsets[ID - 1];
-    const char *Str = IdentifierTableData + Offset;
+  ID -= 1;
+  if (!IdentifiersLoaded[ID]) {
+    unsigned Index = ID;
+    const char *Str = 0;
+    for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+      PerFileData *F = Chain[N - I - 1];
+      if (Index < F->LocalNumIdentifiers) {
+         uint32_t Offset = F->IdentifierOffsets[Index];
+         Str = F->IdentifierTableData + Offset;
+         break;
+      }
+      Index -= F->LocalNumIdentifiers;
+    }
+    assert(Str && "Broken Chain");
 
-    // All of the strings in the PCH file are preceded by a 16-bit
-    // length. Extract that 16-bit length to avoid having to execute
-    // strlen().
+    // All of the strings in the AST file are preceded by a 16-bit length.
+    // Extract that 16-bit length to avoid having to execute strlen().
     // NOTE: 'StrLenPtr' is an 'unsigned char*' so that we load bytes as
     //  unsigned integers.  This is important to avoid integer overflow when
     //  we cast them to 'unsigned'.
     const unsigned char *StrLenPtr = (const unsigned char*) Str - 2;
     unsigned StrLen = (((unsigned) StrLenPtr[0])
                        | (((unsigned) StrLenPtr[1]) << 8)) - 1;
-    IdentifiersLoaded[ID - 1]
+    IdentifiersLoaded[ID]
       = &PP->getIdentifierTable().get(Str, StrLen);
+    if (DeserializationListener)
+      DeserializationListener->IdentifierRead(ID + 1, IdentifiersLoaded[ID]);
   }
 
-  return IdentifiersLoaded[ID - 1];
+  return IdentifiersLoaded[ID];
 }
 
-void PCHReader::ReadSLocEntry(unsigned ID) {
+void ASTReader::ReadSLocEntry(unsigned ID) {
   ReadSLocEntryRecord(ID);
 }
 
-Selector PCHReader::DecodeSelector(unsigned ID) {
+Selector ASTReader::DecodeSelector(unsigned ID) {
   if (ID == 0)
     return Selector();
 
-  if (!MethodPoolLookupTableData)
-    return Selector();
-
-  if (ID > TotalNumSelectors) {
-    Error("selector ID out of range in PCH file");
+  if (ID > SelectorsLoaded.size()) {
+    Error("selector ID out of range in AST file");
     return Selector();
   }
 
-  unsigned Index = ID - 1;
-  if (SelectorsLoaded[Index].getAsOpaquePtr() == 0) {
+  if (SelectorsLoaded[ID - 1].getAsOpaquePtr() == 0) {
     // Load this selector from the selector table.
-    // FIXME: endianness portability issues with SelectorOffsets table
-    PCHMethodPoolLookupTrait Trait(*this);
-    SelectorsLoaded[Index]
-      = Trait.ReadKey(MethodPoolLookupTableData + SelectorOffsets[Index], 0);
+    unsigned Idx = ID - 1;
+    for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+      PerFileData &F = *Chain[N - I - 1];
+      if (Idx < F.LocalNumSelectors) {
+        ASTSelectorLookupTrait Trait(*this);
+        SelectorsLoaded[ID - 1] =
+           Trait.ReadKey(F.SelectorLookupTableData + F.SelectorOffsets[Idx], 0);
+        if (DeserializationListener)
+          DeserializationListener->SelectorRead(ID, SelectorsLoaded[ID - 1]);
+        break;
+      }
+      Idx -= F.LocalNumSelectors;
+    }
   }
 
-  return SelectorsLoaded[Index];
+  return SelectorsLoaded[ID - 1];
 }
 
-Selector PCHReader::GetExternalSelector(uint32_t ID) { 
+Selector ASTReader::GetExternalSelector(uint32_t ID) { 
   return DecodeSelector(ID);
 }
 
-uint32_t PCHReader::GetNumExternalSelectors() {
-  return TotalNumSelectors + 1;
+uint32_t ASTReader::GetNumExternalSelectors() {
+  // ID 0 (the null selector) is considered an external selector.
+  return getTotalNumSelectors() + 1;
 }
 
 DeclarationName
-PCHReader::ReadDeclarationName(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadDeclarationName(const RecordData &Record, unsigned &Idx) {
   DeclarationName::NameKind Kind = (DeclarationName::NameKind)Record[Idx++];
   switch (Kind) {
   case DeclarationName::Identifier:
@@ -3149,7 +3694,7 @@ PCHReader::ReadDeclarationName(const RecordData &Record, unsigned &Idx) {
 }
 
 TemplateName
-PCHReader::ReadTemplateName(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadTemplateName(const RecordData &Record, unsigned &Idx) {
   TemplateName::NameKind Kind = (TemplateName::NameKind)Record[Idx++]; 
   switch (Kind) {
   case TemplateName::Template:
@@ -3186,7 +3731,8 @@ PCHReader::ReadTemplateName(const RecordData &Record, unsigned &Idx) {
 }
 
 TemplateArgument
-PCHReader::ReadTemplateArgument(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadTemplateArgument(llvm::BitstreamCursor &DeclsCursor,
+                                const RecordData &Record, unsigned &Idx) {
   switch ((TemplateArgument::ArgKind)Record[Idx++]) {
   case TemplateArgument::Null:
     return TemplateArgument();
@@ -3202,13 +3748,13 @@ PCHReader::ReadTemplateArgument(const RecordData &Record, unsigned &Idx) {
   case TemplateArgument::Template:
     return TemplateArgument(ReadTemplateName(Record, Idx));
   case TemplateArgument::Expression:
-    return TemplateArgument(ReadExpr());
+    return TemplateArgument(ReadExpr(DeclsCursor));
   case TemplateArgument::Pack: {
     unsigned NumArgs = Record[Idx++];
     llvm::SmallVector<TemplateArgument, 8> Args;
     Args.reserve(NumArgs);
     while (NumArgs--)
-      Args.push_back(ReadTemplateArgument(Record, Idx));
+      Args.push_back(ReadTemplateArgument(DeclsCursor, Record, Idx));
     TemplateArgument TemplArg;
     TemplArg.setArgumentPack(Args.data(), Args.size(), /*CopyArgs=*/true);
     return TemplArg;
@@ -3220,7 +3766,7 @@ PCHReader::ReadTemplateArgument(const RecordData &Record, unsigned &Idx) {
 }
 
 TemplateParameterList *
-PCHReader::ReadTemplateParameterList(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadTemplateParameterList(const RecordData &Record, unsigned &Idx) {
   SourceLocation TemplateLoc = ReadSourceLocation(Record, Idx);
   SourceLocation LAngleLoc = ReadSourceLocation(Record, Idx);
   SourceLocation RAngleLoc = ReadSourceLocation(Record, Idx);
@@ -3238,17 +3784,18 @@ PCHReader::ReadTemplateParameterList(const RecordData &Record, unsigned &Idx) {
 }
 
 void
-PCHReader::
+ASTReader::
 ReadTemplateArgumentList(llvm::SmallVector<TemplateArgument, 8> &TemplArgs,
+                         llvm::BitstreamCursor &DeclsCursor,
                          const RecordData &Record, unsigned &Idx) {
   unsigned NumTemplateArgs = Record[Idx++];
   TemplArgs.reserve(NumTemplateArgs);
   while (NumTemplateArgs--)
-    TemplArgs.push_back(ReadTemplateArgument(Record, Idx));
+    TemplArgs.push_back(ReadTemplateArgument(DeclsCursor, Record, Idx));
 }
 
 /// \brief Read a UnresolvedSet structure.
-void PCHReader::ReadUnresolvedSet(UnresolvedSetImpl &Set,
+void ASTReader::ReadUnresolvedSet(UnresolvedSetImpl &Set,
                                   const RecordData &Record, unsigned &Idx) {
   unsigned NumDecls = Record[Idx++];
   while (NumDecls--) {
@@ -3259,17 +3806,82 @@ void PCHReader::ReadUnresolvedSet(UnresolvedSetImpl &Set,
 }
 
 CXXBaseSpecifier
-PCHReader::ReadCXXBaseSpecifier(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadCXXBaseSpecifier(llvm::BitstreamCursor &DeclsCursor,
+                                const RecordData &Record, unsigned &Idx) {
   bool isVirtual = static_cast<bool>(Record[Idx++]);
   bool isBaseOfClass = static_cast<bool>(Record[Idx++]);
   AccessSpecifier AS = static_cast<AccessSpecifier>(Record[Idx++]);
-  QualType T = GetType(Record[Idx++]);
+  TypeSourceInfo *TInfo = GetTypeSourceInfo(DeclsCursor, Record, Idx);
   SourceRange Range = ReadSourceRange(Record, Idx);
-  return CXXBaseSpecifier(Range, isVirtual, isBaseOfClass, AS, T);
+  return CXXBaseSpecifier(Range, isVirtual, isBaseOfClass, AS, TInfo);
+}
+
+std::pair<CXXBaseOrMemberInitializer **, unsigned>
+ASTReader::ReadCXXBaseOrMemberInitializers(llvm::BitstreamCursor &Cursor,
+                                           const RecordData &Record,
+                                           unsigned &Idx) {
+  CXXBaseOrMemberInitializer **BaseOrMemberInitializers = 0;
+  unsigned NumInitializers = Record[Idx++];
+  if (NumInitializers) {
+    ASTContext &C = *getContext();
+
+    BaseOrMemberInitializers
+        = new (C) CXXBaseOrMemberInitializer*[NumInitializers];
+    for (unsigned i=0; i != NumInitializers; ++i) {
+      TypeSourceInfo *BaseClassInfo = 0;
+      bool IsBaseVirtual = false;
+      FieldDecl *Member = 0;
+  
+      bool IsBaseInitializer = Record[Idx++];
+      if (IsBaseInitializer) {
+        BaseClassInfo = GetTypeSourceInfo(Cursor, Record, Idx);
+        IsBaseVirtual = Record[Idx++];
+      } else {
+        Member = cast<FieldDecl>(GetDecl(Record[Idx++]));
+      }
+      SourceLocation MemberLoc = ReadSourceLocation(Record, Idx);
+      Expr *Init = ReadExpr(Cursor);
+      FieldDecl *AnonUnionMember
+          = cast_or_null<FieldDecl>(GetDecl(Record[Idx++]));
+      SourceLocation LParenLoc = ReadSourceLocation(Record, Idx);
+      SourceLocation RParenLoc = ReadSourceLocation(Record, Idx);
+      bool IsWritten = Record[Idx++];
+      unsigned SourceOrderOrNumArrayIndices;
+      llvm::SmallVector<VarDecl *, 8> Indices;
+      if (IsWritten) {
+        SourceOrderOrNumArrayIndices = Record[Idx++];
+      } else {
+        SourceOrderOrNumArrayIndices = Record[Idx++];
+        Indices.reserve(SourceOrderOrNumArrayIndices);
+        for (unsigned i=0; i != SourceOrderOrNumArrayIndices; ++i)
+          Indices.push_back(cast<VarDecl>(GetDecl(Record[Idx++])));
+      }
+      
+      CXXBaseOrMemberInitializer *BOMInit;
+      if (IsBaseInitializer) {
+        BOMInit = new (C) CXXBaseOrMemberInitializer(C, BaseClassInfo,
+                                                     IsBaseVirtual, LParenLoc,
+                                                     Init, RParenLoc);
+      } else if (IsWritten) {
+        BOMInit = new (C) CXXBaseOrMemberInitializer(C, Member, MemberLoc,
+                                                     LParenLoc, Init, RParenLoc);
+      } else {
+        BOMInit = CXXBaseOrMemberInitializer::Create(C, Member, MemberLoc,
+                                                     LParenLoc, Init, RParenLoc,
+                                                     Indices.data(),
+                                                     Indices.size());
+      }
+
+      BOMInit->setAnonUnionMember(AnonUnionMember);
+      BaseOrMemberInitializers[i] = BOMInit;
+    }
+  }
+
+  return std::make_pair(BaseOrMemberInitializers, NumInitializers);
 }
 
 NestedNameSpecifier *
-PCHReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
   unsigned N = Record[Idx++];
   NestedNameSpecifier *NNS = 0, *Prev = 0;
   for (unsigned I = 0; I != N; ++I) {
@@ -3308,14 +3920,14 @@ PCHReader::ReadNestedNameSpecifier(const RecordData &Record, unsigned &Idx) {
 }
 
 SourceRange
-PCHReader::ReadSourceRange(const RecordData &Record, unsigned &Idx) {
+ASTReader::ReadSourceRange(const RecordData &Record, unsigned &Idx) {
   SourceLocation beg = SourceLocation::getFromRawEncoding(Record[Idx++]);
   SourceLocation end = SourceLocation::getFromRawEncoding(Record[Idx++]);
   return SourceRange(beg, end);
 }
 
 /// \brief Read an integral value
-llvm::APInt PCHReader::ReadAPInt(const RecordData &Record, unsigned &Idx) {
+llvm::APInt ASTReader::ReadAPInt(const RecordData &Record, unsigned &Idx) {
   unsigned BitWidth = Record[Idx++];
   unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
   llvm::APInt Result(BitWidth, NumWords, &Record[Idx]);
@@ -3324,61 +3936,61 @@ llvm::APInt PCHReader::ReadAPInt(const RecordData &Record, unsigned &Idx) {
 }
 
 /// \brief Read a signed integral value
-llvm::APSInt PCHReader::ReadAPSInt(const RecordData &Record, unsigned &Idx) {
+llvm::APSInt ASTReader::ReadAPSInt(const RecordData &Record, unsigned &Idx) {
   bool isUnsigned = Record[Idx++];
   return llvm::APSInt(ReadAPInt(Record, Idx), isUnsigned);
 }
 
 /// \brief Read a floating-point value
-llvm::APFloat PCHReader::ReadAPFloat(const RecordData &Record, unsigned &Idx) {
+llvm::APFloat ASTReader::ReadAPFloat(const RecordData &Record, unsigned &Idx) {
   return llvm::APFloat(ReadAPInt(Record, Idx));
 }
 
 // \brief Read a string
-std::string PCHReader::ReadString(const RecordData &Record, unsigned &Idx) {
+std::string ASTReader::ReadString(const RecordData &Record, unsigned &Idx) {
   unsigned Len = Record[Idx++];
   std::string Result(Record.data() + Idx, Record.data() + Idx + Len);
   Idx += Len;
   return Result;
 }
 
-CXXTemporary *PCHReader::ReadCXXTemporary(const RecordData &Record,
+CXXTemporary *ASTReader::ReadCXXTemporary(const RecordData &Record,
                                           unsigned &Idx) {
   CXXDestructorDecl *Decl = cast<CXXDestructorDecl>(GetDecl(Record[Idx++]));
   return CXXTemporary::Create(*Context, Decl);
 }
 
-DiagnosticBuilder PCHReader::Diag(unsigned DiagID) {
+DiagnosticBuilder ASTReader::Diag(unsigned DiagID) {
   return Diag(SourceLocation(), DiagID);
 }
 
-DiagnosticBuilder PCHReader::Diag(SourceLocation Loc, unsigned DiagID) {
+DiagnosticBuilder ASTReader::Diag(SourceLocation Loc, unsigned DiagID) {
   return Diags.Report(FullSourceLoc(Loc, SourceMgr), DiagID);
 }
 
 /// \brief Retrieve the identifier table associated with the
 /// preprocessor.
-IdentifierTable &PCHReader::getIdentifierTable() {
+IdentifierTable &ASTReader::getIdentifierTable() {
   assert(PP && "Forgot to set Preprocessor ?");
   return PP->getIdentifierTable();
 }
 
 /// \brief Record that the given ID maps to the given switch-case
 /// statement.
-void PCHReader::RecordSwitchCaseID(SwitchCase *SC, unsigned ID) {
+void ASTReader::RecordSwitchCaseID(SwitchCase *SC, unsigned ID) {
   assert(SwitchCaseStmts[ID] == 0 && "Already have a SwitchCase with this ID");
   SwitchCaseStmts[ID] = SC;
 }
 
 /// \brief Retrieve the switch-case statement with the given ID.
-SwitchCase *PCHReader::getSwitchCaseWithID(unsigned ID) {
+SwitchCase *ASTReader::getSwitchCaseWithID(unsigned ID) {
   assert(SwitchCaseStmts[ID] != 0 && "No SwitchCase with this ID");
   return SwitchCaseStmts[ID];
 }
 
 /// \brief Record that the given label statement has been
 /// deserialized and has the given ID.
-void PCHReader::RecordLabelStmt(LabelStmt *S, unsigned ID) {
+void ASTReader::RecordLabelStmt(LabelStmt *S, unsigned ID) {
   assert(LabelStmts.find(ID) == LabelStmts.end() &&
          "Deserialized label twice");
   LabelStmts[ID] = S;
@@ -3409,7 +4021,7 @@ void PCHReader::RecordLabelStmt(LabelStmt *S, unsigned ID) {
 /// referencing that label occur, this operation may complete
 /// immediately (updating the statement) or it may queue the
 /// statement to be back-patched later.
-void PCHReader::SetLabelOf(GotoStmt *S, unsigned ID) {
+void ASTReader::SetLabelOf(GotoStmt *S, unsigned ID) {
   std::map<unsigned, LabelStmt *>::iterator Label = LabelStmts.find(ID);
   if (Label != LabelStmts.end()) {
     // We've already seen this label, so set the label of the goto and
@@ -3429,7 +4041,7 @@ void PCHReader::SetLabelOf(GotoStmt *S, unsigned ID) {
 /// referencing that label occur, this operation may complete
 /// immediately (updating the statement) or it may queue the
 /// statement to be back-patched later.
-void PCHReader::SetLabelOf(AddrLabelExpr *S, unsigned ID) {
+void ASTReader::SetLabelOf(AddrLabelExpr *S, unsigned ID) {
   std::map<unsigned, LabelStmt *>::iterator Label = LabelStmts.find(ID);
   if (Label != LabelStmts.end()) {
     // We've already seen this label, so set the label of the
@@ -3442,28 +4054,94 @@ void PCHReader::SetLabelOf(AddrLabelExpr *S, unsigned ID) {
   }
 }
 
-
-PCHReader::LoadingTypeOrDecl::LoadingTypeOrDecl(PCHReader &Reader)
-  : Reader(Reader), Parent(Reader.CurrentlyLoadingTypeOrDecl) {
-  Reader.CurrentlyLoadingTypeOrDecl = this;
-}
-
-PCHReader::LoadingTypeOrDecl::~LoadingTypeOrDecl() {
-  if (!Parent) {
+void ASTReader::FinishedDeserializing() {
+  assert(NumCurrentElementsDeserializing &&
+         "FinishedDeserializing not paired with StartedDeserializing");
+  if (NumCurrentElementsDeserializing == 1) {
     // If any identifiers with corresponding top-level declarations have
     // been loaded, load those declarations now.
-    while (!Reader.PendingIdentifierInfos.empty()) {
-      Reader.SetGloballyVisibleDecls(Reader.PendingIdentifierInfos.front().II,
-                                 Reader.PendingIdentifierInfos.front().DeclIDs,
-                                     true);
-      Reader.PendingIdentifierInfos.pop_front();
+    while (!PendingIdentifierInfos.empty()) {
+      SetGloballyVisibleDecls(PendingIdentifierInfos.front().II,
+                              PendingIdentifierInfos.front().DeclIDs, true);
+      PendingIdentifierInfos.pop_front();
     }
 
     // We are not in recursive loading, so it's safe to pass the "interesting"
     // decls to the consumer.
-    if (Reader.Consumer)
-      Reader.PassInterestingDeclsToConsumer();
+    if (Consumer)
+      PassInterestingDeclsToConsumer();
   }
+  --NumCurrentElementsDeserializing;
+}
 
-  Reader.CurrentlyLoadingTypeOrDecl = Parent;
+ASTReader::ASTReader(Preprocessor &PP, ASTContext *Context,
+                     const char *isysroot, bool DisableValidation)
+  : Listener(new PCHValidator(PP, *this)), DeserializationListener(0),
+    SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
+    Diags(PP.getDiagnostics()), SemaObj(0), PP(&PP), Context(Context),
+    Consumer(0), isysroot(isysroot), DisableValidation(DisableValidation),
+    NumStatHits(0), NumStatMisses(0), NumSLocEntriesRead(0),
+    TotalNumSLocEntries(0), NumStatementsRead(0), TotalNumStatements(0),
+    NumMacrosRead(0), TotalNumMacros(0), NumSelectorsRead(0),
+    NumMethodPoolEntriesRead(0), NumMethodPoolMisses(0),
+    TotalNumMethodPoolEntries(0), NumLexicalDeclContextsRead(0),
+    TotalLexicalDeclContexts(0), NumVisibleDeclContextsRead(0),
+    TotalVisibleDeclContexts(0), NumCurrentElementsDeserializing(0) {
+  RelocatablePCH = false;
 }
+
+ASTReader::ASTReader(SourceManager &SourceMgr, FileManager &FileMgr,
+                     Diagnostic &Diags, const char *isysroot,
+                     bool DisableValidation)
+  : DeserializationListener(0), SourceMgr(SourceMgr), FileMgr(FileMgr),
+    Diags(Diags), SemaObj(0), PP(0), Context(0), Consumer(0),
+    isysroot(isysroot), DisableValidation(DisableValidation), NumStatHits(0),
+    NumStatMisses(0), NumSLocEntriesRead(0), TotalNumSLocEntries(0),
+    NumStatementsRead(0), TotalNumStatements(0), NumMacrosRead(0),
+    TotalNumMacros(0), NumSelectorsRead(0), NumMethodPoolEntriesRead(0),
+    NumMethodPoolMisses(0), TotalNumMethodPoolEntries(0),
+    NumLexicalDeclContextsRead(0), TotalLexicalDeclContexts(0),
+    NumVisibleDeclContextsRead(0), TotalVisibleDeclContexts(0),
+    NumCurrentElementsDeserializing(0) {
+  RelocatablePCH = false;
+}
+
+ASTReader::~ASTReader() {
+  for (unsigned i = 0, e = Chain.size(); i != e; ++i)
+    delete Chain[e - i - 1];
+  // Delete all visible decl lookup tables
+  for (DeclContextOffsetsMap::iterator I = DeclContextOffsets.begin(),
+                                       E = DeclContextOffsets.end();
+       I != E; ++I) {
+    for (DeclContextInfos::iterator J = I->second.begin(), F = I->second.end();
+         J != F; ++J) {
+      if (J->NameLookupTableData)
+        delete static_cast<ASTDeclContextNameLookupTable*>(
+            J->NameLookupTableData);
+    }
+  }
+  for (DeclContextVisibleUpdatesPending::iterator
+           I = PendingVisibleUpdates.begin(),
+           E = PendingVisibleUpdates.end();
+       I != E; ++I) {
+    for (DeclContextVisibleUpdates::iterator J = I->second.begin(),
+                                             F = I->second.end();
+         J != F; ++J)
+      delete static_cast<ASTDeclContextNameLookupTable*>(*J);
+  }
+}
+
+ASTReader::PerFileData::PerFileData()
+  : StatCache(0), LocalNumSLocEntries(0), LocalNumTypes(0), TypeOffsets(0),
+    LocalNumDecls(0), DeclOffsets(0), LocalNumIdentifiers(0),
+    IdentifierOffsets(0), IdentifierTableData(0), IdentifierLookupTable(0),
+    LocalNumMacroDefinitions(0), MacroDefinitionOffsets(0),
+    NumPreallocatedPreprocessingEntities(0), SelectorLookupTable(0),
+    SelectorLookupTableData(0), SelectorOffsets(0), LocalNumSelectors(0)
+{}
+
+ASTReader::PerFileData::~PerFileData() {
+  delete static_cast<ASTIdentifierLookupTable *>(IdentifierLookupTable);
+  delete static_cast<ASTSelectorLookupTable *>(SelectorLookupTable);
+}
+
diff --git a/lib/Frontend/PCHReaderDecl.cpp b/lib/Serialization/ASTReaderDecl.cpp
index 742f0e46b92c..7adbe1220ff7 100644
--- a/lib/Frontend/PCHReaderDecl.cpp
+++ b/lib/Serialization/ASTReaderDecl.cpp
@@ -1,4 +1,4 @@
-//===--- PCHReaderDecl.cpp - Decl Deserialization ---------------*- C++ -*-===//
+//===--- ASTReaderDecl.cpp - Decl Deserialization ---------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,12 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the PCHReader::ReadDeclRecord method, which is the
+// This file implements the ASTReader::ReadDeclRecord method, which is the
 // entrypoint for loading a decl.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/PCHReader.h"
+#include "clang/Serialization/ASTReader.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclVisitor.h"
@@ -21,23 +21,29 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 using namespace clang;
-
+using namespace clang::serialization;
 
 //===----------------------------------------------------------------------===//
 // Declaration deserialization
 //===----------------------------------------------------------------------===//
 
 namespace clang {
-  class PCHDeclReader : public DeclVisitor<PCHDeclReader, void> {
-    PCHReader &Reader;
-    const PCHReader::RecordData &Record;
+  class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
+    ASTReader &Reader;
+    llvm::BitstreamCursor &Cursor;
+    const DeclID ThisDeclID;
+    const ASTReader::RecordData &Record;
     unsigned &Idx;
-    pch::TypeID TypeIDForTypeDecl;
+    TypeID TypeIDForTypeDecl;
+
+    uint64_t GetCurrentCursorOffset();
 
   public:
-    PCHDeclReader(PCHReader &Reader, const PCHReader::RecordData &Record,
+    ASTDeclReader(ASTReader &Reader, llvm::BitstreamCursor &Cursor,
+                  DeclID thisDeclID, const ASTReader::RecordData &Record,
                   unsigned &Idx)
-      : Reader(Reader), Record(Record), Idx(Idx), TypeIDForTypeDecl(0) { }
+      : Reader(Reader), Cursor(Cursor), ThisDeclID(thisDeclID), Record(Record),
+        Idx(Idx), TypeIDForTypeDecl(0) { }
 
     void Visit(Decl *D);
 
@@ -74,6 +80,7 @@ namespace clang {
     void VisitParmVarDecl(ParmVarDecl *PD);
     void VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D);
     void VisitTemplateDecl(TemplateDecl *D);
+    void VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D);
     void VisitClassTemplateDecl(ClassTemplateDecl *D);
     void VisitFunctionTemplateDecl(FunctionTemplateDecl *D);
     void VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D);
@@ -88,6 +95,7 @@ namespace clang {
     void VisitBlockDecl(BlockDecl *BD);
 
     std::pair<uint64_t, uint64_t> VisitDeclContext(DeclContext *DC);
+    template <typename T> void VisitRedeclarable(Redeclarable<T> *D);
 
     // FIXME: Reorder according to DeclNodes.td?
     void VisitObjCMethodDecl(ObjCMethodDecl *D);
@@ -108,8 +116,21 @@ namespace clang {
   };
 }
 
-void PCHDeclReader::Visit(Decl *D) {
-  DeclVisitor<PCHDeclReader, void>::Visit(D);
+uint64_t ASTDeclReader::GetCurrentCursorOffset() {
+  uint64_t Off = 0;
+  for (unsigned I = 0, N = Reader.Chain.size(); I != N; ++I) {
+    ASTReader::PerFileData &F = *Reader.Chain[N - I - 1];
+    if (&Cursor == &F.DeclsCursor) {
+      Off += F.DeclsCursor.GetCurrentBitNo();
+      break;
+    }
+    Off += F.SizeInBits;
+  }
+  return Off;
+}
+
+void ASTDeclReader::Visit(Decl *D) {
+  DeclVisitor<ASTDeclReader, void>::Visit(D);
 
   if (TypeDecl *TD = dyn_cast<TypeDecl>(D)) {
     // if we have a fully initialized TypeDecl, we can safely read its type now.
@@ -117,51 +138,53 @@ void PCHDeclReader::Visit(Decl *D) {
   } else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // FunctionDecl's body was written last after all other Stmts/Exprs.
     if (Record[Idx++])
-      FD->setLazyBody(Reader.getDeclsCursor().GetCurrentBitNo());
+      FD->setLazyBody(GetCurrentCursorOffset());
   }
 }
 
-void PCHDeclReader::VisitDecl(Decl *D) {
+void ASTDeclReader::VisitDecl(Decl *D) {
   D->setDeclContext(cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++])));
   D->setLexicalDeclContext(
                      cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++])));
   D->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
   D->setInvalidDecl(Record[Idx++]);
-  if (Record[Idx++])
-    D->initAttrs(Reader.ReadAttributes());
+  if (Record[Idx++]) {
+    AttrVec Attrs;
+    Reader.ReadAttributes(Cursor, Attrs);
+    D->setAttrs(Attrs);
+  }
   D->setImplicit(Record[Idx++]);
   D->setUsed(Record[Idx++]);
   D->setAccess((AccessSpecifier)Record[Idx++]);
   D->setPCHLevel(Record[Idx++] + 1);
 }
 
-void PCHDeclReader::VisitTranslationUnitDecl(TranslationUnitDecl *TU) {
+void ASTDeclReader::VisitTranslationUnitDecl(TranslationUnitDecl *TU) {
   VisitDecl(TU);
   TU->setAnonymousNamespace(
                     cast_or_null<NamespaceDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitNamedDecl(NamedDecl *ND) {
+void ASTDeclReader::VisitNamedDecl(NamedDecl *ND) {
   VisitDecl(ND);
   ND->setDeclName(Reader.ReadDeclarationName(Record, Idx));
 }
 
-void PCHDeclReader::VisitTypeDecl(TypeDecl *TD) {
+void ASTDeclReader::VisitTypeDecl(TypeDecl *TD) {
   VisitNamedDecl(TD);
   // Delay type reading until after we have fully initialized the decl.
   TypeIDForTypeDecl = Record[Idx++];
 }
 
-void PCHDeclReader::VisitTypedefDecl(TypedefDecl *TD) {
+void ASTDeclReader::VisitTypedefDecl(TypedefDecl *TD) {
   VisitTypeDecl(TD);
-  TD->setTypeSourceInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  TD->setTypeSourceInfo(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
 }
 
-void PCHDeclReader::VisitTagDecl(TagDecl *TD) {
+void ASTDeclReader::VisitTagDecl(TagDecl *TD) {
   VisitTypeDecl(TD);
   TD->IdentifierNamespace = Record[Idx++];
-  TD->setPreviousDeclaration(
-                        cast_or_null<TagDecl>(Reader.GetDecl(Record[Idx++])));
+  VisitRedeclarable(TD);
   TD->setTagKind((TagDecl::TagKind)Record[Idx++]);
   TD->setDefinition(Record[Idx++]);
   TD->setEmbeddedInDeclarator(Record[Idx++]);
@@ -172,7 +195,7 @@ void PCHDeclReader::VisitTagDecl(TagDecl *TD) {
                     cast_or_null<TypedefDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitEnumDecl(EnumDecl *ED) {
+void ASTDeclReader::VisitEnumDecl(EnumDecl *ED) {
   VisitTagDecl(ED);
   ED->setIntegerType(Reader.GetType(Record[Idx++]));
   ED->setPromotionType(Reader.GetType(Record[Idx++]));
@@ -182,35 +205,36 @@ void PCHDeclReader::VisitEnumDecl(EnumDecl *ED) {
                          cast_or_null<EnumDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitRecordDecl(RecordDecl *RD) {
+void ASTDeclReader::VisitRecordDecl(RecordDecl *RD) {
   VisitTagDecl(RD);
   RD->setHasFlexibleArrayMember(Record[Idx++]);
   RD->setAnonymousStructOrUnion(Record[Idx++]);
   RD->setHasObjectMember(Record[Idx++]);
 }
 
-void PCHDeclReader::VisitValueDecl(ValueDecl *VD) {
+void ASTDeclReader::VisitValueDecl(ValueDecl *VD) {
   VisitNamedDecl(VD);
   VD->setType(Reader.GetType(Record[Idx++]));
 }
 
-void PCHDeclReader::VisitEnumConstantDecl(EnumConstantDecl *ECD) {
+void ASTDeclReader::VisitEnumConstantDecl(EnumConstantDecl *ECD) {
   VisitValueDecl(ECD);
   if (Record[Idx++])
-    ECD->setInitExpr(Reader.ReadExpr());
+    ECD->setInitExpr(Reader.ReadExpr(Cursor));
   ECD->setInitVal(Reader.ReadAPSInt(Record, Idx));
 }
 
-void PCHDeclReader::VisitDeclaratorDecl(DeclaratorDecl *DD) {
+void ASTDeclReader::VisitDeclaratorDecl(DeclaratorDecl *DD) {
   VisitValueDecl(DD);
-  TypeSourceInfo *TInfo = Reader.GetTypeSourceInfo(Record, Idx);
+  TypeSourceInfo *TInfo = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
   if (TInfo)
     DD->setTypeSourceInfo(TInfo);
   // FIXME: read optional qualifier and its range.
 }
 
-void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
+void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   VisitDeclaratorDecl(FD);
+  // FIXME: read DeclarationNameLoc.
 
   FD->IdentifierNamespace = Record[Idx++];
   switch ((FunctionDecl::TemplatedKind)Record[Idx++]) {
@@ -237,7 +261,7 @@ void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     
     // Template arguments.
     llvm::SmallVector<TemplateArgument, 8> TemplArgs;
-    Reader.ReadTemplateArgumentList(TemplArgs, Record, Idx);
+    Reader.ReadTemplateArgumentList(TemplArgs, Cursor, Record, Idx);
     
     // Template args as written.
     llvm::SmallVector<TemplateArgumentLoc, 8> TemplArgLocs;
@@ -246,7 +270,8 @@ void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
       unsigned NumTemplateArgLocs = Record[Idx++];
       TemplArgLocs.reserve(NumTemplateArgLocs);
       for (unsigned i=0; i != NumTemplateArgLocs; ++i)
-        TemplArgLocs.push_back(Reader.ReadTemplateArgumentLoc(Record, Idx));
+        TemplArgLocs.push_back(
+            Reader.ReadTemplateArgumentLoc(Cursor, Record, Idx));
   
       LAngleLoc = Reader.ReadSourceLocation(Record, Idx);
       RAngleLoc = Reader.ReadSourceLocation(Record, Idx);
@@ -254,11 +279,12 @@ void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     
     SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
 
-    FD->setFunctionTemplateSpecialization(Template, TemplArgs.size(),
-                                          TemplArgs.data(), TSK,
-                                          TemplArgLocs.size(),
-                                          TemplArgLocs.data(),
-                                          LAngleLoc, RAngleLoc, POI);
+    if (FD->isCanonicalDecl()) // if canonical add to template's set.
+      FD->setFunctionTemplateSpecialization(Template, TemplArgs.size(),
+                                            TemplArgs.data(), TSK,
+                                            TemplArgLocs.size(),
+                                            TemplArgLocs.data(),
+                                            LAngleLoc, RAngleLoc, POI);
     break;
   }
   case FunctionDecl::TK_DependentFunctionTemplateSpecialization: {
@@ -272,7 +298,7 @@ void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     TemplateArgumentListInfo TemplArgs;
     unsigned NumArgs = Record[Idx++];
     while (NumArgs--)
-      TemplArgs.addArgument(Reader.ReadTemplateArgumentLoc(Record, Idx));
+      TemplArgs.addArgument(Reader.ReadTemplateArgumentLoc(Cursor,Record, Idx));
     TemplArgs.setLAngleLoc(Reader.ReadSourceLocation(Record, Idx));
     TemplArgs.setRAngleLoc(Reader.ReadSourceLocation(Record, Idx));
     
@@ -282,15 +308,12 @@ void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   }
   }
 
-  // FunctionDecl's body is handled last at PCHReaderDecl::Visit,
+  // FunctionDecl's body is handled last at ASTDeclReader::Visit,
   // after everything else is read.
 
-  // Avoid side effects and invariant checking of FunctionDecl's
-  // setPreviousDeclaration.
-  FD->redeclarable_base::setPreviousDeclaration(
-                   cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++])));
-  FD->setStorageClass((FunctionDecl::StorageClass)Record[Idx++]);
-  FD->setStorageClassAsWritten((FunctionDecl::StorageClass)Record[Idx++]);
+  VisitRedeclarable(FD);
+  FD->setStorageClass((StorageClass)Record[Idx++]);
+  FD->setStorageClassAsWritten((StorageClass)Record[Idx++]);
   FD->setInlineSpecified(Record[Idx++]);
   FD->setVirtualAsWritten(Record[Idx++]);
   FD->setPure(Record[Idx++]);
@@ -311,23 +334,24 @@ void PCHDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
   FD->setParams(Params.data(), NumParams);
 }
 
-void PCHDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
+void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
   VisitNamedDecl(MD);
   if (Record[Idx++]) {
     // In practice, this won't be executed (since method definitions
     // don't occur in header files).
-    MD->setBody(Reader.ReadStmt());
+    MD->setBody(Reader.ReadStmt(Cursor));
     MD->setSelfDecl(cast<ImplicitParamDecl>(Reader.GetDecl(Record[Idx++])));
     MD->setCmdDecl(cast<ImplicitParamDecl>(Reader.GetDecl(Record[Idx++])));
   }
   MD->setInstanceMethod(Record[Idx++]);
   MD->setVariadic(Record[Idx++]);
   MD->setSynthesized(Record[Idx++]);
+  MD->setDefined(Record[Idx++]);
   MD->setDeclImplementation((ObjCMethodDecl::ImplementationControl)Record[Idx++]);
   MD->setObjCDeclQualifier((Decl::ObjCDeclQualifier)Record[Idx++]);
   MD->setNumSelectorArgs(unsigned(Record[Idx++]));
   MD->setResultType(Reader.GetType(Record[Idx++]));
-  MD->setResultTypeSourceInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  MD->setResultTypeSourceInfo(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
   MD->setEndLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   unsigned NumParams = Record[Idx++];
   llvm::SmallVector<ParmVarDecl *, 16> Params;
@@ -338,18 +362,20 @@ void PCHDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
                       NumParams);
 }
 
-void PCHDeclReader::VisitObjCContainerDecl(ObjCContainerDecl *CD) {
+void ASTDeclReader::VisitObjCContainerDecl(ObjCContainerDecl *CD) {
   VisitNamedDecl(CD);
   SourceLocation A = SourceLocation::getFromRawEncoding(Record[Idx++]);
   SourceLocation B = SourceLocation::getFromRawEncoding(Record[Idx++]);
   CD->setAtEndRange(SourceRange(A, B));
 }
 
-void PCHDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
+void ASTDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
   VisitObjCContainerDecl(ID);
   ID->setTypeForDecl(Reader.GetType(Record[Idx++]).getTypePtr());
   ID->setSuperClass(cast_or_null<ObjCInterfaceDecl>
                        (Reader.GetDecl(Record[Idx++])));
+  
+  // Read the directly referenced protocols and their SourceLocations.
   unsigned NumProtocols = Record[Idx++];
   llvm::SmallVector<ObjCProtocolDecl *, 16> Protocols;
   Protocols.reserve(NumProtocols);
@@ -361,6 +387,17 @@ void PCHDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
     ProtoLocs.push_back(SourceLocation::getFromRawEncoding(Record[Idx++]));
   ID->setProtocolList(Protocols.data(), NumProtocols, ProtoLocs.data(),
                       *Reader.getContext());
+  
+  // Read the transitive closure of protocols referenced by this class.
+  NumProtocols = Record[Idx++];
+  Protocols.clear();
+  Protocols.reserve(NumProtocols);
+  for (unsigned I = 0; I != NumProtocols; ++I)
+    Protocols.push_back(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
+  ID->AllReferencedProtocols.set(Protocols.data(), NumProtocols,
+                                 *Reader.getContext());
+  
+  // Read the ivars.
   unsigned NumIvars = Record[Idx++];
   llvm::SmallVector<ObjCIvarDecl *, 16> IVars;
   IVars.reserve(NumIvars);
@@ -368,6 +405,8 @@ void PCHDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
     IVars.push_back(cast<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
   ID->setCategoryList(
                cast_or_null<ObjCCategoryDecl>(Reader.GetDecl(Record[Idx++])));
+  // We will rebuild this list lazily.
+  ID->setIvarList(0);
   ID->setForwardDecl(Record[Idx++]);
   ID->setImplicitInterfaceDecl(Record[Idx++]);
   ID->setClassLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -375,12 +414,16 @@ void PCHDeclReader::VisitObjCInterfaceDecl(ObjCInterfaceDecl *ID) {
   ID->setLocEnd(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHDeclReader::VisitObjCIvarDecl(ObjCIvarDecl *IVD) {
+void ASTDeclReader::VisitObjCIvarDecl(ObjCIvarDecl *IVD) {
   VisitFieldDecl(IVD);
   IVD->setAccessControl((ObjCIvarDecl::AccessControl)Record[Idx++]);
+  // This field will be built lazily.
+  IVD->setNextIvar(0);
+  bool synth = Record[Idx++];
+  IVD->setSynthesize(synth);
 }
 
-void PCHDeclReader::VisitObjCProtocolDecl(ObjCProtocolDecl *PD) {
+void ASTDeclReader::VisitObjCProtocolDecl(ObjCProtocolDecl *PD) {
   VisitObjCContainerDecl(PD);
   PD->setForwardDecl(Record[Idx++]);
   PD->setLocEnd(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -397,11 +440,11 @@ void PCHDeclReader::VisitObjCProtocolDecl(ObjCProtocolDecl *PD) {
                       *Reader.getContext());
 }
 
-void PCHDeclReader::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *FD) {
+void ASTDeclReader::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *FD) {
   VisitFieldDecl(FD);
 }
 
-void PCHDeclReader::VisitObjCClassDecl(ObjCClassDecl *CD) {
+void ASTDeclReader::VisitObjCClassDecl(ObjCClassDecl *CD) {
   VisitDecl(CD);
   unsigned NumClassRefs = Record[Idx++];
   llvm::SmallVector<ObjCInterfaceDecl *, 16> ClassRefs;
@@ -416,7 +459,7 @@ void PCHDeclReader::VisitObjCClassDecl(ObjCClassDecl *CD) {
                    NumClassRefs);
 }
 
-void PCHDeclReader::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *FPD) {
+void ASTDeclReader::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *FPD) {
   VisitDecl(FPD);
   unsigned NumProtoRefs = Record[Idx++];
   llvm::SmallVector<ObjCProtocolDecl *, 16> ProtoRefs;
@@ -431,7 +474,7 @@ void PCHDeclReader::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *FPD) {
                        *Reader.getContext());
 }
 
-void PCHDeclReader::VisitObjCCategoryDecl(ObjCCategoryDecl *CD) {
+void ASTDeclReader::VisitObjCCategoryDecl(ObjCCategoryDecl *CD) {
   VisitObjCContainerDecl(CD);
   CD->setClassInterface(cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
   unsigned NumProtoRefs = Record[Idx++];
@@ -446,19 +489,20 @@ void PCHDeclReader::VisitObjCCategoryDecl(ObjCCategoryDecl *CD) {
   CD->setProtocolList(ProtoRefs.data(), NumProtoRefs, ProtoLocs.data(),
                       *Reader.getContext());
   CD->setNextClassCategory(cast_or_null<ObjCCategoryDecl>(Reader.GetDecl(Record[Idx++])));
+  CD->setHasSynthBitfield(Record[Idx++]);
   CD->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   CD->setCategoryNameLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHDeclReader::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *CAD) {
+void ASTDeclReader::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *CAD) {
   VisitNamedDecl(CAD);
   CAD->setClassInterface(cast<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
+void ASTDeclReader::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   VisitNamedDecl(D);
   D->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  D->setType(Reader.GetTypeSourceInfo(Record, Idx));
+  D->setType(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
   // FIXME: stable encoding
   D->setPropertyAttributes(
                       (ObjCPropertyDecl::PropertyAttributeKind)Record[Idx++]);
@@ -477,40 +521,43 @@ void PCHDeclReader::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
                    cast_or_null<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitObjCImplDecl(ObjCImplDecl *D) {
+void ASTDeclReader::VisitObjCImplDecl(ObjCImplDecl *D) {
   VisitObjCContainerDecl(D);
   D->setClassInterface(
               cast_or_null<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
+void ASTDeclReader::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
   VisitObjCImplDecl(D);
   D->setIdentifier(Reader.GetIdentifierInfo(Record, Idx));
 }
 
-void PCHDeclReader::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
+void ASTDeclReader::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   VisitObjCImplDecl(D);
   D->setSuperClass(
               cast_or_null<ObjCInterfaceDecl>(Reader.GetDecl(Record[Idx++])));
-  // FIXME. Add reading of IvarInitializers and NumIvarInitializers.
+  llvm::tie(D->IvarInitializers, D->NumIvarInitializers)
+      = Reader.ReadCXXBaseOrMemberInitializers(Cursor, Record, Idx);
+  D->setHasSynthBitfield(Record[Idx++]);
 }
 
 
-void PCHDeclReader::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
+void ASTDeclReader::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
   VisitDecl(D);
   D->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   D->setPropertyDecl(
                cast_or_null<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
   D->setPropertyIvarDecl(
                    cast_or_null<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
-  // FIXME. read GetterCXXConstructor and SetterCXXAssignment
+  D->setGetterCXXConstructor(Reader.ReadExpr(Cursor));
+  D->setSetterCXXAssignment(Reader.ReadExpr(Cursor));
 }
 
-void PCHDeclReader::VisitFieldDecl(FieldDecl *FD) {
+void ASTDeclReader::VisitFieldDecl(FieldDecl *FD) {
   VisitDeclaratorDecl(FD);
   FD->setMutable(Record[Idx++]);
   if (Record[Idx++])
-    FD->setBitWidth(Reader.ReadExpr());
+    FD->setBitWidth(Reader.ReadExpr(Cursor));
   if (!FD->getDeclName()) {
     FieldDecl *Tmpl = cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++]));
     if (Tmpl)
@@ -518,19 +565,17 @@ void PCHDeclReader::VisitFieldDecl(FieldDecl *FD) {
   }
 }
 
-void PCHDeclReader::VisitVarDecl(VarDecl *VD) {
+void ASTDeclReader::VisitVarDecl(VarDecl *VD) {
   VisitDeclaratorDecl(VD);
-  VD->setStorageClass((VarDecl::StorageClass)Record[Idx++]);
-  VD->setStorageClassAsWritten((VarDecl::StorageClass)Record[Idx++]);
+  VD->setStorageClass((StorageClass)Record[Idx++]);
+  VD->setStorageClassAsWritten((StorageClass)Record[Idx++]);
   VD->setThreadSpecified(Record[Idx++]);
   VD->setCXXDirectInitializer(Record[Idx++]);
-  VD->setDeclaredInCondition(Record[Idx++]);
   VD->setExceptionVariable(Record[Idx++]);
   VD->setNRVOVariable(Record[Idx++]);
-  VD->setPreviousDeclaration(
-                         cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
+  VisitRedeclarable(VD);
   if (Record[Idx++])
-    VD->setInit(Reader.ReadExpr());
+    VD->setInit(Reader.ReadExpr(Cursor));
 
   if (Record[Idx++]) { // HasMemberSpecializationInfo.
     VarDecl *Tmpl = cast<VarDecl>(Reader.GetDecl(Record[Idx++]));
@@ -540,27 +585,27 @@ void PCHDeclReader::VisitVarDecl(VarDecl *VD) {
   }
 }
 
-void PCHDeclReader::VisitImplicitParamDecl(ImplicitParamDecl *PD) {
+void ASTDeclReader::VisitImplicitParamDecl(ImplicitParamDecl *PD) {
   VisitVarDecl(PD);
 }
 
-void PCHDeclReader::VisitParmVarDecl(ParmVarDecl *PD) {
+void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) {
   VisitVarDecl(PD);
   PD->setObjCDeclQualifier((Decl::ObjCDeclQualifier)Record[Idx++]);
   PD->setHasInheritedDefaultArg(Record[Idx++]);
   if (Record[Idx++]) // hasUninstantiatedDefaultArg.
-    PD->setUninstantiatedDefaultArg(Reader.ReadExpr());
+    PD->setUninstantiatedDefaultArg(Reader.ReadExpr(Cursor));
 }
 
-void PCHDeclReader::VisitFileScopeAsmDecl(FileScopeAsmDecl *AD) {
+void ASTDeclReader::VisitFileScopeAsmDecl(FileScopeAsmDecl *AD) {
   VisitDecl(AD);
-  AD->setAsmString(cast<StringLiteral>(Reader.ReadExpr()));
+  AD->setAsmString(cast<StringLiteral>(Reader.ReadExpr(Cursor)));
 }
 
-void PCHDeclReader::VisitBlockDecl(BlockDecl *BD) {
+void ASTDeclReader::VisitBlockDecl(BlockDecl *BD) {
   VisitDecl(BD);
-  BD->setBody(cast_or_null<CompoundStmt>(Reader.ReadStmt()));
-  BD->setSignatureAsWritten(Reader.GetTypeSourceInfo(Record, Idx));
+  BD->setBody(cast_or_null<CompoundStmt>(Reader.ReadStmt(Cursor)));
+  BD->setSignatureAsWritten(Reader.GetTypeSourceInfo(Cursor, Record, Idx));
   unsigned NumParams = Record[Idx++];
   llvm::SmallVector<ParmVarDecl *, 16> Params;
   Params.reserve(NumParams);
@@ -569,13 +614,13 @@ void PCHDeclReader::VisitBlockDecl(BlockDecl *BD) {
   BD->setParams(Params.data(), NumParams);
 }
 
-void PCHDeclReader::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
+void ASTDeclReader::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
   VisitDecl(D);
   D->setLanguage((LinkageSpecDecl::LanguageIDs)Record[Idx++]);
   D->setHasBraces(Record[Idx++]);
 }
 
-void PCHDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
+void ASTDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
   VisitNamedDecl(D);
   D->setLBracLoc(Reader.ReadSourceLocation(Record, Idx));
   D->setRBracLoc(Reader.ReadSourceLocation(Record, Idx));
@@ -588,21 +633,21 @@ void PCHDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
                     cast_or_null<NamespaceDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHDeclReader::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
+void ASTDeclReader::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
   VisitNamedDecl(D);
-
-  D->setAliasLoc(Reader.ReadSourceLocation(Record, Idx));
+  D->NamespaceLoc = Reader.ReadSourceLocation(Record, Idx);
   D->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
   D->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  D->setTargetNameLoc(Reader.ReadSourceLocation(Record, Idx));
-  D->setAliasedNamespace(cast<NamedDecl>(Reader.GetDecl(Record[Idx++])));
+  D->IdentLoc = Reader.ReadSourceLocation(Record, Idx);
+  D->Namespace = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
 }
 
-void PCHDeclReader::VisitUsingDecl(UsingDecl *D) {
+void ASTDeclReader::VisitUsingDecl(UsingDecl *D) {
   VisitNamedDecl(D);
   D->setUsingLocation(Reader.ReadSourceLocation(Record, Idx));
   D->setNestedNameRange(Reader.ReadSourceRange(Record, Idx));
   D->setTargetNestedNameDecl(Reader.ReadNestedNameSpecifier(Record, Idx));
+  // FIXME: read the DNLoc component.
 
   // FIXME: It would probably be more efficient to read these into a vector
   // and then re-cosntruct the shadow decl set over that vector since it
@@ -619,7 +664,7 @@ void PCHDeclReader::VisitUsingDecl(UsingDecl *D) {
     Reader.getContext()->setInstantiatedFromUsingDecl(D, Pattern);
 }
 
-void PCHDeclReader::VisitUsingShadowDecl(UsingShadowDecl *D) {
+void ASTDeclReader::VisitUsingShadowDecl(UsingShadowDecl *D) {
   VisitNamedDecl(D);
   D->setTargetDecl(cast<NamedDecl>(Reader.GetDecl(Record[Idx++])));
   D->setUsingDecl(cast<UsingDecl>(Reader.GetDecl(Record[Idx++])));
@@ -629,34 +674,34 @@ void PCHDeclReader::VisitUsingShadowDecl(UsingShadowDecl *D) {
     Reader.getContext()->setInstantiatedFromUsingShadowDecl(D, Pattern);
 }
 
-void PCHDeclReader::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
+void ASTDeclReader::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   VisitNamedDecl(D);
-  D->setNamespaceKeyLocation(Reader.ReadSourceLocation(Record, Idx));
-  D->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
-  D->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
-  D->setIdentLocation(Reader.ReadSourceLocation(Record, Idx));
-  D->setNominatedNamespace(cast<NamedDecl>(Reader.GetDecl(Record[Idx++])));
-  D->setCommonAncestor(cast_or_null<DeclContext>(
-                                                Reader.GetDecl(Record[Idx++])));
+  D->UsingLoc = Reader.ReadSourceLocation(Record, Idx);
+  D->NamespaceLoc = Reader.ReadSourceLocation(Record, Idx);
+  D->QualifierRange = Reader.ReadSourceRange(Record, Idx);
+  D->Qualifier = Reader.ReadNestedNameSpecifier(Record, Idx);
+  D->NominatedNamespace = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+  D->CommonAncestor = cast_or_null<DeclContext>(Reader.GetDecl(Record[Idx++]));
 }
 
-void PCHDeclReader::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
+void ASTDeclReader::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
   VisitValueDecl(D);
   D->setTargetNestedNameRange(Reader.ReadSourceRange(Record, Idx));
   D->setUsingLoc(Reader.ReadSourceLocation(Record, Idx));
   D->setTargetNestedNameSpecifier(Reader.ReadNestedNameSpecifier(Record, Idx));
+  // FIXME: read the DNLoc component.
 }
 
-void PCHDeclReader::VisitUnresolvedUsingTypenameDecl(
+void ASTDeclReader::VisitUnresolvedUsingTypenameDecl(
                                                UnresolvedUsingTypenameDecl *D) {
   VisitTypeDecl(D);
-  D->setTargetNestedNameRange(Reader.ReadSourceRange(Record, Idx));
-  D->setUsingLoc(Reader.ReadSourceLocation(Record, Idx));
-  D->setTypenameLoc(Reader.ReadSourceLocation(Record, Idx));
-  D->setTargetNestedNameSpecifier(Reader.ReadNestedNameSpecifier(Record, Idx));
+  D->TargetNestedNameRange = Reader.ReadSourceRange(Record, Idx);
+  D->UsingLocation = Reader.ReadSourceLocation(Record, Idx);
+  D->TypenameLocation = Reader.ReadSourceLocation(Record, Idx);
+  D->TargetNestedNameSpecifier = Reader.ReadNestedNameSpecifier(Record, Idx);
 }
 
-void PCHDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
+void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   ASTContext &C = *Reader.getContext();
 
   // We need to allocate the DefinitionData struct ahead of VisitRecordDecl
@@ -666,7 +711,7 @@ void PCHDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   enum DataOwnership { Data_NoDefData, Data_Owner, Data_NotOwner };
   switch ((DataOwnership)Record[Idx++]) {
   default:
-    assert(0 && "Out of sync with PCHDeclWriter or messed up reading");
+    assert(0 && "Out of sync with ASTDeclWriter or messed up reading");
   case Data_NoDefData:
     break;
   case Data_Owner:
@@ -705,17 +750,17 @@ void PCHDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
     Data.DeclaredDestructor = Record[Idx++];
 
     // setBases() is unsuitable since it may try to iterate the bases of an
-    // unitialized base.
+    // uninitialized base.
     Data.NumBases = Record[Idx++];
     Data.Bases = new(C) CXXBaseSpecifier [Data.NumBases];
     for (unsigned i = 0; i != Data.NumBases; ++i)
-      Data.Bases[i] = Reader.ReadCXXBaseSpecifier(Record, Idx);
+      Data.Bases[i] = Reader.ReadCXXBaseSpecifier(Cursor, Record, Idx);
 
     // FIXME: Make VBases lazily computed when needed to avoid storing them.
     Data.NumVBases = Record[Idx++];
     Data.VBases = new(C) CXXBaseSpecifier [Data.NumVBases];
     for (unsigned i = 0; i != Data.NumVBases; ++i)
-      Data.VBases[i] = Reader.ReadCXXBaseSpecifier(Record, Idx);
+      Data.VBases[i] = Reader.ReadCXXBaseSpecifier(Cursor, Record, Idx);
 
     Reader.ReadUnresolvedSet(Data.Conversions, Record, Idx);
     Reader.ReadUnresolvedSet(Data.VisibleConversions, Record, Idx);
@@ -729,7 +774,7 @@ void PCHDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   };
   switch ((CXXRecKind)Record[Idx++]) {
   default:
-    assert(false && "Out of sync with PCHDeclWriter::VisitCXXRecordDecl?");
+    assert(false && "Out of sync with ASTDeclWriter::VisitCXXRecordDecl?");
   case CXXRecNotTemplate:
     break;
   case CXXRecTemplate:
@@ -747,7 +792,7 @@ void PCHDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
   }
 }
 
-void PCHDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
+void ASTDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
   VisitFunctionDecl(D);
   unsigned NumOverridenMethods = Record[Idx++];
   while (NumOverridenMethods--) {
@@ -758,102 +803,57 @@ void PCHDeclReader::VisitCXXMethodDecl(CXXMethodDecl *D) {
   }
 }
 
-void PCHDeclReader::VisitCXXConstructorDecl(CXXConstructorDecl *D) {
+void ASTDeclReader::VisitCXXConstructorDecl(CXXConstructorDecl *D) {
   VisitCXXMethodDecl(D);
   
   D->IsExplicitSpecified = Record[Idx++];
   D->ImplicitlyDefined = Record[Idx++];
-  
-  unsigned NumInitializers = Record[Idx++];
-  D->NumBaseOrMemberInitializers = NumInitializers;
-  if (NumInitializers) {
-    ASTContext &C = *Reader.getContext();
-
-    D->BaseOrMemberInitializers
-        = new (C) CXXBaseOrMemberInitializer*[NumInitializers];
-    for (unsigned i=0; i != NumInitializers; ++i) {
-      TypeSourceInfo *BaseClassInfo = 0;
-      bool IsBaseVirtual = false;
-      FieldDecl *Member = 0;
-  
-      bool IsBaseInitializer = Record[Idx++];
-      if (IsBaseInitializer) {
-        BaseClassInfo = Reader.GetTypeSourceInfo(Record, Idx);
-        IsBaseVirtual = Record[Idx++];
-      } else {
-        Member = cast<FieldDecl>(Reader.GetDecl(Record[Idx++]));
-      }
-      SourceLocation MemberLoc = Reader.ReadSourceLocation(Record, Idx);
-      Expr *Init = Reader.ReadExpr();
-      FieldDecl *AnonUnionMember
-          = cast_or_null<FieldDecl>(Reader.GetDecl(Record[Idx++]));
-      SourceLocation LParenLoc = Reader.ReadSourceLocation(Record, Idx);
-      SourceLocation RParenLoc = Reader.ReadSourceLocation(Record, Idx);
-      bool IsWritten = Record[Idx++];
-      unsigned SourceOrderOrNumArrayIndices;
-      llvm::SmallVector<VarDecl *, 8> Indices;
-      if (IsWritten) {
-        SourceOrderOrNumArrayIndices = Record[Idx++];
-      } else {
-        SourceOrderOrNumArrayIndices = Record[Idx++];
-        Indices.reserve(SourceOrderOrNumArrayIndices);
-        for (unsigned i=0; i != SourceOrderOrNumArrayIndices; ++i)
-          Indices.push_back(cast<VarDecl>(Reader.GetDecl(Record[Idx++])));
-      }
-      
-      CXXBaseOrMemberInitializer *BOMInit;
-      if (IsBaseInitializer) {
-        BOMInit = new (C) CXXBaseOrMemberInitializer(C, BaseClassInfo,
-                                                     IsBaseVirtual, LParenLoc,
-                                                     Init, RParenLoc);
-      } else if (IsWritten) {
-        BOMInit = new (C) CXXBaseOrMemberInitializer(C, Member, MemberLoc,
-                                                     LParenLoc, Init, RParenLoc);
-      } else {
-        BOMInit = CXXBaseOrMemberInitializer::Create(C, Member, MemberLoc,
-                                                     LParenLoc, Init, RParenLoc,
-                                                     Indices.data(),
-                                                     Indices.size());
-      }
-
-      BOMInit->setAnonUnionMember(AnonUnionMember);
-      D->BaseOrMemberInitializers[i] = BOMInit;
-    }
-  }
+  llvm::tie(D->BaseOrMemberInitializers, D->NumBaseOrMemberInitializers)
+      = Reader.ReadCXXBaseOrMemberInitializers(Cursor, Record, Idx);
 }
 
-void PCHDeclReader::VisitCXXDestructorDecl(CXXDestructorDecl *D) {
+void ASTDeclReader::VisitCXXDestructorDecl(CXXDestructorDecl *D) {
   VisitCXXMethodDecl(D);
 
   D->ImplicitlyDefined = Record[Idx++];
   D->OperatorDelete = cast_or_null<FunctionDecl>(Reader.GetDecl(Record[Idx++]));
 }
 
-void PCHDeclReader::VisitCXXConversionDecl(CXXConversionDecl *D) {
+void ASTDeclReader::VisitCXXConversionDecl(CXXConversionDecl *D) {
   VisitCXXMethodDecl(D);
   D->IsExplicitSpecified = Record[Idx++];
 }
 
-void PCHDeclReader::VisitAccessSpecDecl(AccessSpecDecl *D) {
+void ASTDeclReader::VisitAccessSpecDecl(AccessSpecDecl *D) {
   VisitDecl(D);
   D->setColonLoc(Reader.ReadSourceLocation(Record, Idx));
 }
 
-void PCHDeclReader::VisitFriendDecl(FriendDecl *D) {
+void ASTDeclReader::VisitFriendDecl(FriendDecl *D) {
   VisitDecl(D);
   if (Record[Idx++])
-    D->Friend = Reader.GetTypeSourceInfo(Record, Idx);
+    D->Friend = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
   else
     D->Friend = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
   D->NextFriend = cast_or_null<FriendDecl>(Reader.GetDecl(Record[Idx++]));
   D->FriendLoc = Reader.ReadSourceLocation(Record, Idx);
 }
 
-void PCHDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
-  assert(false && "cannot read FriendTemplateDecl");
+void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
+  VisitDecl(D);
+  unsigned NumParams = Record[Idx++];
+  D->NumParams = NumParams;
+  D->Params = new TemplateParameterList*[NumParams];
+  for (unsigned i = 0; i != NumParams; ++i)
+    D->Params[i] = Reader.ReadTemplateParameterList(Record, Idx);
+  if (Record[Idx++]) // HasFriendDecl
+    D->Friend = cast<NamedDecl>(Reader.GetDecl(Record[Idx++]));
+  else
+    D->Friend = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
+  D->FriendLoc = Reader.ReadSourceLocation(Record, Idx);
 }
 
-void PCHDeclReader::VisitTemplateDecl(TemplateDecl *D) {
+void ASTDeclReader::VisitTemplateDecl(TemplateDecl *D) {
   VisitNamedDecl(D);
 
   NamedDecl *TemplatedDecl
@@ -863,14 +863,56 @@ void PCHDeclReader::VisitTemplateDecl(TemplateDecl *D) {
   D->init(TemplatedDecl, TemplateParams);
 }
 
-void PCHDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
+void ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
   VisitTemplateDecl(D);
 
   D->IdentifierNamespace = Record[Idx++];
-  ClassTemplateDecl *PrevDecl =
-      cast_or_null<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]));
-  D->setPreviousDeclaration(PrevDecl);
+  RedeclarableTemplateDecl *PrevDecl =
+      cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+  assert((PrevDecl == 0 || PrevDecl->getKind() == D->getKind()) &&
+         "PrevDecl kind mismatch");
+  if (PrevDecl)
+    D->CommonOrPrev = PrevDecl;
   if (PrevDecl == 0) {
+    if (RedeclarableTemplateDecl *RTD
+          = cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]))) {
+      assert(RTD->getKind() == D->getKind() &&
+             "InstantiatedFromMemberTemplate kind mismatch");
+      D->setInstantiatedFromMemberTemplateImpl(RTD);
+      if (Record[Idx++])
+        D->setMemberSpecialization();
+    }
+
+    RedeclarableTemplateDecl *LatestDecl = 
+        cast_or_null<RedeclarableTemplateDecl>(Reader.GetDecl(Record[Idx++]));
+  
+    // This decl is a first one and the latest declaration that it points to is
+    // in the same AST file. However, if this actually needs to point to a
+    // redeclaration in another AST file, we need to update it by checking
+    // the FirstLatestDeclIDs map which tracks this kind of decls.
+    assert(Reader.GetDecl(ThisDeclID) == D && "Invalid ThisDeclID ?");
+    ASTReader::FirstLatestDeclIDMap::iterator I
+        = Reader.FirstLatestDeclIDs.find(ThisDeclID);
+    if (I != Reader.FirstLatestDeclIDs.end()) {
+      Decl *NewLatest = Reader.GetDecl(I->second);
+      assert((LatestDecl->getLocation().isInvalid() ||
+              NewLatest->getLocation().isInvalid()  ||
+              Reader.SourceMgr.isBeforeInTranslationUnit(
+                                                   LatestDecl->getLocation(),
+                                                   NewLatest->getLocation())) &&
+             "The new latest is supposed to come after the previous latest");
+      LatestDecl = cast<RedeclarableTemplateDecl>(NewLatest);
+    }
+
+    assert(LatestDecl->getKind() == D->getKind() && "Latest kind mismatch");
+    D->getCommonPtr()->Latest = LatestDecl;
+  }
+}
+
+void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
+  VisitRedeclarableTemplateDecl(D);
+
+  if (D->getPreviousDeclaration() == 0) {
     // This ClassTemplateDecl owns a CommonPtr; read it.
 
     // FoldingSets are filled in VisitClassTemplateSpecializationDecl.
@@ -884,17 +926,10 @@ void PCHDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
                                                  Reader.GetDecl(Record[Idx++]));
 
     // InjectedClassNameType is computed.
-
-    if (ClassTemplateDecl *CTD
-          = cast_or_null<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]))) {
-      D->setInstantiatedFromMemberTemplate(CTD);
-      if (Record[Idx++])
-        D->setMemberSpecialization();
-    }
   }
 }
 
-void PCHDeclReader::VisitClassTemplateSpecializationDecl(
+void ASTDeclReader::VisitClassTemplateSpecializationDecl(
                                            ClassTemplateSpecializationDecl *D) {
   VisitCXXRecordDecl(D);
 
@@ -903,28 +938,28 @@ void PCHDeclReader::VisitClassTemplateSpecializationDecl(
       D->setInstantiationOf(CTD);
     } else {
       llvm::SmallVector<TemplateArgument, 8> TemplArgs;
-      Reader.ReadTemplateArgumentList(TemplArgs, Record, Idx);
+      Reader.ReadTemplateArgumentList(TemplArgs, Cursor, Record, Idx);
       D->setInstantiationOf(cast<ClassTemplatePartialSpecializationDecl>(InstD),
                             TemplArgs.data(), TemplArgs.size());
     }
   }
 
   // Explicit info.
-  if (TypeSourceInfo *TyInfo = Reader.GetTypeSourceInfo(Record, Idx)) {
+  if (TypeSourceInfo *TyInfo = Reader.GetTypeSourceInfo(Cursor, Record, Idx)) {
     D->setTypeAsWritten(TyInfo);
     D->setExternLoc(Reader.ReadSourceLocation(Record, Idx));
     D->setTemplateKeywordLoc(Reader.ReadSourceLocation(Record, Idx));
   }
 
   llvm::SmallVector<TemplateArgument, 8> TemplArgs;
-  Reader.ReadTemplateArgumentList(TemplArgs, Record, Idx);
+  Reader.ReadTemplateArgumentList(TemplArgs, Cursor, Record, Idx);
   D->initTemplateArgs(TemplArgs.data(), TemplArgs.size());
   SourceLocation POI = Reader.ReadSourceLocation(Record, Idx);
   if (POI.isValid())
     D->setPointOfInstantiation(POI);
   D->setSpecializationKind((TemplateSpecializationKind)Record[Idx++]);
 
-  if (Record[Idx++]) { // IsKeptInFoldingSet.
+  if (D->isCanonicalDecl()) { // It's kept in the folding set.
     ClassTemplateDecl *CanonPattern
                        = cast<ClassTemplateDecl>(Reader.GetDecl(Record[Idx++]));
     if (ClassTemplatePartialSpecializationDecl *Partial
@@ -936,7 +971,7 @@ void PCHDeclReader::VisitClassTemplateSpecializationDecl(
   }
 }
 
-void PCHDeclReader::VisitClassTemplatePartialSpecializationDecl(
+void ASTDeclReader::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   VisitClassTemplateSpecializationDecl(D);
 
@@ -945,7 +980,7 @@ void PCHDeclReader::VisitClassTemplatePartialSpecializationDecl(
   TemplateArgumentListInfo ArgInfos;
   unsigned NumArgs = Record[Idx++];
   while (NumArgs--)
-    ArgInfos.addArgument(Reader.ReadTemplateArgumentLoc(Record, Idx));
+    ArgInfos.addArgument(Reader.ReadTemplateArgumentLoc(Cursor, Record, Idx));
   D->initTemplateArgsAsWritten(ArgInfos);
   
   D->setSequenceNumber(Record[Idx++]);
@@ -960,14 +995,10 @@ void PCHDeclReader::VisitClassTemplatePartialSpecializationDecl(
   }
 }
 
-void PCHDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
-  VisitTemplateDecl(D);
+void ASTDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
+  VisitRedeclarableTemplateDecl(D);
 
-  D->IdentifierNamespace = Record[Idx++];
-  FunctionTemplateDecl *PrevDecl =
-      cast_or_null<FunctionTemplateDecl>(Reader.GetDecl(Record[Idx++]));
-  D->setPreviousDeclaration(PrevDecl);
-  if (PrevDecl == 0) {
+  if (D->getPreviousDeclaration() == 0) {
     // This FunctionTemplateDecl owns a CommonPtr; read it.
 
     // Read the function specialization declarations.
@@ -976,68 +1007,112 @@ void PCHDeclReader::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
     unsigned NumSpecs = Record[Idx++];
     while (NumSpecs--)
       Reader.GetDecl(Record[Idx++]);
-
-    if (FunctionTemplateDecl *CTD
-          = cast_or_null<FunctionTemplateDecl>(Reader.GetDecl(Record[Idx++]))) {
-      D->setInstantiatedFromMemberTemplate(CTD);
-      if (Record[Idx++])
-        D->setMemberSpecialization();
-    }
   }
 }
 
-void PCHDeclReader::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
+void ASTDeclReader::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
   VisitTypeDecl(D);
 
   D->setDeclaredWithTypename(Record[Idx++]);
   D->setParameterPack(Record[Idx++]);
 
   bool Inherited = Record[Idx++];
-  TypeSourceInfo *DefArg = Reader.GetTypeSourceInfo(Record, Idx);
+  TypeSourceInfo *DefArg = Reader.GetTypeSourceInfo(Cursor, Record, Idx);
   D->setDefaultArgument(DefArg, Inherited);
 }
 
-void PCHDeclReader::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
+void ASTDeclReader::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
   VisitVarDecl(D);
   // TemplateParmPosition.
   D->setDepth(Record[Idx++]);
   D->setPosition(Record[Idx++]);
   // Rest of NonTypeTemplateParmDecl.
   if (Record[Idx++]) {
-    Expr *DefArg = Reader.ReadExpr();
+    Expr *DefArg = Reader.ReadExpr(Cursor);
     bool Inherited = Record[Idx++];
     D->setDefaultArgument(DefArg, Inherited);
  }
 }
 
-void PCHDeclReader::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
+void ASTDeclReader::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   VisitTemplateDecl(D);
   // TemplateParmPosition.
   D->setDepth(Record[Idx++]);
   D->setPosition(Record[Idx++]);
   // Rest of TemplateTemplateParmDecl.
-  TemplateArgumentLoc Arg = Reader.ReadTemplateArgumentLoc(Record, Idx);
+  TemplateArgumentLoc Arg = Reader.ReadTemplateArgumentLoc(Cursor, Record, Idx);
   bool IsInherited = Record[Idx++];
   D->setDefaultArgument(Arg, IsInherited);
 }
 
-void PCHDeclReader::VisitStaticAssertDecl(StaticAssertDecl *D) {
-  assert(false && "cannot read StaticAssertDecl");
+void ASTDeclReader::VisitStaticAssertDecl(StaticAssertDecl *D) {
+  VisitDecl(D);
+  D->AssertExpr = Reader.ReadExpr(Cursor);
+  D->Message = cast<StringLiteral>(Reader.ReadExpr(Cursor));
 }
 
 std::pair<uint64_t, uint64_t>
-PCHDeclReader::VisitDeclContext(DeclContext *DC) {
+ASTDeclReader::VisitDeclContext(DeclContext *DC) {
   uint64_t LexicalOffset = Record[Idx++];
   uint64_t VisibleOffset = Record[Idx++];
   return std::make_pair(LexicalOffset, VisibleOffset);
 }
 
+template <typename T>
+void ASTDeclReader::VisitRedeclarable(Redeclarable<T> *D) {
+  enum RedeclKind { NoRedeclaration = 0, PointsToPrevious, PointsToLatest };
+  RedeclKind Kind = (RedeclKind)Record[Idx++];
+  switch (Kind) {
+  default:
+    assert(0 && "Out of sync with ASTDeclWriter::VisitRedeclarable or messed up"
+                " reading");
+  case NoRedeclaration:
+    break;
+  case PointsToPrevious:
+    D->RedeclLink = typename Redeclarable<T>::PreviousDeclLink(
+                                cast_or_null<T>(Reader.GetDecl(Record[Idx++])));
+    break;
+  case PointsToLatest:
+    D->RedeclLink = typename Redeclarable<T>::LatestDeclLink(
+                                cast_or_null<T>(Reader.GetDecl(Record[Idx++])));
+    break;
+  }
+
+  assert(!(Kind == PointsToPrevious &&
+           Reader.FirstLatestDeclIDs.find(ThisDeclID) !=
+               Reader.FirstLatestDeclIDs.end()) &&
+         "This decl is not first, it should not be in the map");
+  if (Kind == PointsToPrevious)
+    return;
+
+  // This decl is a first one and the latest declaration that it points to is in
+  // the same AST file. However, if this actually needs to point to a
+  // redeclaration in another AST file, we need to update it by checking the
+  // FirstLatestDeclIDs map which tracks this kind of decls.
+  assert(Reader.GetDecl(ThisDeclID) == static_cast<T*>(D) &&
+         "Invalid ThisDeclID ?");
+  ASTReader::FirstLatestDeclIDMap::iterator I
+      = Reader.FirstLatestDeclIDs.find(ThisDeclID);
+  if (I != Reader.FirstLatestDeclIDs.end()) {
+    Decl *NewLatest = Reader.GetDecl(I->second);
+    assert((D->getMostRecentDeclaration()->getLocation().isInvalid() ||
+            NewLatest->getLocation().isInvalid() ||
+            Reader.SourceMgr.isBeforeInTranslationUnit(
+                                   D->getMostRecentDeclaration()->getLocation(),
+                                   NewLatest->getLocation())) &&
+           "The new latest is supposed to come after the previous latest");
+    D->RedeclLink
+        = typename Redeclarable<T>::LatestDeclLink(cast_or_null<T>(NewLatest));
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Attribute Reading
 //===----------------------------------------------------------------------===//
 
 /// \brief Reads attributes from the current stream position.
-Attr *PCHReader::ReadAttributes() {
+void ASTReader::ReadAttributes(llvm::BitstreamCursor &DeclsCursor,
+                               AttrVec &Attrs) {
   unsigned Code = DeclsCursor.ReadCode();
   assert(Code == llvm::bitc::UNABBREV_RECORD &&
          "Expected unabbreviated record"); (void)Code;
@@ -1045,181 +1120,25 @@ Attr *PCHReader::ReadAttributes() {
   RecordData Record;
   unsigned Idx = 0;
   unsigned RecCode = DeclsCursor.ReadRecord(Code, Record);
-  assert(RecCode == pch::DECL_ATTR && "Expected attribute record");
+  assert(RecCode == DECL_ATTR && "Expected attribute record");
   (void)RecCode;
 
-#define SIMPLE_ATTR(Name)                       \
- case attr::Name:                               \
-   New = ::new (*Context) Name##Attr();         \
-   break
-
-#define STRING_ATTR(Name)                                       \
- case attr::Name:                                               \
-   New = ::new (*Context) Name##Attr(*Context, ReadString(Record, Idx));  \
-   break
-
-#define UNSIGNED_ATTR(Name)                             \
- case attr::Name:                                       \
-   New = ::new (*Context) Name##Attr(Record[Idx++]);    \
-   break
-
-  Attr *Attrs = 0;
   while (Idx < Record.size()) {
     Attr *New = 0;
     attr::Kind Kind = (attr::Kind)Record[Idx++];
-    bool IsInherited = Record[Idx++];
-
-    switch (Kind) {
-    default:
-      assert(0 && "Unknown attribute!");
-      break;
-    STRING_ATTR(Alias);
-    SIMPLE_ATTR(AlignMac68k);
-    UNSIGNED_ATTR(Aligned);
-    SIMPLE_ATTR(AlwaysInline);
-    SIMPLE_ATTR(AnalyzerNoReturn);
-    STRING_ATTR(Annotate);
-    STRING_ATTR(AsmLabel);
-    SIMPLE_ATTR(BaseCheck);
-
-    case attr::Blocks:
-      New = ::new (*Context) BlocksAttr(
-                                  (BlocksAttr::BlocksAttrTypes)Record[Idx++]);
-      break;
-
-    SIMPLE_ATTR(CDecl);
-
-    case attr::Cleanup:
-      New = ::new (*Context) CleanupAttr(
-                                  cast<FunctionDecl>(GetDecl(Record[Idx++])));
-      break;
-
-    SIMPLE_ATTR(Const);
-    UNSIGNED_ATTR(Constructor);
-    SIMPLE_ATTR(DLLExport);
-    SIMPLE_ATTR(DLLImport);
-    SIMPLE_ATTR(Deprecated);
-    UNSIGNED_ATTR(Destructor);
-    SIMPLE_ATTR(FastCall);
-    SIMPLE_ATTR(Final);
-
-    case attr::Format: {
-      std::string Type = ReadString(Record, Idx);
-      unsigned FormatIdx = Record[Idx++];
-      unsigned FirstArg = Record[Idx++];
-      New = ::new (*Context) FormatAttr(*Context, Type, FormatIdx, FirstArg);
-      break;
-    }
-
-    case attr::FormatArg: {
-      unsigned FormatIdx = Record[Idx++];
-      New = ::new (*Context) FormatArgAttr(FormatIdx);
-      break;
-    }
-
-    case attr::Sentinel: {
-      int sentinel = Record[Idx++];
-      int nullPos = Record[Idx++];
-      New = ::new (*Context) SentinelAttr(sentinel, nullPos);
-      break;
-    }
-
-    SIMPLE_ATTR(GNUInline);
-    SIMPLE_ATTR(Hiding);
-
-    case attr::IBAction:
-      New = ::new (*Context) IBActionAttr();
-      break;
+    SourceLocation Loc = SourceLocation::getFromRawEncoding(Record[Idx++]);
+    bool isInherited = Record[Idx++];
 
-    case attr::IBOutlet:
-      New = ::new (*Context) IBOutletAttr();
-      break;
-
-    case attr::IBOutletCollection: {
-      ObjCInterfaceDecl *D =
-        cast_or_null<ObjCInterfaceDecl>(GetDecl(Record[Idx++]));
-      New = ::new (*Context) IBOutletCollectionAttr(D);
-      break;
-    }
-
-    SIMPLE_ATTR(Malloc);
-    SIMPLE_ATTR(NoDebug);
-    SIMPLE_ATTR(NoInline);
-    SIMPLE_ATTR(NoReturn);
-    SIMPLE_ATTR(NoThrow);
-
-    case attr::NonNull: {
-      unsigned Size = Record[Idx++];
-      llvm::SmallVector<unsigned, 16> ArgNums;
-      ArgNums.insert(ArgNums.end(), &Record[Idx], &Record[Idx] + Size);
-      Idx += Size;
-      New = ::new (*Context) NonNullAttr(*Context, ArgNums.data(), Size);
-      break;
-    }
-
-    case attr::ReqdWorkGroupSize: {
-      unsigned X = Record[Idx++];
-      unsigned Y = Record[Idx++];
-      unsigned Z = Record[Idx++];
-      New = ::new (*Context) ReqdWorkGroupSizeAttr(X, Y, Z);
-      break;
-    }
-
-    SIMPLE_ATTR(ObjCException);
-    SIMPLE_ATTR(ObjCNSObject);
-    SIMPLE_ATTR(CFReturnsNotRetained);
-    SIMPLE_ATTR(CFReturnsRetained);
-    SIMPLE_ATTR(NSReturnsNotRetained);
-    SIMPLE_ATTR(NSReturnsRetained);
-    SIMPLE_ATTR(Overloadable);
-    SIMPLE_ATTR(Override);
-    SIMPLE_ATTR(Packed);
-    UNSIGNED_ATTR(MaxFieldAlignment);
-    SIMPLE_ATTR(Pure);
-    UNSIGNED_ATTR(Regparm);
-    STRING_ATTR(Section);
-    SIMPLE_ATTR(StdCall);
-    SIMPLE_ATTR(ThisCall);
-    SIMPLE_ATTR(TransparentUnion);
-    SIMPLE_ATTR(Unavailable);
-    SIMPLE_ATTR(Unused);
-    SIMPLE_ATTR(Used);
-
-    case attr::Visibility:
-      New = ::new (*Context) VisibilityAttr(
-                              (VisibilityAttr::VisibilityTypes)Record[Idx++]);
-      break;
-
-    SIMPLE_ATTR(WarnUnusedResult);
-    SIMPLE_ATTR(Weak);
-    SIMPLE_ATTR(WeakRef);
-    SIMPLE_ATTR(WeakImport);
-    }
+#include "clang/Serialization/AttrPCHRead.inc"
 
     assert(New && "Unable to decode attribute?");
-    New->setInherited(IsInherited);
-    New->setNext(Attrs);
-    Attrs = New;
-  }
-#undef UNSIGNED_ATTR
-#undef STRING_ATTR
-#undef SIMPLE_ATTR
-
-  // The list of attributes was built backwards. Reverse the list
-  // before returning it.
-  Attr *PrevAttr = 0, *NextAttr = 0;
-  while (Attrs) {
-    NextAttr = Attrs->getNext();
-    Attrs->setNext(PrevAttr);
-    PrevAttr = Attrs;
-    Attrs = NextAttr;
+    New->setInherited(isInherited);
+    Attrs.push_back(New);
   }
-
-  return PrevAttr;
 }
 
 //===----------------------------------------------------------------------===//
-// PCHReader Implementation
+// ASTReader Implementation
 //===----------------------------------------------------------------------===//
 
 /// \brief Note that we have loaded the declaration with the given
@@ -1228,7 +1147,7 @@ Attr *PCHReader::ReadAttributes() {
 /// This routine notes that this declaration has already been loaded,
 /// so that future GetDecl calls will return this declaration rather
 /// than trying to load a new declaration.
-inline void PCHReader::LoadedDecl(unsigned Index, Decl *D) {
+inline void ASTReader::LoadedDecl(unsigned Index, Decl *D) {
   assert(!DeclsLoaded[Index] && "Decl loaded twice?");
   DeclsLoaded[Index] = D;
 }
@@ -1244,14 +1163,36 @@ static bool isConsumerInterestedIn(Decl *D) {
   if (isa<FileScopeAsmDecl>(D))
     return true;
   if (VarDecl *Var = dyn_cast<VarDecl>(D))
-    return Var->isFileVarDecl() && Var->getInit();
+    return Var->isFileVarDecl() &&
+           Var->isThisDeclarationADefinition() == VarDecl::Definition;
   if (FunctionDecl *Func = dyn_cast<FunctionDecl>(D))
     return Func->isThisDeclarationADefinition();
-  return isa<ObjCProtocolDecl>(D);
+  return isa<ObjCProtocolDecl>(D) || isa<ObjCImplementationDecl>(D);
+}
+
+/// \brief Get the correct cursor and offset for loading a type.
+ASTReader::RecordLocation
+ASTReader::DeclCursorForIndex(unsigned Index, DeclID ID) {
+  // See if there's an override.
+  DeclReplacementMap::iterator It = ReplacedDecls.find(ID);
+  if (It != ReplacedDecls.end())
+    return RecordLocation(&It->second.first->DeclsCursor, It->second.second);
+
+  PerFileData *F = 0;
+  for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
+    F = Chain[N - I - 1];
+    if (Index < F->LocalNumDecls)
+      break;
+    Index -= F->LocalNumDecls;
+  }
+  assert(F && F->LocalNumDecls > Index && "Broken chain");
+  return RecordLocation(&F->DeclsCursor, F->DeclOffsets[Index]);
 }
 
-/// \brief Read the declaration at the given offset from the PCH file.
-Decl *PCHReader::ReadDeclRecord(uint64_t Offset, unsigned Index) {
+/// \brief Read the declaration at the given offset from the AST file.
+Decl *ASTReader::ReadDeclRecord(unsigned Index, DeclID ID) {
+  RecordLocation Loc = DeclCursorForIndex(Index, ID);
+  llvm::BitstreamCursor &DeclsCursor = *Loc.first;
   // Keep track of where we are in the stream, then jump back there
   // after reading this declaration.
   SavedStreamPosition SavedPosition(DeclsCursor);
@@ -1259,205 +1200,205 @@ Decl *PCHReader::ReadDeclRecord(uint64_t Offset, unsigned Index) {
   ReadingKindTracker ReadingKind(Read_Decl, *this);
 
   // Note that we are loading a declaration record.
-  LoadingTypeOrDecl Loading(*this);
+  Deserializing ADecl(this);
 
-  DeclsCursor.JumpToBit(Offset);
+  DeclsCursor.JumpToBit(Loc.second);
   RecordData Record;
   unsigned Code = DeclsCursor.ReadCode();
   unsigned Idx = 0;
-  PCHDeclReader Reader(*this, Record, Idx);
+  ASTDeclReader Reader(*this, DeclsCursor, ID, Record, Idx);
 
   Decl *D = 0;
-  switch ((pch::DeclCode)DeclsCursor.ReadRecord(Code, Record)) {
-  case pch::DECL_ATTR:
-  case pch::DECL_CONTEXT_LEXICAL:
-  case pch::DECL_CONTEXT_VISIBLE:
+  switch ((DeclCode)DeclsCursor.ReadRecord(Code, Record)) {
+  case DECL_ATTR:
+  case DECL_CONTEXT_LEXICAL:
+  case DECL_CONTEXT_VISIBLE:
     assert(false && "Record cannot be de-serialized with ReadDeclRecord");
     break;
-  case pch::DECL_TRANSLATION_UNIT:
+  case DECL_TRANSLATION_UNIT:
     assert(Index == 0 && "Translation unit must be at index 0");
     D = Context->getTranslationUnitDecl();
     break;
-  case pch::DECL_TYPEDEF:
+  case DECL_TYPEDEF:
     D = TypedefDecl::Create(*Context, 0, SourceLocation(), 0, 0);
     break;
-  case pch::DECL_ENUM:
+  case DECL_ENUM:
     D = EnumDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_RECORD:
+  case DECL_RECORD:
     D = RecordDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_ENUM_CONSTANT:
+  case DECL_ENUM_CONSTANT:
     D = EnumConstantDecl::Create(*Context, 0, SourceLocation(), 0, QualType(),
                                  0, llvm::APSInt());
     break;
-  case pch::DECL_FUNCTION:
+  case DECL_FUNCTION:
     D = FunctionDecl::Create(*Context, 0, SourceLocation(), DeclarationName(),
                              QualType(), 0);
     break;
-  case pch::DECL_LINKAGE_SPEC:
+  case DECL_LINKAGE_SPEC:
     D = LinkageSpecDecl::Create(*Context, 0, SourceLocation(),
                                 (LinkageSpecDecl::LanguageIDs)0,
                                 false);
     break;
-  case pch::DECL_NAMESPACE:
+  case DECL_NAMESPACE:
     D = NamespaceDecl::Create(*Context, 0, SourceLocation(), 0);
     break;
-  case pch::DECL_NAMESPACE_ALIAS:
+  case DECL_NAMESPACE_ALIAS:
     D = NamespaceAliasDecl::Create(*Context, 0, SourceLocation(),
                                    SourceLocation(), 0, SourceRange(), 0,
                                    SourceLocation(), 0);
     break;
-  case pch::DECL_USING:
-    D = UsingDecl::Create(*Context, 0, SourceLocation(), SourceRange(),
-                          SourceLocation(), 0, DeclarationName(), false);
+  case DECL_USING:
+    D = UsingDecl::Create(*Context, 0, SourceRange(), SourceLocation(),
+                          0, DeclarationNameInfo(), false);
     break;
-  case pch::DECL_USING_SHADOW:
+  case DECL_USING_SHADOW:
     D = UsingShadowDecl::Create(*Context, 0, SourceLocation(), 0, 0);
     break;
-  case pch::DECL_USING_DIRECTIVE:
+  case DECL_USING_DIRECTIVE:
     D = UsingDirectiveDecl::Create(*Context, 0, SourceLocation(),
                                    SourceLocation(), SourceRange(), 0,
                                    SourceLocation(), 0, 0);
     break;
-  case pch::DECL_UNRESOLVED_USING_VALUE:
+  case DECL_UNRESOLVED_USING_VALUE:
     D = UnresolvedUsingValueDecl::Create(*Context, 0, SourceLocation(),
-                                         SourceRange(), 0, SourceLocation(),
-                                         DeclarationName());
+                                         SourceRange(), 0,
+                                         DeclarationNameInfo());
     break;
-  case pch::DECL_UNRESOLVED_USING_TYPENAME:
+  case DECL_UNRESOLVED_USING_TYPENAME:
     D = UnresolvedUsingTypenameDecl::Create(*Context, 0, SourceLocation(),
                                             SourceLocation(), SourceRange(),
                                             0, SourceLocation(),
                                             DeclarationName());
     break;
-  case pch::DECL_CXX_RECORD:
+  case DECL_CXX_RECORD:
     D = CXXRecordDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_CXX_METHOD:
-    D = CXXMethodDecl::Create(*Context, 0, SourceLocation(), DeclarationName(),
+  case DECL_CXX_METHOD:
+    D = CXXMethodDecl::Create(*Context, 0, DeclarationNameInfo(),
                               QualType(), 0);
     break;
-  case pch::DECL_CXX_CONSTRUCTOR:
+  case DECL_CXX_CONSTRUCTOR:
     D = CXXConstructorDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_CXX_DESTRUCTOR:
+  case DECL_CXX_DESTRUCTOR:
     D = CXXDestructorDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_CXX_CONVERSION:
+  case DECL_CXX_CONVERSION:
     D = CXXConversionDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_ACCESS_SPEC:
+  case DECL_ACCESS_SPEC:
     D = AccessSpecDecl::Create(*Context, AS_none, 0, SourceLocation(),
                                SourceLocation());
     break;
-  case pch::DECL_FRIEND:
+  case DECL_FRIEND:
     D = FriendDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_FRIEND_TEMPLATE:
-    assert(false && "cannot read FriendTemplateDecl");
+  case DECL_FRIEND_TEMPLATE:
+    D = FriendTemplateDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_CLASS_TEMPLATE:
+  case DECL_CLASS_TEMPLATE:
     D = ClassTemplateDecl::Create(*Context, 0, SourceLocation(),
                                   DeclarationName(), 0, 0, 0);
     break;
-  case pch::DECL_CLASS_TEMPLATE_SPECIALIZATION:
+  case DECL_CLASS_TEMPLATE_SPECIALIZATION:
     D = ClassTemplateSpecializationDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION:
+  case DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION:
     D = ClassTemplatePartialSpecializationDecl::Create(*Context,
                                                             Decl::EmptyShell());
     break;
-  case pch::DECL_FUNCTION_TEMPLATE:
+  case DECL_FUNCTION_TEMPLATE:
     D = FunctionTemplateDecl::Create(*Context, 0, SourceLocation(),
                                      DeclarationName(), 0, 0);
     break;
-  case pch::DECL_TEMPLATE_TYPE_PARM:
+  case DECL_TEMPLATE_TYPE_PARM:
     D = TemplateTypeParmDecl::Create(*Context, Decl::EmptyShell());
     break;
-  case pch::DECL_NON_TYPE_TEMPLATE_PARM:
+  case DECL_NON_TYPE_TEMPLATE_PARM:
     D = NonTypeTemplateParmDecl::Create(*Context, 0, SourceLocation(), 0,0,0,
                                         QualType(),0);
     break;
-  case pch::DECL_TEMPLATE_TEMPLATE_PARM:
+  case DECL_TEMPLATE_TEMPLATE_PARM:
     D = TemplateTemplateParmDecl::Create(*Context, 0, SourceLocation(),0,0,0,0);
     break;
-  case pch::DECL_STATIC_ASSERT:
-    assert(false && "cannot read StaticAssertDecl");
+  case DECL_STATIC_ASSERT:
+    D = StaticAssertDecl::Create(*Context, 0, SourceLocation(), 0, 0);
     break;
 
-  case pch::DECL_OBJC_METHOD:
+  case DECL_OBJC_METHOD:
     D = ObjCMethodDecl::Create(*Context, SourceLocation(), SourceLocation(),
                                Selector(), QualType(), 0, 0);
     break;
-  case pch::DECL_OBJC_INTERFACE:
+  case DECL_OBJC_INTERFACE:
     D = ObjCInterfaceDecl::Create(*Context, 0, SourceLocation(), 0);
     break;
-  case pch::DECL_OBJC_IVAR:
+  case DECL_OBJC_IVAR:
     D = ObjCIvarDecl::Create(*Context, 0, SourceLocation(), 0, QualType(), 0,
                              ObjCIvarDecl::None);
     break;
-  case pch::DECL_OBJC_PROTOCOL:
+  case DECL_OBJC_PROTOCOL:
     D = ObjCProtocolDecl::Create(*Context, 0, SourceLocation(), 0);
     break;
-  case pch::DECL_OBJC_AT_DEFS_FIELD:
+  case DECL_OBJC_AT_DEFS_FIELD:
     D = ObjCAtDefsFieldDecl::Create(*Context, 0, SourceLocation(), 0,
                                     QualType(), 0);
     break;
-  case pch::DECL_OBJC_CLASS:
+  case DECL_OBJC_CLASS:
     D = ObjCClassDecl::Create(*Context, 0, SourceLocation());
     break;
-  case pch::DECL_OBJC_FORWARD_PROTOCOL:
+  case DECL_OBJC_FORWARD_PROTOCOL:
     D = ObjCForwardProtocolDecl::Create(*Context, 0, SourceLocation());
     break;
-  case pch::DECL_OBJC_CATEGORY:
+  case DECL_OBJC_CATEGORY:
     D = ObjCCategoryDecl::Create(*Context, 0, SourceLocation(), 
                                  SourceLocation(), SourceLocation(), 0);
     break;
-  case pch::DECL_OBJC_CATEGORY_IMPL:
+  case DECL_OBJC_CATEGORY_IMPL:
     D = ObjCCategoryImplDecl::Create(*Context, 0, SourceLocation(), 0, 0);
     break;
-  case pch::DECL_OBJC_IMPLEMENTATION:
+  case DECL_OBJC_IMPLEMENTATION:
     D = ObjCImplementationDecl::Create(*Context, 0, SourceLocation(), 0, 0);
     break;
-  case pch::DECL_OBJC_COMPATIBLE_ALIAS:
+  case DECL_OBJC_COMPATIBLE_ALIAS:
     D = ObjCCompatibleAliasDecl::Create(*Context, 0, SourceLocation(), 0, 0);
     break;
-  case pch::DECL_OBJC_PROPERTY:
+  case DECL_OBJC_PROPERTY:
     D = ObjCPropertyDecl::Create(*Context, 0, SourceLocation(), 0, SourceLocation(),
                                  0);
     break;
-  case pch::DECL_OBJC_PROPERTY_IMPL:
+  case DECL_OBJC_PROPERTY_IMPL:
     D = ObjCPropertyImplDecl::Create(*Context, 0, SourceLocation(),
                                      SourceLocation(), 0,
                                      ObjCPropertyImplDecl::Dynamic, 0);
     break;
-  case pch::DECL_FIELD:
+  case DECL_FIELD:
     D = FieldDecl::Create(*Context, 0, SourceLocation(), 0, QualType(), 0, 0,
                           false);
     break;
-  case pch::DECL_VAR:
+  case DECL_VAR:
     D = VarDecl::Create(*Context, 0, SourceLocation(), 0, QualType(), 0,
-                        VarDecl::None, VarDecl::None);
+                        SC_None, SC_None);
     break;
 
-  case pch::DECL_IMPLICIT_PARAM:
+  case DECL_IMPLICIT_PARAM:
     D = ImplicitParamDecl::Create(*Context, 0, SourceLocation(), 0, QualType());
     break;
 
-  case pch::DECL_PARM_VAR:
+  case DECL_PARM_VAR:
     D = ParmVarDecl::Create(*Context, 0, SourceLocation(), 0, QualType(), 0,
-                            VarDecl::None, VarDecl::None, 0);
+                            SC_None, SC_None, 0);
     break;
-  case pch::DECL_FILE_SCOPE_ASM:
+  case DECL_FILE_SCOPE_ASM:
     D = FileScopeAsmDecl::Create(*Context, 0, SourceLocation(), 0);
     break;
-  case pch::DECL_BLOCK:
+  case DECL_BLOCK:
     D = BlockDecl::Create(*Context, 0, SourceLocation());
     break;
   }
 
-  assert(D && "Unknown declaration reading PCH file");
+  assert(D && "Unknown declaration reading AST file");
   LoadedDecl(Index, D);
   Reader.Visit(D);
 
@@ -1468,7 +1409,44 @@ Decl *PCHReader::ReadDeclRecord(uint64_t Offset, unsigned Index) {
     if (Offsets.first || Offsets.second) {
       DC->setHasExternalLexicalStorage(Offsets.first != 0);
       DC->setHasExternalVisibleStorage(Offsets.second != 0);
-      DeclContextOffsets[DC] = Offsets;
+      DeclContextInfo Info;
+      if (ReadDeclContextStorage(DeclsCursor, Offsets, Info))
+        return 0;
+      DeclContextInfos &Infos = DeclContextOffsets[DC];
+      // Reading the TU will happen after reading its lexical update blocks,
+      // so we need to make sure we insert in front. For all other contexts,
+      // the vector is empty here anyway, so there's no loss in efficiency.
+      Infos.insert(Infos.begin(), Info);
+
+      // Now add the pending visible updates for this decl context, if it has
+      // any.
+      DeclContextVisibleUpdatesPending::iterator I =
+          PendingVisibleUpdates.find(ID);
+      if (I != PendingVisibleUpdates.end()) {
+        DeclContextVisibleUpdates &U = I->second;
+        Info.LexicalDecls = 0;
+        Info.NumLexicalDecls = 0;
+        for (DeclContextVisibleUpdates::iterator UI = U.begin(), UE = U.end();
+             UI != UE; ++UI) {
+          Info.NameLookupTableData = *UI;
+          Infos.push_back(Info);
+        }
+        PendingVisibleUpdates.erase(I);
+      }
+    }
+  }
+
+  // If this is a template, read additional specializations that may be in a
+  // different part of the chain.
+  if (isa<RedeclarableTemplateDecl>(D)) {
+    AdditionalTemplateSpecializationsMap::iterator F =
+        AdditionalTemplateSpecializationsPending.find(ID);
+    if (F != AdditionalTemplateSpecializationsPending.end()) {
+      for (AdditionalTemplateSpecializations::iterator I = F->second.begin(),
+                                                       E = F->second.end();
+           I != E; ++I)
+        GetDecl(*I);
+      AdditionalTemplateSpecializationsPending.erase(F);
     }
   }
   assert(Idx == Record.size());
diff --git a/lib/Frontend/PCHReaderStmt.cpp b/lib/Serialization/ASTReaderStmt.cpp
index ace62d787ed9..ee5d40a3698e 100644
--- a/lib/Frontend/PCHReaderStmt.cpp
+++ b/lib/Serialization/ASTReaderStmt.cpp
@@ -1,4 +1,4 @@
-//===--- PCHReaderStmt.cpp - Stmt/Expr Deserialization ----------*- C++ -*-===//
+//===--- ASTReaderStmt.cpp - Stmt/Expr Deserialization ----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -8,26 +8,28 @@
 //===----------------------------------------------------------------------===//
 //
 // Statement/expression deserialization.  This implements the
-// PCHReader::ReadStmt method.
+// ASTReader::ReadStmt method.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/PCHReader.h"
+#include "clang/Serialization/ASTReader.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/StmtVisitor.h"
 using namespace clang;
+using namespace clang::serialization;
 
 namespace clang {
 
-  class PCHStmtReader : public StmtVisitor<PCHStmtReader> {
-    PCHReader &Reader;
-    const PCHReader::RecordData &Record;
+  class ASTStmtReader : public StmtVisitor<ASTStmtReader> {
+    ASTReader &Reader;
+    llvm::BitstreamCursor &DeclsCursor;
+    const ASTReader::RecordData &Record;
     unsigned &Idx;
 
   public:
-    PCHStmtReader(PCHReader &Reader, const PCHReader::RecordData &Record,
-                  unsigned &Idx)
-      : Reader(Reader), Record(Record), Idx(Idx) { }
+    ASTStmtReader(ASTReader &Reader, llvm::BitstreamCursor &Cursor,
+                  const ASTReader::RecordData &Record, unsigned &Idx)
+      : Reader(Reader), DeclsCursor(Cursor), Record(Record), Idx(Idx) { }
 
     /// \brief The number of record fields required for the Stmt class
     /// itself.
@@ -116,6 +118,10 @@ namespace clang {
     void VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *);
     void VisitObjCAtThrowStmt(ObjCAtThrowStmt *);
 
+    // C++ Statements
+    void VisitCXXCatchStmt(CXXCatchStmt *S);
+    void VisitCXXTryStmt(CXXTryStmt *S);
+
     void VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E);
     void VisitCXXConstructExpr(CXXConstructExpr *E);
     void VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E);
@@ -132,7 +138,6 @@ namespace clang {
     void VisitCXXThrowExpr(CXXThrowExpr *E);
     void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E);
     void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
-    void VisitCXXBindReferenceExpr(CXXBindReferenceExpr *E);
     
     void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
     void VisitCXXNewExpr(CXXNewExpr *E);
@@ -153,27 +158,28 @@ namespace clang {
   };
 }
 
-void PCHStmtReader::
+void ASTStmtReader::
 ReadExplicitTemplateArgumentList(ExplicitTemplateArgumentList &ArgList,
                                  unsigned NumTemplateArgs) {
   TemplateArgumentListInfo ArgInfo;
   ArgInfo.setLAngleLoc(Reader.ReadSourceLocation(Record, Idx));
   ArgInfo.setRAngleLoc(Reader.ReadSourceLocation(Record, Idx));
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
-    ArgInfo.addArgument(Reader.ReadTemplateArgumentLoc(Record, Idx));
+    ArgInfo.addArgument(
+        Reader.ReadTemplateArgumentLoc(DeclsCursor, Record, Idx));
   ArgList.initializeFrom(ArgInfo);
 }
 
-void PCHStmtReader::VisitStmt(Stmt *S) {
+void ASTStmtReader::VisitStmt(Stmt *S) {
   assert(Idx == NumStmtFields && "Incorrect statement field count");
 }
 
-void PCHStmtReader::VisitNullStmt(NullStmt *S) {
+void ASTStmtReader::VisitNullStmt(NullStmt *S) {
   VisitStmt(S);
   S->setSemiLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCompoundStmt(CompoundStmt *S) {
+void ASTStmtReader::VisitCompoundStmt(CompoundStmt *S) {
   VisitStmt(S);
   llvm::SmallVector<Stmt *, 16> Stmts;
   unsigned NumStmts = Record[Idx++];
@@ -184,12 +190,12 @@ void PCHStmtReader::VisitCompoundStmt(CompoundStmt *S) {
   S->setRBracLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitSwitchCase(SwitchCase *S) {
+void ASTStmtReader::VisitSwitchCase(SwitchCase *S) {
   VisitStmt(S);
   Reader.RecordSwitchCaseID(S, Record[Idx++]);
 }
 
-void PCHStmtReader::VisitCaseStmt(CaseStmt *S) {
+void ASTStmtReader::VisitCaseStmt(CaseStmt *S) {
   VisitSwitchCase(S);
   S->setLHS(Reader.ReadSubExpr());
   S->setRHS(Reader.ReadSubExpr());
@@ -199,14 +205,14 @@ void PCHStmtReader::VisitCaseStmt(CaseStmt *S) {
   S->setColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitDefaultStmt(DefaultStmt *S) {
+void ASTStmtReader::VisitDefaultStmt(DefaultStmt *S) {
   VisitSwitchCase(S);
   S->setSubStmt(Reader.ReadSubStmt());
   S->setDefaultLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   S->setColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitLabelStmt(LabelStmt *S) {
+void ASTStmtReader::VisitLabelStmt(LabelStmt *S) {
   VisitStmt(S);
   S->setID(Reader.GetIdentifierInfo(Record, Idx));
   S->setSubStmt(Reader.ReadSubStmt());
@@ -214,7 +220,7 @@ void PCHStmtReader::VisitLabelStmt(LabelStmt *S) {
   Reader.RecordLabelStmt(S, Record[Idx++]);
 }
 
-void PCHStmtReader::VisitIfStmt(IfStmt *S) {
+void ASTStmtReader::VisitIfStmt(IfStmt *S) {
   VisitStmt(S);
   S->setConditionVariable(*Reader.getContext(),
                           cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
@@ -225,7 +231,7 @@ void PCHStmtReader::VisitIfStmt(IfStmt *S) {
   S->setElseLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitSwitchStmt(SwitchStmt *S) {
+void ASTStmtReader::VisitSwitchStmt(SwitchStmt *S) {
   VisitStmt(S);
   S->setConditionVariable(*Reader.getContext(),
                           cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
@@ -247,7 +253,7 @@ void PCHStmtReader::VisitSwitchStmt(SwitchStmt *S) {
   }
 }
 
-void PCHStmtReader::VisitWhileStmt(WhileStmt *S) {
+void ASTStmtReader::VisitWhileStmt(WhileStmt *S) {
   VisitStmt(S);
   S->setConditionVariable(*Reader.getContext(),
                           cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
@@ -256,7 +262,7 @@ void PCHStmtReader::VisitWhileStmt(WhileStmt *S) {
   S->setWhileLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitDoStmt(DoStmt *S) {
+void ASTStmtReader::VisitDoStmt(DoStmt *S) {
   VisitStmt(S);
   S->setCond(Reader.ReadSubExpr());
   S->setBody(Reader.ReadSubStmt());
@@ -265,7 +271,7 @@ void PCHStmtReader::VisitDoStmt(DoStmt *S) {
   S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitForStmt(ForStmt *S) {
+void ASTStmtReader::VisitForStmt(ForStmt *S) {
   VisitStmt(S);
   S->setInit(Reader.ReadSubStmt());
   S->setCond(Reader.ReadSubExpr());
@@ -278,38 +284,38 @@ void PCHStmtReader::VisitForStmt(ForStmt *S) {
   S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitGotoStmt(GotoStmt *S) {
+void ASTStmtReader::VisitGotoStmt(GotoStmt *S) {
   VisitStmt(S);
   Reader.SetLabelOf(S, Record[Idx++]);
   S->setGotoLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   S->setLabelLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitIndirectGotoStmt(IndirectGotoStmt *S) {
+void ASTStmtReader::VisitIndirectGotoStmt(IndirectGotoStmt *S) {
   VisitStmt(S);
   S->setGotoLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   S->setStarLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   S->setTarget(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitContinueStmt(ContinueStmt *S) {
+void ASTStmtReader::VisitContinueStmt(ContinueStmt *S) {
   VisitStmt(S);
   S->setContinueLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitBreakStmt(BreakStmt *S) {
+void ASTStmtReader::VisitBreakStmt(BreakStmt *S) {
   VisitStmt(S);
   S->setBreakLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitReturnStmt(ReturnStmt *S) {
+void ASTStmtReader::VisitReturnStmt(ReturnStmt *S) {
   VisitStmt(S);
   S->setRetValue(Reader.ReadSubExpr());
   S->setReturnLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   S->setNRVOCandidate(cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHStmtReader::VisitDeclStmt(DeclStmt *S) {
+void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
   VisitStmt(S);
   S->setStartLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   S->setEndLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -328,7 +334,7 @@ void PCHStmtReader::VisitDeclStmt(DeclStmt *S) {
   }
 }
 
-void PCHStmtReader::VisitAsmStmt(AsmStmt *S) {
+void ASTStmtReader::VisitAsmStmt(AsmStmt *S) {
   VisitStmt(S);
   unsigned NumOutputs = Record[Idx++];
   unsigned NumInputs = Record[Idx++];
@@ -362,7 +368,7 @@ void PCHStmtReader::VisitAsmStmt(AsmStmt *S) {
                                     Clobbers.data(), NumClobbers);
 }
 
-void PCHStmtReader::VisitExpr(Expr *E) {
+void ASTStmtReader::VisitExpr(Expr *E) {
   VisitStmt(E);
   E->setType(Reader.GetType(Record[Idx++]));
   E->setTypeDependent(Record[Idx++]);
@@ -370,13 +376,13 @@ void PCHStmtReader::VisitExpr(Expr *E) {
   assert(Idx == NumExprFields && "Incorrect expression field count");
 }
 
-void PCHStmtReader::VisitPredefinedExpr(PredefinedExpr *E) {
+void ASTStmtReader::VisitPredefinedExpr(PredefinedExpr *E) {
   VisitExpr(E);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setIdentType((PredefinedExpr::IdentType)Record[Idx++]);
 }
 
-void PCHStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
+void ASTStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
   VisitExpr(E);
 
   bool HasQualifier = Record[Idx++];
@@ -391,32 +397,33 @@ void PCHStmtReader::VisitDeclRefExpr(DeclRefExpr *E) {
   }
 
   if (NumTemplateArgs)
-    ReadExplicitTemplateArgumentList(*E->getExplicitTemplateArgumentList(),
+    ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
                                      NumTemplateArgs);
 
   E->setDecl(cast<ValueDecl>(Reader.GetDecl(Record[Idx++])));
+  // FIXME: read DeclarationNameLoc.
   E->setLocation(Reader.ReadSourceLocation(Record, Idx));
 }
 
-void PCHStmtReader::VisitIntegerLiteral(IntegerLiteral *E) {
+void ASTStmtReader::VisitIntegerLiteral(IntegerLiteral *E) {
   VisitExpr(E);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setValue(Reader.ReadAPInt(Record, Idx));
+  E->setValue(*Reader.getContext(), Reader.ReadAPInt(Record, Idx));
 }
 
-void PCHStmtReader::VisitFloatingLiteral(FloatingLiteral *E) {
+void ASTStmtReader::VisitFloatingLiteral(FloatingLiteral *E) {
   VisitExpr(E);
-  E->setValue(Reader.ReadAPFloat(Record, Idx));
+  E->setValue(*Reader.getContext(), Reader.ReadAPFloat(Record, Idx));
   E->setExact(Record[Idx++]);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitImaginaryLiteral(ImaginaryLiteral *E) {
+void ASTStmtReader::VisitImaginaryLiteral(ImaginaryLiteral *E) {
   VisitExpr(E);
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitStringLiteral(StringLiteral *E) {
+void ASTStmtReader::VisitStringLiteral(StringLiteral *E) {
   VisitExpr(E);
   unsigned Len = Record[Idx++];
   assert(Record[Idx] == E->getNumConcatenated() &&
@@ -434,21 +441,21 @@ void PCHStmtReader::VisitStringLiteral(StringLiteral *E) {
     E->setStrTokenLoc(I, SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCharacterLiteral(CharacterLiteral *E) {
+void ASTStmtReader::VisitCharacterLiteral(CharacterLiteral *E) {
   VisitExpr(E);
   E->setValue(Record[Idx++]);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setWide(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitParenExpr(ParenExpr *E) {
+void ASTStmtReader::VisitParenExpr(ParenExpr *E) {
   VisitExpr(E);
   E->setLParen(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParen(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitParenListExpr(ParenListExpr *E) {
+void ASTStmtReader::VisitParenListExpr(ParenListExpr *E) {
   VisitExpr(E);
   unsigned NumExprs = Record[Idx++];
   E->Exprs = new (*Reader.getContext()) Stmt*[NumExprs];
@@ -459,14 +466,14 @@ void PCHStmtReader::VisitParenListExpr(ParenListExpr *E) {
   E->RParenLoc = Reader.ReadSourceLocation(Record, Idx);
 }
 
-void PCHStmtReader::VisitUnaryOperator(UnaryOperator *E) {
+void ASTStmtReader::VisitUnaryOperator(UnaryOperator *E) {
   VisitExpr(E);
   E->setSubExpr(Reader.ReadSubExpr());
   E->setOpcode((UnaryOperator::Opcode)Record[Idx++]);
   E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
+void ASTStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
   typedef OffsetOfExpr::OffsetOfNode Node;
   VisitExpr(E);
   assert(E->getNumComponents() == Record[Idx]);
@@ -475,7 +482,7 @@ void PCHStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
   ++Idx;
   E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setTypeSourceInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  E->setTypeSourceInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
   for (unsigned I = 0, N = E->getNumComponents(); I != N; ++I) {
     Node::Kind Kind = static_cast<Node::Kind>(Record[Idx++]);
     SourceLocation Start = SourceLocation::getFromRawEncoding(Record[Idx++]);
@@ -496,38 +503,40 @@ void PCHStmtReader::VisitOffsetOfExpr(OffsetOfExpr *E) {
       E->setComponent(I, Node(Start, Reader.GetIdentifier(Record[Idx++]), End));
       break;
         
-    case Node::Base:
-      // FIXME: Implement this!
-      llvm_unreachable("PCH for offsetof(base-specifier) not implemented");
+    case Node::Base: {
+      CXXBaseSpecifier *Base = new (*Reader.getContext()) CXXBaseSpecifier();
+      *Base = Reader.ReadCXXBaseSpecifier(DeclsCursor, Record, Idx);
+      E->setComponent(I, Node(Base));
       break;
     }
+    }
   }
   
   for (unsigned I = 0, N = E->getNumExpressions(); I != N; ++I)
     E->setIndexExpr(I, Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
+void ASTStmtReader::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
   VisitExpr(E);
   E->setSizeof(Record[Idx++]);
   if (Record[Idx] == 0) {
     E->setArgument(Reader.ReadSubExpr());
     ++Idx;
   } else {
-    E->setArgument(Reader.GetTypeSourceInfo(Record, Idx));
+    E->setArgument(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
   }
   E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
+void ASTStmtReader::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
   VisitExpr(E);
   E->setLHS(Reader.ReadSubExpr());
   E->setRHS(Reader.ReadSubExpr());
   E->setRBracketLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCallExpr(CallExpr *E) {
+void ASTStmtReader::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   E->setNumArgs(*Reader.getContext(), Record[Idx++]);
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -536,34 +545,34 @@ void PCHStmtReader::VisitCallExpr(CallExpr *E) {
     E->setArg(I, Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitMemberExpr(MemberExpr *E) {
+void ASTStmtReader::VisitMemberExpr(MemberExpr *E) {
   // Don't call VisitExpr, this is fully initialized at creation.
   assert(E->getStmtClass() == Stmt::MemberExprClass &&
          "It's a subclass, we must advance Idx!");
 }
 
-void PCHStmtReader::VisitObjCIsaExpr(ObjCIsaExpr *E) {
+void ASTStmtReader::VisitObjCIsaExpr(ObjCIsaExpr *E) {
   VisitExpr(E);
   E->setBase(Reader.ReadSubExpr());
   E->setIsaMemberLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setArrow(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitCastExpr(CastExpr *E) {
+void ASTStmtReader::VisitCastExpr(CastExpr *E) {
   VisitExpr(E);
+  unsigned NumBaseSpecs = Record[Idx++];
+  assert(NumBaseSpecs == E->path_size());
   E->setSubExpr(Reader.ReadSubExpr());
   E->setCastKind((CastExpr::CastKind)Record[Idx++]);
-  CXXBaseSpecifierArray &BasePath = E->getBasePath();
-  unsigned NumBaseSpecs = Record[Idx++];
+  CastExpr::path_iterator BaseI = E->path_begin();
   while (NumBaseSpecs--) {
-    // FIXME: These gets leaked.
     CXXBaseSpecifier *BaseSpec = new (*Reader.getContext()) CXXBaseSpecifier;
-    *BaseSpec = Reader.ReadCXXBaseSpecifier(Record, Idx);
-    BasePath.push_back(BaseSpec);
+    *BaseSpec = Reader.ReadCXXBaseSpecifier(DeclsCursor, Record, Idx);
+    *BaseI++ = BaseSpec;
   }
 }
 
-void PCHStmtReader::VisitBinaryOperator(BinaryOperator *E) {
+void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
   VisitExpr(E);
   E->setLHS(Reader.ReadSubExpr());
   E->setRHS(Reader.ReadSubExpr());
@@ -571,53 +580,54 @@ void PCHStmtReader::VisitBinaryOperator(BinaryOperator *E) {
   E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
+void ASTStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
   VisitBinaryOperator(E);
   E->setComputationLHSType(Reader.GetType(Record[Idx++]));
   E->setComputationResultType(Reader.GetType(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitConditionalOperator(ConditionalOperator *E) {
+void ASTStmtReader::VisitConditionalOperator(ConditionalOperator *E) {
   VisitExpr(E);
   E->setCond(Reader.ReadSubExpr());
   E->setLHS(Reader.ReadSubExpr());
   E->setRHS(Reader.ReadSubExpr());
+  E->setSAVE(Reader.ReadSubExpr());
   E->setQuestionLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setColonLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitImplicitCastExpr(ImplicitCastExpr *E) {
+void ASTStmtReader::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   VisitCastExpr(E);
-  E->setLvalueCast(Record[Idx++]);
+  E->setValueKind(static_cast<ExprValueKind>(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitExplicitCastExpr(ExplicitCastExpr *E) {
+void ASTStmtReader::VisitExplicitCastExpr(ExplicitCastExpr *E) {
   VisitCastExpr(E);
-  E->setTypeInfoAsWritten(Reader.GetTypeSourceInfo(Record, Idx));
+  E->setTypeInfoAsWritten(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
 }
 
-void PCHStmtReader::VisitCStyleCastExpr(CStyleCastExpr *E) {
+void ASTStmtReader::VisitCStyleCastExpr(CStyleCastExpr *E) {
   VisitExplicitCastExpr(E);
   E->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
+void ASTStmtReader::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
   VisitExpr(E);
   E->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
-  E->setTypeSourceInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  E->setTypeSourceInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
   E->setInitializer(Reader.ReadSubExpr());
   E->setFileScope(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
+void ASTStmtReader::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
   VisitExpr(E);
   E->setBase(Reader.ReadSubExpr());
   E->setAccessor(Reader.GetIdentifierInfo(Record, Idx));
   E->setAccessorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitInitListExpr(InitListExpr *E) {
+void ASTStmtReader::VisitInitListExpr(InitListExpr *E) {
   VisitExpr(E);
   unsigned NumInits = Record[Idx++];
   E->reserveInits(*Reader.getContext(), NumInits);
@@ -631,7 +641,7 @@ void PCHStmtReader::VisitInitListExpr(InitListExpr *E) {
   E->sawArrayRangeDesignator(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
+void ASTStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
   typedef DesignatedInitExpr::Designator Designator;
 
   VisitExpr(E);
@@ -644,8 +654,8 @@ void PCHStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
 
   llvm::SmallVector<Designator, 4> Designators;
   while (Idx < Record.size()) {
-    switch ((pch::DesignatorTypes)Record[Idx++]) {
-    case pch::DESIG_FIELD_DECL: {
+    switch ((DesignatorTypes)Record[Idx++]) {
+    case DESIG_FIELD_DECL: {
       FieldDecl *Field = cast<FieldDecl>(Reader.GetDecl(Record[Idx++]));
       SourceLocation DotLoc
         = SourceLocation::getFromRawEncoding(Record[Idx++]);
@@ -657,7 +667,7 @@ void PCHStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
       break;
     }
 
-    case pch::DESIG_FIELD_NAME: {
+    case DESIG_FIELD_NAME: {
       const IdentifierInfo *Name = Reader.GetIdentifierInfo(Record, Idx);
       SourceLocation DotLoc
         = SourceLocation::getFromRawEncoding(Record[Idx++]);
@@ -667,7 +677,7 @@ void PCHStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
       break;
     }
 
-    case pch::DESIG_ARRAY: {
+    case DESIG_ARRAY: {
       unsigned Index = Record[Idx++];
       SourceLocation LBracketLoc
         = SourceLocation::getFromRawEncoding(Record[Idx++]);
@@ -677,7 +687,7 @@ void PCHStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
       break;
     }
 
-    case pch::DESIG_ARRAY_RANGE: {
+    case DESIG_ARRAY_RANGE: {
       unsigned Index = Record[Idx++];
       SourceLocation LBracketLoc
         = SourceLocation::getFromRawEncoding(Record[Idx++]);
@@ -695,40 +705,41 @@ void PCHStmtReader::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
                     Designators.data(), Designators.size());
 }
 
-void PCHStmtReader::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
+void ASTStmtReader::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
   VisitExpr(E);
 }
 
-void PCHStmtReader::VisitVAArgExpr(VAArgExpr *E) {
+void ASTStmtReader::VisitVAArgExpr(VAArgExpr *E) {
   VisitExpr(E);
   E->setSubExpr(Reader.ReadSubExpr());
+  E->setWrittenTypeInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
   E->setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitAddrLabelExpr(AddrLabelExpr *E) {
+void ASTStmtReader::VisitAddrLabelExpr(AddrLabelExpr *E) {
   VisitExpr(E);
   E->setAmpAmpLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setLabelLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   Reader.SetLabelOf(E, Record[Idx++]);
 }
 
-void PCHStmtReader::VisitStmtExpr(StmtExpr *E) {
+void ASTStmtReader::VisitStmtExpr(StmtExpr *E) {
   VisitExpr(E);
   E->setLParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setSubStmt(cast_or_null<CompoundStmt>(Reader.ReadSubStmt()));
 }
 
-void PCHStmtReader::VisitTypesCompatibleExpr(TypesCompatibleExpr *E) {
+void ASTStmtReader::VisitTypesCompatibleExpr(TypesCompatibleExpr *E) {
   VisitExpr(E);
-  E->setArgType1(Reader.GetType(Record[Idx++]));
-  E->setArgType2(Reader.GetType(Record[Idx++]));
+  E->setArgTInfo1(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
+  E->setArgTInfo2(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
   E->setBuiltinLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitChooseExpr(ChooseExpr *E) {
+void ASTStmtReader::VisitChooseExpr(ChooseExpr *E) {
   VisitExpr(E);
   E->setCond(Reader.ReadSubExpr());
   E->setLHS(Reader.ReadSubExpr());
@@ -737,12 +748,12 @@ void PCHStmtReader::VisitChooseExpr(ChooseExpr *E) {
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitGNUNullExpr(GNUNullExpr *E) {
+void ASTStmtReader::VisitGNUNullExpr(GNUNullExpr *E) {
   VisitExpr(E);
   E->setTokenLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
+void ASTStmtReader::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
   VisitExpr(E);
   llvm::SmallVector<Expr *, 16> Exprs;
   unsigned NumExprs = Record[Idx++];
@@ -753,13 +764,13 @@ void PCHStmtReader::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitBlockExpr(BlockExpr *E) {
+void ASTStmtReader::VisitBlockExpr(BlockExpr *E) {
   VisitExpr(E);
   E->setBlockDecl(cast_or_null<BlockDecl>(Reader.GetDecl(Record[Idx++])));
   E->setHasBlockDeclRefExprs(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
+void ASTStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
   VisitExpr(E);
   E->setDecl(cast<ValueDecl>(Reader.GetDecl(Record[Idx++])));
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -771,34 +782,34 @@ void PCHStmtReader::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
 //===----------------------------------------------------------------------===//
 // Objective-C Expressions and Statements
 
-void PCHStmtReader::VisitObjCStringLiteral(ObjCStringLiteral *E) {
+void ASTStmtReader::VisitObjCStringLiteral(ObjCStringLiteral *E) {
   VisitExpr(E);
   E->setString(cast<StringLiteral>(Reader.ReadSubStmt()));
   E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCEncodeExpr(ObjCEncodeExpr *E) {
+void ASTStmtReader::VisitObjCEncodeExpr(ObjCEncodeExpr *E) {
   VisitExpr(E);
-  E->setEncodedTypeSourceInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  E->setEncodedTypeSourceInfo(Reader.GetTypeSourceInfo(DeclsCursor,Record,Idx));
   E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
+void ASTStmtReader::VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
   VisitExpr(E);
   E->setSelector(Reader.GetSelector(Record, Idx));
   E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
+void ASTStmtReader::VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
   VisitExpr(E);
   E->setProtocol(cast<ObjCProtocolDecl>(Reader.GetDecl(Record[Idx++])));
   E->setAtLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
+void ASTStmtReader::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   VisitExpr(E);
   E->setDecl(cast<ObjCIvarDecl>(Reader.GetDecl(Record[Idx++])));
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -807,14 +818,14 @@ void PCHStmtReader::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   E->setIsFreeIvar(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
+void ASTStmtReader::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   VisitExpr(E);
   E->setProperty(cast<ObjCPropertyDecl>(Reader.GetDecl(Record[Idx++])));
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setBase(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitObjCImplicitSetterGetterRefExpr(
+void ASTStmtReader::VisitObjCImplicitSetterGetterRefExpr(
                                       ObjCImplicitSetterGetterRefExpr *E) {
   VisitExpr(E);
   E->setGetterMethod(
@@ -828,7 +839,7 @@ void PCHStmtReader::VisitObjCImplicitSetterGetterRefExpr(
   E->setClassLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
+void ASTStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   VisitExpr(E);
   assert(Record[Idx] == E->getNumArgs());
   ++Idx;
@@ -840,7 +851,7 @@ void PCHStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
     break;
 
   case ObjCMessageExpr::Class:
-    E->setClassReceiver(Reader.GetTypeSourceInfo(Record, Idx));
+    E->setClassReceiver(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
     break;
 
   case ObjCMessageExpr::SuperClass:
@@ -866,12 +877,12 @@ void PCHStmtReader::VisitObjCMessageExpr(ObjCMessageExpr *E) {
     E->setArg(I, Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitObjCSuperExpr(ObjCSuperExpr *E) {
+void ASTStmtReader::VisitObjCSuperExpr(ObjCSuperExpr *E) {
   VisitExpr(E);
   E->setLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
+void ASTStmtReader::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
   VisitStmt(S);
   S->setElement(Reader.ReadSubStmt());
   S->setCollection(Reader.ReadSubExpr());
@@ -880,7 +891,7 @@ void PCHStmtReader::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
   S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
+void ASTStmtReader::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   VisitStmt(S);
   S->setCatchBody(Reader.ReadSubStmt());
   S->setCatchParamDecl(cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++])));
@@ -888,13 +899,13 @@ void PCHStmtReader::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   S->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
+void ASTStmtReader::VisitObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
   VisitStmt(S);
   S->setFinallyBody(Reader.ReadSubStmt());
   S->setAtFinallyLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
+void ASTStmtReader::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
   VisitStmt(S);
   assert(Record[Idx] == S->getNumCatchStmts());
   ++Idx;
@@ -908,14 +919,14 @@ void PCHStmtReader::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
   S->setAtTryLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
+void ASTStmtReader::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   VisitStmt(S);
   S->setSynchExpr(Reader.ReadSubStmt());
   S->setSynchBody(Reader.ReadSubStmt());
   S->setAtSynchronizedLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
+void ASTStmtReader::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
   VisitStmt(S);
   S->setThrowExpr(Reader.ReadSubStmt());
   S->setThrowLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
@@ -923,13 +934,31 @@ void PCHStmtReader::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
 
 //===----------------------------------------------------------------------===//
 // C++ Expressions and Statements
+//===----------------------------------------------------------------------===//
+
+void ASTStmtReader::VisitCXXCatchStmt(CXXCatchStmt *S) {
+  VisitStmt(S);
+  S->CatchLoc = Reader.ReadSourceLocation(Record, Idx);
+  S->ExceptionDecl = cast_or_null<VarDecl>(Reader.GetDecl(Record[Idx++]));
+  S->HandlerBlock = Reader.ReadSubStmt();
+}
+
+void ASTStmtReader::VisitCXXTryStmt(CXXTryStmt *S) {
+  VisitStmt(S);
+  assert(Record[Idx] == S->getNumHandlers() && "NumStmtFields is wrong ?");
+  ++Idx;
+  S->TryLoc = Reader.ReadSourceLocation(Record, Idx);
+  S->getStmts()[0] = Reader.ReadSubStmt();
+  for (unsigned i = 0, e = S->getNumHandlers(); i != e; ++i)
+    S->getStmts()[i + 1] = Reader.ReadSubStmt();
+}
 
-void PCHStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
+void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   E->setOperator((OverloadedOperatorKind)Record[Idx++]);
 }
 
-void PCHStmtReader::VisitCXXConstructExpr(CXXConstructExpr *E) {
+void ASTStmtReader::VisitCXXConstructExpr(CXXConstructExpr *E) {
   VisitExpr(E);
   E->NumArgs = Record[Idx++];
   if (E->NumArgs)
@@ -943,55 +972,56 @@ void PCHStmtReader::VisitCXXConstructExpr(CXXConstructExpr *E) {
   E->setConstructionKind((CXXConstructExpr::ConstructionKind)Record[Idx++]);
 }
 
-void PCHStmtReader::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E) {
+void ASTStmtReader::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E) {
   VisitCXXConstructExpr(E);
   E->TyBeginLoc = Reader.ReadSourceLocation(Record, Idx);
   E->RParenLoc = Reader.ReadSourceLocation(Record, Idx);
 }
 
-void PCHStmtReader::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
+void ASTStmtReader::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
   VisitExplicitCastExpr(E);
   E->setOperatorLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) {
+void ASTStmtReader::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) {
   return VisitCXXNamedCastExpr(E);
 }
 
-void PCHStmtReader::VisitCXXDynamicCastExpr(CXXDynamicCastExpr *E) {
+void ASTStmtReader::VisitCXXDynamicCastExpr(CXXDynamicCastExpr *E) {
   return VisitCXXNamedCastExpr(E);
 }
 
-void PCHStmtReader::VisitCXXReinterpretCastExpr(CXXReinterpretCastExpr *E) {
+void ASTStmtReader::VisitCXXReinterpretCastExpr(CXXReinterpretCastExpr *E) {
   return VisitCXXNamedCastExpr(E);
 }
 
-void PCHStmtReader::VisitCXXConstCastExpr(CXXConstCastExpr *E) {
+void ASTStmtReader::VisitCXXConstCastExpr(CXXConstCastExpr *E) {
   return VisitCXXNamedCastExpr(E);
 }
 
-void PCHStmtReader::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E) {
+void ASTStmtReader::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E) {
   VisitExplicitCastExpr(E);
   E->setTypeBeginLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
+void ASTStmtReader::VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
   VisitExpr(E);
   E->setValue(Record[Idx++]);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E) {
+void ASTStmtReader::VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E) {
   VisitExpr(E);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
+void ASTStmtReader::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
   VisitExpr(E);
   E->setSourceRange(Reader.ReadSourceRange(Record, Idx));
   if (E->isTypeOperand()) { // typeid(int)
-    E->setTypeOperandSourceInfo(Reader.GetTypeSourceInfo(Record, Idx));
+    E->setTypeOperandSourceInfo(
+        Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
     return;
   }
   
@@ -999,19 +1029,19 @@ void PCHStmtReader::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
   E->setExprOperand(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitCXXThisExpr(CXXThisExpr *E) {
+void ASTStmtReader::VisitCXXThisExpr(CXXThisExpr *E) {
   VisitExpr(E);
   E->setLocation(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setImplicit(Record[Idx++]);
 }
 
-void PCHStmtReader::VisitCXXThrowExpr(CXXThrowExpr *E) {
+void ASTStmtReader::VisitCXXThrowExpr(CXXThrowExpr *E) {
   VisitExpr(E);
   E->setThrowLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
+void ASTStmtReader::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   VisitExpr(E);
 
   assert(Record[Idx] == E->Param.getInt() && "We messed up at creation ?");
@@ -1020,26 +1050,19 @@ void PCHStmtReader::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   E->Loc = Reader.ReadSourceLocation(Record, Idx);
 }
 
-void PCHStmtReader::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
+void ASTStmtReader::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
   VisitExpr(E);
   E->setTemporary(Reader.ReadCXXTemporary(Record, Idx));
   E->setSubExpr(Reader.ReadSubExpr());
 }
 
-void PCHStmtReader::VisitCXXBindReferenceExpr(CXXBindReferenceExpr *E) {
-  VisitExpr(E);
-  E->SubExpr = Reader.ReadSubExpr();
-  E->ExtendsLifetime = Record[Idx++];
-  E->RequiresTemporaryCopy = Record[Idx++];
-}
-
-void PCHStmtReader::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
+void ASTStmtReader::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
   VisitExpr(E);
   E->setTypeBeginLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
   E->setRParenLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
+void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
   VisitExpr(E);
   E->setGlobalNew(Record[Idx++]);
   E->setHasInitializer(Record[Idx++]);
@@ -1067,7 +1090,7 @@ void PCHStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
     *I = Reader.ReadSubStmt();
 }
 
-void PCHStmtReader::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+void ASTStmtReader::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   VisitExpr(E);
   E->setGlobalDelete(Record[Idx++]);
   E->setArrayForm(Record[Idx++]);
@@ -1077,7 +1100,7 @@ void PCHStmtReader::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   E->setStartLoc(SourceLocation::getFromRawEncoding(Record[Idx++]));
 }
 
-void PCHStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
+void ASTStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   VisitExpr(E);
 
   E->setBase(Reader.ReadSubExpr());
@@ -1085,7 +1108,7 @@ void PCHStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   E->setOperatorLoc(Reader.ReadSourceLocation(Record, Idx));
   E->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
   E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
-  E->setScopeTypeInfo(Reader.GetTypeSourceInfo(Record, Idx));
+  E->setScopeTypeInfo(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
   E->setColonColonLoc(Reader.ReadSourceLocation(Record, Idx));
   E->setTildeLoc(Reader.ReadSourceLocation(Record, Idx));
   
@@ -1093,10 +1116,10 @@ void PCHStmtReader::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   if (II)
     E->setDestroyedType(II, Reader.ReadSourceLocation(Record, Idx));
   else
-    E->setDestroyedType(Reader.GetTypeSourceInfo(Record, Idx));
+    E->setDestroyedType(Reader.GetTypeSourceInfo(DeclsCursor, Record, Idx));
 }
 
-void PCHStmtReader::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
+void ASTStmtReader::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
   VisitExpr(E);
   unsigned NumTemps = Record[Idx++];
   if (NumTemps) {
@@ -1108,14 +1131,14 @@ void PCHStmtReader::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
 }
 
 void
-PCHStmtReader::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
+ASTStmtReader::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   VisitExpr(E);
   
   unsigned NumTemplateArgs = Record[Idx++];
   assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
          "Read wrong record during creation ?");
   if (E->hasExplicitTemplateArgs())
-    ReadExplicitTemplateArgumentList(*E->getExplicitTemplateArgumentList(),
+    ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
                                      NumTemplateArgs);
 
   E->setBase(Reader.ReadSubExpr());
@@ -1126,12 +1149,13 @@ PCHStmtReader::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
   E->setFirstQualifierFoundInScope(
                         cast_or_null<NamedDecl>(Reader.GetDecl(Record[Idx++])));
+  // FIXME: read whole DeclarationNameInfo.
   E->setMember(Reader.ReadDeclarationName(Record, Idx));
   E->setMemberLoc(Reader.ReadSourceLocation(Record, Idx));
 }
 
 void
-PCHStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
+ASTStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
   VisitExpr(E);
   
   unsigned NumTemplateArgs = Record[Idx++];
@@ -1141,6 +1165,7 @@ PCHStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
     ReadExplicitTemplateArgumentList(E->getExplicitTemplateArgs(),
                                      NumTemplateArgs);
 
+  // FIXME: read whole DeclarationNameInfo.
   E->setDeclName(Reader.ReadDeclarationName(Record, Idx));
   E->setLocation(Reader.ReadSourceLocation(Record, Idx));
   E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
@@ -1148,7 +1173,7 @@ PCHStmtReader::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
 }
 
 void
-PCHStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
+ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
   VisitExpr(E);
   assert(Record[Idx] == E->arg_size() && "Read wrong record during creation ?");
   ++Idx; // NumArgs;
@@ -1160,7 +1185,7 @@ PCHStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
   E->setRParenLoc(Reader.ReadSourceLocation(Record, Idx));
 }
 
-void PCHStmtReader::VisitOverloadExpr(OverloadExpr *E) {
+void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) {
   VisitExpr(E);
   
   unsigned NumTemplateArgs = Record[Idx++];
@@ -1179,13 +1204,14 @@ void PCHStmtReader::VisitOverloadExpr(OverloadExpr *E) {
   }
   E->initializeResults(*Reader.getContext(), Decls.begin(), Decls.end());
 
+  // FIXME: read whole DeclarationNameInfo.
   E->setName(Reader.ReadDeclarationName(Record, Idx));
   E->setQualifier(Reader.ReadNestedNameSpecifier(Record, Idx));
   E->setQualifierRange(Reader.ReadSourceRange(Record, Idx));
   E->setNameLoc(Reader.ReadSourceLocation(Record, Idx));
 }
 
-void PCHStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
+void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
   VisitOverloadExpr(E);
   E->setArrow(Record[Idx++]);
   E->setHasUnresolvedUsing(Record[Idx++]);
@@ -1194,14 +1220,14 @@ void PCHStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
   E->setOperatorLoc(Reader.ReadSourceLocation(Record, Idx));
 }
 
-void PCHStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
+void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
   VisitOverloadExpr(E);
   E->setRequiresADL(Record[Idx++]);
   E->setOverloaded(Record[Idx++]);
   E->setNamingClass(cast_or_null<CXXRecordDecl>(Reader.GetDecl(Record[Idx++])));
 }
 
-void PCHStmtReader::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
+void ASTStmtReader::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
   VisitExpr(E);
   E->UTT = (UnaryTypeTrait)Record[Idx++];
   SourceRange Range = Reader.ReadSourceRange(Record, Idx);
@@ -1210,12 +1236,11 @@ void PCHStmtReader::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
   E->QueriedType = Reader.GetType(Record[Idx++]);
 }
 
-Stmt *PCHReader::ReadStmt() {
+Stmt *ASTReader::ReadStmt(llvm::BitstreamCursor &Cursor) {
   switch (ReadingKind) {
   case Read_Decl:
   case Read_Type:
-    // Read a statement from the current DeclCursor.
-    return ReadStmtFromStream(DeclsCursor);
+    return ReadStmtFromStream(Cursor);
   case Read_Stmt:
     return ReadSubStmt();
   }
@@ -1224,11 +1249,11 @@ Stmt *PCHReader::ReadStmt() {
   return 0;
 }
 
-Expr *PCHReader::ReadExpr() {
-  return cast_or_null<Expr>(ReadStmt());
+Expr *ASTReader::ReadExpr(llvm::BitstreamCursor &Cursor) {
+  return cast_or_null<Expr>(ReadStmt(Cursor));
 }
 
-Expr *PCHReader::ReadSubExpr() {
+Expr *ASTReader::ReadSubExpr() {
   return cast_or_null<Expr>(ReadSubStmt());
 }
 
@@ -1239,7 +1264,7 @@ Expr *PCHReader::ReadSubExpr() {
 // the stack, with expressions having operands removing those operands from the
 // stack. Evaluation terminates when we see a STMT_STOP record, and
 // the single remaining expression on the stack is our result.
-Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
+Stmt *ASTReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
 
   ReadingKindTracker ReadingKind(Read_Stmt, *this);
   
@@ -1249,14 +1274,14 @@ Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
 
   RecordData Record;
   unsigned Idx;
-  PCHStmtReader Reader(*this, Record, Idx);
+  ASTStmtReader Reader(*this, Cursor, Record, Idx);
   Stmt::EmptyShell Empty;
 
   while (true) {
     unsigned Code = Cursor.ReadCode();
     if (Code == llvm::bitc::END_BLOCK) {
       if (Cursor.ReadBlockEnd()) {
-        Error("error at end of block in PCH file");
+        Error("error at end of block in AST file");
         return 0;
       }
       break;
@@ -1266,7 +1291,7 @@ Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       // No known subblocks, always skip them.
       Cursor.ReadSubBlockID();
       if (Cursor.SkipBlock()) {
-        Error("malformed block record in PCH file");
+        Error("malformed block record in AST file");
         return 0;
       }
       continue;
@@ -1281,145 +1306,145 @@ Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
     Idx = 0;
     Record.clear();
     bool Finished = false;
-    switch ((pch::StmtCode)Cursor.ReadRecord(Code, Record)) {
-    case pch::STMT_STOP:
+    switch ((StmtCode)Cursor.ReadRecord(Code, Record)) {
+    case STMT_STOP:
       Finished = true;
       break;
 
-    case pch::STMT_NULL_PTR:
+    case STMT_NULL_PTR:
       S = 0;
       break;
 
-    case pch::STMT_NULL:
+    case STMT_NULL:
       S = new (Context) NullStmt(Empty);
       break;
 
-    case pch::STMT_COMPOUND:
+    case STMT_COMPOUND:
       S = new (Context) CompoundStmt(Empty);
       break;
 
-    case pch::STMT_CASE:
+    case STMT_CASE:
       S = new (Context) CaseStmt(Empty);
       break;
 
-    case pch::STMT_DEFAULT:
+    case STMT_DEFAULT:
       S = new (Context) DefaultStmt(Empty);
       break;
 
-    case pch::STMT_LABEL:
+    case STMT_LABEL:
       S = new (Context) LabelStmt(Empty);
       break;
 
-    case pch::STMT_IF:
+    case STMT_IF:
       S = new (Context) IfStmt(Empty);
       break;
 
-    case pch::STMT_SWITCH:
+    case STMT_SWITCH:
       S = new (Context) SwitchStmt(Empty);
       break;
 
-    case pch::STMT_WHILE:
+    case STMT_WHILE:
       S = new (Context) WhileStmt(Empty);
       break;
 
-    case pch::STMT_DO:
+    case STMT_DO:
       S = new (Context) DoStmt(Empty);
       break;
 
-    case pch::STMT_FOR:
+    case STMT_FOR:
       S = new (Context) ForStmt(Empty);
       break;
 
-    case pch::STMT_GOTO:
+    case STMT_GOTO:
       S = new (Context) GotoStmt(Empty);
       break;
 
-    case pch::STMT_INDIRECT_GOTO:
+    case STMT_INDIRECT_GOTO:
       S = new (Context) IndirectGotoStmt(Empty);
       break;
 
-    case pch::STMT_CONTINUE:
+    case STMT_CONTINUE:
       S = new (Context) ContinueStmt(Empty);
       break;
 
-    case pch::STMT_BREAK:
+    case STMT_BREAK:
       S = new (Context) BreakStmt(Empty);
       break;
 
-    case pch::STMT_RETURN:
+    case STMT_RETURN:
       S = new (Context) ReturnStmt(Empty);
       break;
 
-    case pch::STMT_DECL:
+    case STMT_DECL:
       S = new (Context) DeclStmt(Empty);
       break;
 
-    case pch::STMT_ASM:
+    case STMT_ASM:
       S = new (Context) AsmStmt(Empty);
       break;
 
-    case pch::EXPR_PREDEFINED:
+    case EXPR_PREDEFINED:
       S = new (Context) PredefinedExpr(Empty);
       break;
 
-    case pch::EXPR_DECL_REF:
+    case EXPR_DECL_REF:
       S = DeclRefExpr::CreateEmpty(*Context,
-                         /*HasQualifier=*/Record[PCHStmtReader::NumExprFields],
-                  /*NumTemplateArgs=*/Record[PCHStmtReader::NumExprFields + 1]);
+                         /*HasQualifier=*/Record[ASTStmtReader::NumExprFields],
+                  /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields + 1]);
       break;
 
-    case pch::EXPR_INTEGER_LITERAL:
-      S = new (Context) IntegerLiteral(Empty);
+    case EXPR_INTEGER_LITERAL:
+      S = IntegerLiteral::Create(*Context, Empty);
       break;
 
-    case pch::EXPR_FLOATING_LITERAL:
-      S = new (Context) FloatingLiteral(Empty);
+    case EXPR_FLOATING_LITERAL:
+      S = FloatingLiteral::Create(*Context, Empty);
       break;
 
-    case pch::EXPR_IMAGINARY_LITERAL:
+    case EXPR_IMAGINARY_LITERAL:
       S = new (Context) ImaginaryLiteral(Empty);
       break;
 
-    case pch::EXPR_STRING_LITERAL:
+    case EXPR_STRING_LITERAL:
       S = StringLiteral::CreateEmpty(*Context,
-                                     Record[PCHStmtReader::NumExprFields + 1]);
+                                     Record[ASTStmtReader::NumExprFields + 1]);
       break;
 
-    case pch::EXPR_CHARACTER_LITERAL:
+    case EXPR_CHARACTER_LITERAL:
       S = new (Context) CharacterLiteral(Empty);
       break;
 
-    case pch::EXPR_PAREN:
+    case EXPR_PAREN:
       S = new (Context) ParenExpr(Empty);
       break;
 
-    case pch::EXPR_PAREN_LIST:
+    case EXPR_PAREN_LIST:
       S = new (Context) ParenListExpr(Empty);
       break;
 
-    case pch::EXPR_UNARY_OPERATOR:
+    case EXPR_UNARY_OPERATOR:
       S = new (Context) UnaryOperator(Empty);
       break;
 
-    case pch::EXPR_OFFSETOF:
+    case EXPR_OFFSETOF:
       S = OffsetOfExpr::CreateEmpty(*Context, 
-                                    Record[PCHStmtReader::NumExprFields],
-                                    Record[PCHStmtReader::NumExprFields + 1]);
+                                    Record[ASTStmtReader::NumExprFields],
+                                    Record[ASTStmtReader::NumExprFields + 1]);
       break;
         
-    case pch::EXPR_SIZEOF_ALIGN_OF:
+    case EXPR_SIZEOF_ALIGN_OF:
       S = new (Context) SizeOfAlignOfExpr(Empty);
       break;
 
-    case pch::EXPR_ARRAY_SUBSCRIPT:
+    case EXPR_ARRAY_SUBSCRIPT:
       S = new (Context) ArraySubscriptExpr(Empty);
       break;
 
-    case pch::EXPR_CALL:
+    case EXPR_CALL:
       S = new (Context) CallExpr(*Context, Stmt::CallExprClass, Empty);
       break;
 
-    case pch::EXPR_MEMBER: {
+    case EXPR_MEMBER: {
       // We load everything here and fully initialize it at creation.
       // That way we can use MemberExpr::Create and don't have to duplicate its
       // logic with a MemberExpr::CreateEmpty.
@@ -1438,7 +1463,7 @@ Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
         ArgInfo.setLAngleLoc(ReadSourceLocation(Record, Idx));
         ArgInfo.setRAngleLoc(ReadSourceLocation(Record, Idx));
         for (unsigned i = 0; i != NumTemplateArgs; ++i)
-          ArgInfo.addArgument(ReadTemplateArgumentLoc(Record, Idx));
+          ArgInfo.addArgument(ReadTemplateArgumentLoc(Cursor, Record, Idx));
       }
       
       NamedDecl *FoundD = cast_or_null<NamedDecl>(GetDecl(Record[Idx++]));
@@ -1448,202 +1473,219 @@ Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
       QualType T = GetType(Record[Idx++]);
       Expr *Base = ReadSubExpr();
       ValueDecl *MemberD = cast<ValueDecl>(GetDecl(Record[Idx++]));
+      // FIXME: read DeclarationNameLoc.
       SourceLocation MemberLoc = ReadSourceLocation(Record, Idx);
+      DeclarationNameInfo MemberNameInfo(MemberD->getDeclName(), MemberLoc);
       bool IsArrow = Record[Idx++];
 
       S = MemberExpr::Create(*Context, Base, IsArrow, NNS, QualifierRange,
-                             MemberD, FoundDecl, MemberLoc,
+                             MemberD, FoundDecl, MemberNameInfo,
                              NumTemplateArgs ? &ArgInfo : 0, T);
       break;
     }
 
-    case pch::EXPR_BINARY_OPERATOR:
+    case EXPR_BINARY_OPERATOR:
       S = new (Context) BinaryOperator(Empty);
       break;
 
-    case pch::EXPR_COMPOUND_ASSIGN_OPERATOR:
+    case EXPR_COMPOUND_ASSIGN_OPERATOR:
       S = new (Context) CompoundAssignOperator(Empty);
       break;
 
-    case pch::EXPR_CONDITIONAL_OPERATOR:
+    case EXPR_CONDITIONAL_OPERATOR:
       S = new (Context) ConditionalOperator(Empty);
       break;
 
-    case pch::EXPR_IMPLICIT_CAST:
-      S = new (Context) ImplicitCastExpr(Empty);
+    case EXPR_IMPLICIT_CAST:
+      S = ImplicitCastExpr::CreateEmpty(*Context,
+                       /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
-    case pch::EXPR_CSTYLE_CAST:
-      S = new (Context) CStyleCastExpr(Empty);
+    case EXPR_CSTYLE_CAST:
+      S = CStyleCastExpr::CreateEmpty(*Context,
+                       /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
-    case pch::EXPR_COMPOUND_LITERAL:
+    case EXPR_COMPOUND_LITERAL:
       S = new (Context) CompoundLiteralExpr(Empty);
       break;
 
-    case pch::EXPR_EXT_VECTOR_ELEMENT:
+    case EXPR_EXT_VECTOR_ELEMENT:
       S = new (Context) ExtVectorElementExpr(Empty);
       break;
 
-    case pch::EXPR_INIT_LIST:
+    case EXPR_INIT_LIST:
       S = new (Context) InitListExpr(*getContext(), Empty);
       break;
 
-    case pch::EXPR_DESIGNATED_INIT:
+    case EXPR_DESIGNATED_INIT:
       S = DesignatedInitExpr::CreateEmpty(*Context,
-                                     Record[PCHStmtReader::NumExprFields] - 1);
+                                     Record[ASTStmtReader::NumExprFields] - 1);
 
       break;
 
-    case pch::EXPR_IMPLICIT_VALUE_INIT:
+    case EXPR_IMPLICIT_VALUE_INIT:
       S = new (Context) ImplicitValueInitExpr(Empty);
       break;
 
-    case pch::EXPR_VA_ARG:
+    case EXPR_VA_ARG:
       S = new (Context) VAArgExpr(Empty);
       break;
 
-    case pch::EXPR_ADDR_LABEL:
+    case EXPR_ADDR_LABEL:
       S = new (Context) AddrLabelExpr(Empty);
       break;
 
-    case pch::EXPR_STMT:
+    case EXPR_STMT:
       S = new (Context) StmtExpr(Empty);
       break;
 
-    case pch::EXPR_TYPES_COMPATIBLE:
+    case EXPR_TYPES_COMPATIBLE:
       S = new (Context) TypesCompatibleExpr(Empty);
       break;
 
-    case pch::EXPR_CHOOSE:
+    case EXPR_CHOOSE:
       S = new (Context) ChooseExpr(Empty);
       break;
 
-    case pch::EXPR_GNU_NULL:
+    case EXPR_GNU_NULL:
       S = new (Context) GNUNullExpr(Empty);
       break;
 
-    case pch::EXPR_SHUFFLE_VECTOR:
+    case EXPR_SHUFFLE_VECTOR:
       S = new (Context) ShuffleVectorExpr(Empty);
       break;
 
-    case pch::EXPR_BLOCK:
+    case EXPR_BLOCK:
       S = new (Context) BlockExpr(Empty);
       break;
 
-    case pch::EXPR_BLOCK_DECL_REF:
+    case EXPR_BLOCK_DECL_REF:
       S = new (Context) BlockDeclRefExpr(Empty);
       break;
 
-    case pch::EXPR_OBJC_STRING_LITERAL:
+    case EXPR_OBJC_STRING_LITERAL:
       S = new (Context) ObjCStringLiteral(Empty);
       break;
-    case pch::EXPR_OBJC_ENCODE:
+    case EXPR_OBJC_ENCODE:
       S = new (Context) ObjCEncodeExpr(Empty);
       break;
-    case pch::EXPR_OBJC_SELECTOR_EXPR:
+    case EXPR_OBJC_SELECTOR_EXPR:
       S = new (Context) ObjCSelectorExpr(Empty);
       break;
-    case pch::EXPR_OBJC_PROTOCOL_EXPR:
+    case EXPR_OBJC_PROTOCOL_EXPR:
       S = new (Context) ObjCProtocolExpr(Empty);
       break;
-    case pch::EXPR_OBJC_IVAR_REF_EXPR:
+    case EXPR_OBJC_IVAR_REF_EXPR:
       S = new (Context) ObjCIvarRefExpr(Empty);
       break;
-    case pch::EXPR_OBJC_PROPERTY_REF_EXPR:
+    case EXPR_OBJC_PROPERTY_REF_EXPR:
       S = new (Context) ObjCPropertyRefExpr(Empty);
       break;
-    case pch::EXPR_OBJC_KVC_REF_EXPR:
+    case EXPR_OBJC_KVC_REF_EXPR:
       S = new (Context) ObjCImplicitSetterGetterRefExpr(Empty);
       break;
-    case pch::EXPR_OBJC_MESSAGE_EXPR:
+    case EXPR_OBJC_MESSAGE_EXPR:
       S = ObjCMessageExpr::CreateEmpty(*Context,
-                                     Record[PCHStmtReader::NumExprFields]);
+                                     Record[ASTStmtReader::NumExprFields]);
       break;
-    case pch::EXPR_OBJC_SUPER_EXPR:
+    case EXPR_OBJC_SUPER_EXPR:
       S = new (Context) ObjCSuperExpr(Empty);
       break;
-    case pch::EXPR_OBJC_ISA:
+    case EXPR_OBJC_ISA:
       S = new (Context) ObjCIsaExpr(Empty);
       break;
-    case pch::STMT_OBJC_FOR_COLLECTION:
+    case STMT_OBJC_FOR_COLLECTION:
       S = new (Context) ObjCForCollectionStmt(Empty);
       break;
-    case pch::STMT_OBJC_CATCH:
+    case STMT_OBJC_CATCH:
       S = new (Context) ObjCAtCatchStmt(Empty);
       break;
-    case pch::STMT_OBJC_FINALLY:
+    case STMT_OBJC_FINALLY:
       S = new (Context) ObjCAtFinallyStmt(Empty);
       break;
-    case pch::STMT_OBJC_AT_TRY:
+    case STMT_OBJC_AT_TRY:
       S = ObjCAtTryStmt::CreateEmpty(*Context, 
-                                     Record[PCHStmtReader::NumStmtFields],
-                                     Record[PCHStmtReader::NumStmtFields + 1]);
+                                     Record[ASTStmtReader::NumStmtFields],
+                                     Record[ASTStmtReader::NumStmtFields + 1]);
       break;
-    case pch::STMT_OBJC_AT_SYNCHRONIZED:
+    case STMT_OBJC_AT_SYNCHRONIZED:
       S = new (Context) ObjCAtSynchronizedStmt(Empty);
       break;
-    case pch::STMT_OBJC_AT_THROW:
+    case STMT_OBJC_AT_THROW:
       S = new (Context) ObjCAtThrowStmt(Empty);
       break;
 
-    case pch::EXPR_CXX_OPERATOR_CALL:
+    case STMT_CXX_CATCH:
+      S = new (Context) CXXCatchStmt(Empty);
+      break;
+
+    case STMT_CXX_TRY:
+      S = CXXTryStmt::Create(*Context, Empty,
+             /*NumHandlers=*/Record[ASTStmtReader::NumStmtFields]);
+      break;
+
+    case EXPR_CXX_OPERATOR_CALL:
       S = new (Context) CXXOperatorCallExpr(*Context, Empty);
       break;
 
-    case pch::EXPR_CXX_MEMBER_CALL:
+    case EXPR_CXX_MEMBER_CALL:
       S = new (Context) CXXMemberCallExpr(*Context, Empty);
       break;
         
-    case pch::EXPR_CXX_CONSTRUCT:
+    case EXPR_CXX_CONSTRUCT:
       S = new (Context) CXXConstructExpr(Empty);
       break;
       
-    case pch::EXPR_CXX_TEMPORARY_OBJECT:
+    case EXPR_CXX_TEMPORARY_OBJECT:
       S = new (Context) CXXTemporaryObjectExpr(Empty);
       break;
 
-    case pch::EXPR_CXX_STATIC_CAST:
-      S = new (Context) CXXStaticCastExpr(Empty);
+    case EXPR_CXX_STATIC_CAST:
+      S = CXXStaticCastExpr::CreateEmpty(*Context,
+                       /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
-    case pch::EXPR_CXX_DYNAMIC_CAST:
-      S = new (Context) CXXDynamicCastExpr(Empty);
+    case EXPR_CXX_DYNAMIC_CAST:
+      S = CXXDynamicCastExpr::CreateEmpty(*Context,
+                       /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
-    case pch::EXPR_CXX_REINTERPRET_CAST:
-      S = new (Context) CXXReinterpretCastExpr(Empty);
+    case EXPR_CXX_REINTERPRET_CAST:
+      S = CXXReinterpretCastExpr::CreateEmpty(*Context,
+                       /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
-    case pch::EXPR_CXX_CONST_CAST:
-      S = new (Context) CXXConstCastExpr(Empty);
+    case EXPR_CXX_CONST_CAST:
+      S = CXXConstCastExpr::CreateEmpty(*Context);
       break;
 
-    case pch::EXPR_CXX_FUNCTIONAL_CAST:
-      S = new (Context) CXXFunctionalCastExpr(Empty);
+    case EXPR_CXX_FUNCTIONAL_CAST:
+      S = CXXFunctionalCastExpr::CreateEmpty(*Context,
+                       /*PathSize*/ Record[ASTStmtReader::NumExprFields]);
       break;
 
-    case pch::EXPR_CXX_BOOL_LITERAL:
+    case EXPR_CXX_BOOL_LITERAL:
       S = new (Context) CXXBoolLiteralExpr(Empty);
       break;
 
-    case pch::EXPR_CXX_NULL_PTR_LITERAL:
+    case EXPR_CXX_NULL_PTR_LITERAL:
       S = new (Context) CXXNullPtrLiteralExpr(Empty);
       break;
-    case pch::EXPR_CXX_TYPEID_EXPR:
+    case EXPR_CXX_TYPEID_EXPR:
       S = new (Context) CXXTypeidExpr(Empty, true);
       break;
-    case pch::EXPR_CXX_TYPEID_TYPE:
+    case EXPR_CXX_TYPEID_TYPE:
       S = new (Context) CXXTypeidExpr(Empty, false);
       break;
-    case pch::EXPR_CXX_THIS:
+    case EXPR_CXX_THIS:
       S = new (Context) CXXThisExpr(Empty);
       break;
-    case pch::EXPR_CXX_THROW:
+    case EXPR_CXX_THROW:
       S = new (Context) CXXThrowExpr(Empty);
       break;
-    case pch::EXPR_CXX_DEFAULT_ARG: {
-      bool HasOtherExprStored = Record[PCHStmtReader::NumExprFields];
+    case EXPR_CXX_DEFAULT_ARG: {
+      bool HasOtherExprStored = Record[ASTStmtReader::NumExprFields];
       if (HasOtherExprStored) {
         Expr *SubExpr = ReadSubExpr();
         S = CXXDefaultArgExpr::Create(*Context, SourceLocation(), 0, SubExpr);
@@ -1651,56 +1693,53 @@ Stmt *PCHReader::ReadStmtFromStream(llvm::BitstreamCursor &Cursor) {
         S = new (Context) CXXDefaultArgExpr(Empty);
       break;
     }
-    case pch::EXPR_CXX_BIND_TEMPORARY:
+    case EXPR_CXX_BIND_TEMPORARY:
       S = new (Context) CXXBindTemporaryExpr(Empty);
       break;
-    case pch::EXPR_CXX_BIND_REFERENCE:
-      S = new (Context) CXXBindReferenceExpr(Empty);
-      break;
-
-    case pch::EXPR_CXX_SCALAR_VALUE_INIT:
+        
+    case EXPR_CXX_SCALAR_VALUE_INIT:
       S = new (Context) CXXScalarValueInitExpr(Empty);
       break;
-    case pch::EXPR_CXX_NEW:
+    case EXPR_CXX_NEW:
       S = new (Context) CXXNewExpr(Empty);
       break;
-    case pch::EXPR_CXX_DELETE:
+    case EXPR_CXX_DELETE:
       S = new (Context) CXXDeleteExpr(Empty);
       break;
-    case pch::EXPR_CXX_PSEUDO_DESTRUCTOR:
+    case EXPR_CXX_PSEUDO_DESTRUCTOR:
       S = new (Context) CXXPseudoDestructorExpr(Empty);
       break;
         
-    case pch::EXPR_CXX_EXPR_WITH_TEMPORARIES:
+    case EXPR_CXX_EXPR_WITH_TEMPORARIES:
       S = new (Context) CXXExprWithTemporaries(Empty);
       break;
       
-    case pch::EXPR_CXX_DEPENDENT_SCOPE_MEMBER:
+    case EXPR_CXX_DEPENDENT_SCOPE_MEMBER:
       S = CXXDependentScopeMemberExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[PCHStmtReader::NumExprFields]);
+                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
       break;
       
-    case pch::EXPR_CXX_DEPENDENT_SCOPE_DECL_REF:
+    case EXPR_CXX_DEPENDENT_SCOPE_DECL_REF:
       S = DependentScopeDeclRefExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[PCHStmtReader::NumExprFields]);
+                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
       break;
       
-    case pch::EXPR_CXX_UNRESOLVED_CONSTRUCT:
+    case EXPR_CXX_UNRESOLVED_CONSTRUCT:
       S = CXXUnresolvedConstructExpr::CreateEmpty(*Context,
-                              /*NumArgs=*/Record[PCHStmtReader::NumExprFields]);
+                              /*NumArgs=*/Record[ASTStmtReader::NumExprFields]);
       break;
       
-    case pch::EXPR_CXX_UNRESOLVED_MEMBER:
+    case EXPR_CXX_UNRESOLVED_MEMBER:
       S = UnresolvedMemberExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[PCHStmtReader::NumExprFields]);
+                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
       break;
       
-    case pch::EXPR_CXX_UNRESOLVED_LOOKUP:
+    case EXPR_CXX_UNRESOLVED_LOOKUP:
       S = UnresolvedLookupExpr::CreateEmpty(*Context,
-                      /*NumTemplateArgs=*/Record[PCHStmtReader::NumExprFields]);
+                      /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields]);
       break;
       
-    case pch::EXPR_CXX_UNARY_TYPE_TRAIT:
+    case EXPR_CXX_UNARY_TYPE_TRAIT:
       S = new (Context) UnaryTypeTraitExpr(Empty);
       break;
     }
diff --git a/lib/Frontend/PCHWriter.cpp b/lib/Serialization/ASTWriter.cpp
index 093c1e334a3d..3b6b218e744c 100644
--- a/lib/Frontend/PCHWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -1,4 +1,4 @@
-//===--- PCHWriter.cpp - Precompiled Headers Writer -----------------------===//
+//===--- ASTWriter.cpp - AST File Writer ----------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,20 +7,23 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines the PCHWriter class, which writes a precompiled header.
+//  This file defines the ASTWriter class, which writes AST files.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/PCHWriter.h"
-#include "../Sema/Sema.h" // FIXME: move header into include/clang/Sema
-#include "../Sema/IdentifierResolver.h" // FIXME: move header
+#include "clang/Serialization/ASTWriter.h"
+#include "ASTCommon.h"
+#include "clang/Sema/Sema.h"
+#include "clang/Sema/IdentifierResolver.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclContextInternals.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLocVisitor.h"
-#include "clang/Frontend/PCHReader.h"
+#include "clang/Serialization/ASTReader.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/Preprocessor.h"
@@ -39,22 +42,32 @@
 #include "llvm/System/Path.h"
 #include <cstdio>
 using namespace clang;
+using namespace clang::serialization;
+
+template <typename T, typename Allocator>
+T *data(std::vector<T, Allocator> &v) {
+  return v.empty() ? 0 : &v.front();
+}
+template <typename T, typename Allocator>
+const T *data(const std::vector<T, Allocator> &v) {
+  return v.empty() ? 0 : &v.front();
+}
 
 //===----------------------------------------------------------------------===//
 // Type serialization
 //===----------------------------------------------------------------------===//
 
 namespace {
-  class PCHTypeWriter {
-    PCHWriter &Writer;
-    PCHWriter::RecordData &Record;
+  class ASTTypeWriter {
+    ASTWriter &Writer;
+    ASTWriter::RecordData &Record;
 
   public:
     /// \brief Type code that corresponds to the record generated.
-    pch::TypeCode Code;
+    TypeCode Code;
 
-    PCHTypeWriter(PCHWriter &Writer, PCHWriter::RecordData &Record)
-      : Writer(Writer), Record(Record), Code(pch::TYPE_EXT_QUAL) { }
+    ASTTypeWriter(ASTWriter &Writer, ASTWriter::RecordData &Record)
+      : Writer(Writer), Record(Record), Code(TYPE_EXT_QUAL) { }
 
     void VisitArrayType(const ArrayType *T);
     void VisitFunctionType(const FunctionType *T);
@@ -66,79 +79,79 @@ namespace {
   };
 }
 
-void PCHTypeWriter::VisitBuiltinType(const BuiltinType *T) {
+void ASTTypeWriter::VisitBuiltinType(const BuiltinType *T) {
   assert(false && "Built-in types are never serialized");
 }
 
-void PCHTypeWriter::VisitComplexType(const ComplexType *T) {
+void ASTTypeWriter::VisitComplexType(const ComplexType *T) {
   Writer.AddTypeRef(T->getElementType(), Record);
-  Code = pch::TYPE_COMPLEX;
+  Code = TYPE_COMPLEX;
 }
 
-void PCHTypeWriter::VisitPointerType(const PointerType *T) {
+void ASTTypeWriter::VisitPointerType(const PointerType *T) {
   Writer.AddTypeRef(T->getPointeeType(), Record);
-  Code = pch::TYPE_POINTER;
+  Code = TYPE_POINTER;
 }
 
-void PCHTypeWriter::VisitBlockPointerType(const BlockPointerType *T) {
+void ASTTypeWriter::VisitBlockPointerType(const BlockPointerType *T) {
   Writer.AddTypeRef(T->getPointeeType(), Record);
-  Code = pch::TYPE_BLOCK_POINTER;
+  Code = TYPE_BLOCK_POINTER;
 }
 
-void PCHTypeWriter::VisitLValueReferenceType(const LValueReferenceType *T) {
+void ASTTypeWriter::VisitLValueReferenceType(const LValueReferenceType *T) {
   Writer.AddTypeRef(T->getPointeeType(), Record);
-  Code = pch::TYPE_LVALUE_REFERENCE;
+  Code = TYPE_LVALUE_REFERENCE;
 }
 
-void PCHTypeWriter::VisitRValueReferenceType(const RValueReferenceType *T) {
+void ASTTypeWriter::VisitRValueReferenceType(const RValueReferenceType *T) {
   Writer.AddTypeRef(T->getPointeeType(), Record);
-  Code = pch::TYPE_RVALUE_REFERENCE;
+  Code = TYPE_RVALUE_REFERENCE;
 }
 
-void PCHTypeWriter::VisitMemberPointerType(const MemberPointerType *T) {
+void ASTTypeWriter::VisitMemberPointerType(const MemberPointerType *T) {
   Writer.AddTypeRef(T->getPointeeType(), Record);
   Writer.AddTypeRef(QualType(T->getClass(), 0), Record);
-  Code = pch::TYPE_MEMBER_POINTER;
+  Code = TYPE_MEMBER_POINTER;
 }
 
-void PCHTypeWriter::VisitArrayType(const ArrayType *T) {
+void ASTTypeWriter::VisitArrayType(const ArrayType *T) {
   Writer.AddTypeRef(T->getElementType(), Record);
   Record.push_back(T->getSizeModifier()); // FIXME: stable values
   Record.push_back(T->getIndexTypeCVRQualifiers()); // FIXME: stable values
 }
 
-void PCHTypeWriter::VisitConstantArrayType(const ConstantArrayType *T) {
+void ASTTypeWriter::VisitConstantArrayType(const ConstantArrayType *T) {
   VisitArrayType(T);
   Writer.AddAPInt(T->getSize(), Record);
-  Code = pch::TYPE_CONSTANT_ARRAY;
+  Code = TYPE_CONSTANT_ARRAY;
 }
 
-void PCHTypeWriter::VisitIncompleteArrayType(const IncompleteArrayType *T) {
+void ASTTypeWriter::VisitIncompleteArrayType(const IncompleteArrayType *T) {
   VisitArrayType(T);
-  Code = pch::TYPE_INCOMPLETE_ARRAY;
+  Code = TYPE_INCOMPLETE_ARRAY;
 }
 
-void PCHTypeWriter::VisitVariableArrayType(const VariableArrayType *T) {
+void ASTTypeWriter::VisitVariableArrayType(const VariableArrayType *T) {
   VisitArrayType(T);
   Writer.AddSourceLocation(T->getLBracketLoc(), Record);
   Writer.AddSourceLocation(T->getRBracketLoc(), Record);
   Writer.AddStmt(T->getSizeExpr());
-  Code = pch::TYPE_VARIABLE_ARRAY;
+  Code = TYPE_VARIABLE_ARRAY;
 }
 
-void PCHTypeWriter::VisitVectorType(const VectorType *T) {
+void ASTTypeWriter::VisitVectorType(const VectorType *T) {
   Writer.AddTypeRef(T->getElementType(), Record);
   Record.push_back(T->getNumElements());
   Record.push_back(T->getAltiVecSpecific());
-  Code = pch::TYPE_VECTOR;
+  Code = TYPE_VECTOR;
 }
 
-void PCHTypeWriter::VisitExtVectorType(const ExtVectorType *T) {
+void ASTTypeWriter::VisitExtVectorType(const ExtVectorType *T) {
   VisitVectorType(T);
-  Code = pch::TYPE_EXT_VECTOR;
+  Code = TYPE_EXT_VECTOR;
 }
 
-void PCHTypeWriter::VisitFunctionType(const FunctionType *T) {
+void ASTTypeWriter::VisitFunctionType(const FunctionType *T) {
   Writer.AddTypeRef(T->getResultType(), Record);
   FunctionType::ExtInfo C = T->getExtInfo();
   Record.push_back(C.getNoReturn());
@@ -147,12 +160,12 @@ void PCHTypeWriter::VisitFunctionType(const FunctionType *T) {
   Record.push_back(C.getCC());
 }
 
-void PCHTypeWriter::VisitFunctionNoProtoType(const FunctionNoProtoType *T) {
+void ASTTypeWriter::VisitFunctionNoProtoType(const FunctionNoProtoType *T) {
   VisitFunctionType(T);
-  Code = pch::TYPE_FUNCTION_NO_PROTO;
+  Code = TYPE_FUNCTION_NO_PROTO;
 }
 
-void PCHTypeWriter::VisitFunctionProtoType(const FunctionProtoType *T) {
+void ASTTypeWriter::VisitFunctionProtoType(const FunctionProtoType *T) {
   VisitFunctionType(T);
   Record.push_back(T->getNumArgs());
   for (unsigned I = 0, N = T->getNumArgs(); I != N; ++I)
@@ -164,63 +177,63 @@ void PCHTypeWriter::VisitFunctionProtoType(const FunctionProtoType *T) {
   Record.push_back(T->getNumExceptions());
   for (unsigned I = 0, N = T->getNumExceptions(); I != N; ++I)
     Writer.AddTypeRef(T->getExceptionType(I), Record);
-  Code = pch::TYPE_FUNCTION_PROTO;
+  Code = TYPE_FUNCTION_PROTO;
 }
 
-void PCHTypeWriter::VisitUnresolvedUsingType(const UnresolvedUsingType *T) {
+void ASTTypeWriter::VisitUnresolvedUsingType(const UnresolvedUsingType *T) {
   Writer.AddDeclRef(T->getDecl(), Record);
-  Code = pch::TYPE_UNRESOLVED_USING;
+  Code = TYPE_UNRESOLVED_USING;
 }
 
-void PCHTypeWriter::VisitTypedefType(const TypedefType *T) {
+void ASTTypeWriter::VisitTypedefType(const TypedefType *T) {
   Writer.AddDeclRef(T->getDecl(), Record);
   assert(!T->isCanonicalUnqualified() && "Invalid typedef ?");
   Writer.AddTypeRef(T->getCanonicalTypeInternal(), Record);
-  Code = pch::TYPE_TYPEDEF;
+  Code = TYPE_TYPEDEF;
 }
 
-void PCHTypeWriter::VisitTypeOfExprType(const TypeOfExprType *T) {
+void ASTTypeWriter::VisitTypeOfExprType(const TypeOfExprType *T) {
   Writer.AddStmt(T->getUnderlyingExpr());
-  Code = pch::TYPE_TYPEOF_EXPR;
+  Code = TYPE_TYPEOF_EXPR;
 }
 
-void PCHTypeWriter::VisitTypeOfType(const TypeOfType *T) {
+void ASTTypeWriter::VisitTypeOfType(const TypeOfType *T) {
   Writer.AddTypeRef(T->getUnderlyingType(), Record);
-  Code = pch::TYPE_TYPEOF;
+  Code = TYPE_TYPEOF;
 }
 
-void PCHTypeWriter::VisitDecltypeType(const DecltypeType *T) {
+void ASTTypeWriter::VisitDecltypeType(const DecltypeType *T) {
   Writer.AddStmt(T->getUnderlyingExpr());
-  Code = pch::TYPE_DECLTYPE;
+  Code = TYPE_DECLTYPE;
 }
 
-void PCHTypeWriter::VisitTagType(const TagType *T) {
+void ASTTypeWriter::VisitTagType(const TagType *T) {
   Record.push_back(T->isDependentType());
   Writer.AddDeclRef(T->getDecl(), Record);
   assert(!T->isBeingDefined() &&
          "Cannot serialize in the middle of a type definition");
 }
 
-void PCHTypeWriter::VisitRecordType(const RecordType *T) {
+void ASTTypeWriter::VisitRecordType(const RecordType *T) {
   VisitTagType(T);
-  Code = pch::TYPE_RECORD;
+  Code = TYPE_RECORD;
 }
 
-void PCHTypeWriter::VisitEnumType(const EnumType *T) {
+void ASTTypeWriter::VisitEnumType(const EnumType *T) {
   VisitTagType(T);
-  Code = pch::TYPE_ENUM;
+  Code = TYPE_ENUM;
 }
 
 void
-PCHTypeWriter::VisitSubstTemplateTypeParmType(
+ASTTypeWriter::VisitSubstTemplateTypeParmType(
                                         const SubstTemplateTypeParmType *T) {
   Writer.AddTypeRef(QualType(T->getReplacedParameter(), 0), Record);
   Writer.AddTypeRef(T->getReplacementType(), Record);
-  Code = pch::TYPE_SUBST_TEMPLATE_TYPE_PARM;
+  Code = TYPE_SUBST_TEMPLATE_TYPE_PARM;
 }
 
 void
-PCHTypeWriter::VisitTemplateSpecializationType(
+ASTTypeWriter::VisitTemplateSpecializationType(
                                        const TemplateSpecializationType *T) {
   Record.push_back(T->isDependentType());
   Writer.AddTemplateName(T->getTemplateName(), Record);
@@ -231,46 +244,46 @@ PCHTypeWriter::VisitTemplateSpecializationType(
   Writer.AddTypeRef(T->isCanonicalUnqualified() ? QualType()
                                                 : T->getCanonicalTypeInternal(),
                     Record);
-  Code = pch::TYPE_TEMPLATE_SPECIALIZATION;
+  Code = TYPE_TEMPLATE_SPECIALIZATION;
 }
 
 void
-PCHTypeWriter::VisitDependentSizedArrayType(const DependentSizedArrayType *T) {
+ASTTypeWriter::VisitDependentSizedArrayType(const DependentSizedArrayType *T) {
   VisitArrayType(T);
   Writer.AddStmt(T->getSizeExpr());
   Writer.AddSourceRange(T->getBracketsRange(), Record);
-  Code = pch::TYPE_DEPENDENT_SIZED_ARRAY;
+  Code = TYPE_DEPENDENT_SIZED_ARRAY;
 }
 
 void
-PCHTypeWriter::VisitDependentSizedExtVectorType(
+ASTTypeWriter::VisitDependentSizedExtVectorType(
                                         const DependentSizedExtVectorType *T) {
   // FIXME: Serialize this type (C++ only)
   assert(false && "Cannot serialize dependent sized extended vector types");
 }
 
 void
-PCHTypeWriter::VisitTemplateTypeParmType(const TemplateTypeParmType *T) {
+ASTTypeWriter::VisitTemplateTypeParmType(const TemplateTypeParmType *T) {
   Record.push_back(T->getDepth());
   Record.push_back(T->getIndex());
   Record.push_back(T->isParameterPack());
   Writer.AddIdentifierRef(T->getName(), Record);
-  Code = pch::TYPE_TEMPLATE_TYPE_PARM;
+  Code = TYPE_TEMPLATE_TYPE_PARM;
 }
 
 void
-PCHTypeWriter::VisitDependentNameType(const DependentNameType *T) {
+ASTTypeWriter::VisitDependentNameType(const DependentNameType *T) {
   Record.push_back(T->getKeyword());
   Writer.AddNestedNameSpecifier(T->getQualifier(), Record);
   Writer.AddIdentifierRef(T->getIdentifier(), Record);
   Writer.AddTypeRef(T->isCanonicalUnqualified() ? QualType()
                                                 : T->getCanonicalTypeInternal(),
                     Record);
-  Code = pch::TYPE_DEPENDENT_NAME;
+  Code = TYPE_DEPENDENT_NAME;
 }
 
 void
-PCHTypeWriter::VisitDependentTemplateSpecializationType(
+ASTTypeWriter::VisitDependentTemplateSpecializationType(
                                 const DependentTemplateSpecializationType *T) {
   Record.push_back(T->getKeyword());
   Writer.AddNestedNameSpecifier(T->getQualifier(), Record);
@@ -279,50 +292,50 @@ PCHTypeWriter::VisitDependentTemplateSpecializationType(
   for (DependentTemplateSpecializationType::iterator
          I = T->begin(), E = T->end(); I != E; ++I)
     Writer.AddTemplateArgument(*I, Record);
-  Code = pch::TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION;
+  Code = TYPE_DEPENDENT_TEMPLATE_SPECIALIZATION;
 }
 
-void PCHTypeWriter::VisitElaboratedType(const ElaboratedType *T) {
+void ASTTypeWriter::VisitElaboratedType(const ElaboratedType *T) {
   Record.push_back(T->getKeyword());
   Writer.AddNestedNameSpecifier(T->getQualifier(), Record);
   Writer.AddTypeRef(T->getNamedType(), Record);
-  Code = pch::TYPE_ELABORATED;
+  Code = TYPE_ELABORATED;
 }
 
-void PCHTypeWriter::VisitInjectedClassNameType(const InjectedClassNameType *T) {
+void ASTTypeWriter::VisitInjectedClassNameType(const InjectedClassNameType *T) {
   Writer.AddDeclRef(T->getDecl(), Record);
   Writer.AddTypeRef(T->getInjectedSpecializationType(), Record);
-  Code = pch::TYPE_INJECTED_CLASS_NAME;
+  Code = TYPE_INJECTED_CLASS_NAME;
 }
 
-void PCHTypeWriter::VisitObjCInterfaceType(const ObjCInterfaceType *T) {
+void ASTTypeWriter::VisitObjCInterfaceType(const ObjCInterfaceType *T) {
   Writer.AddDeclRef(T->getDecl(), Record);
-  Code = pch::TYPE_OBJC_INTERFACE;
+  Code = TYPE_OBJC_INTERFACE;
 }
 
-void PCHTypeWriter::VisitObjCObjectType(const ObjCObjectType *T) {
+void ASTTypeWriter::VisitObjCObjectType(const ObjCObjectType *T) {
   Writer.AddTypeRef(T->getBaseType(), Record);
   Record.push_back(T->getNumProtocols());
   for (ObjCObjectType::qual_iterator I = T->qual_begin(),
        E = T->qual_end(); I != E; ++I)
     Writer.AddDeclRef(*I, Record);
-  Code = pch::TYPE_OBJC_OBJECT;
+  Code = TYPE_OBJC_OBJECT;
 }
 
 void
-PCHTypeWriter::VisitObjCObjectPointerType(const ObjCObjectPointerType *T) {
+ASTTypeWriter::VisitObjCObjectPointerType(const ObjCObjectPointerType *T) {
   Writer.AddTypeRef(T->getPointeeType(), Record);
-  Code = pch::TYPE_OBJC_OBJECT_POINTER;
+  Code = TYPE_OBJC_OBJECT_POINTER;
 }
 
 namespace {
 
 class TypeLocWriter : public TypeLocVisitor<TypeLocWriter> {
-  PCHWriter &Writer;
-  PCHWriter::RecordData &Record;
+  ASTWriter &Writer;
+  ASTWriter::RecordData &Record;
 
 public:
-  TypeLocWriter(PCHWriter &Writer, PCHWriter::RecordData &Record)
+  TypeLocWriter(ASTWriter &Writer, ASTWriter::RecordData &Record)
     : Writer(Writer), Record(Record) { }
 
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
@@ -488,12 +501,12 @@ void TypeLocWriter::VisitObjCObjectPointerTypeLoc(ObjCObjectPointerTypeLoc TL) {
 }
 
 //===----------------------------------------------------------------------===//
-// PCHWriter Implementation
+// ASTWriter Implementation
 //===----------------------------------------------------------------------===//
 
 static void EmitBlockID(unsigned ID, const char *Name,
                         llvm::BitstreamWriter &Stream,
-                        PCHWriter::RecordData &Record) {
+                        ASTWriter::RecordData &Record) {
   Record.clear();
   Record.push_back(ID);
   Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
@@ -508,7 +521,7 @@ static void EmitBlockID(unsigned ID, const char *Name,
 
 static void EmitRecordID(unsigned ID, const char *Name,
                          llvm::BitstreamWriter &Stream,
-                         PCHWriter::RecordData &Record) {
+                         ASTWriter::RecordData &Record) {
   Record.clear();
   Record.push_back(ID);
   while (*Name)
@@ -517,8 +530,8 @@ static void EmitRecordID(unsigned ID, const char *Name,
 }
 
 static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
-                          PCHWriter::RecordData &Record) {
-#define RECORD(X) EmitRecordID(pch::X, #X, Stream, Record)
+                          ASTWriter::RecordData &Record) {
+#define RECORD(X) EmitRecordID(X, #X, Stream, Record)
   RECORD(STMT_STOP);
   RECORD(STMT_NULL_PTR);
   RECORD(STMT_NULL);
@@ -597,15 +610,15 @@ static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
 #undef RECORD
 }
 
-void PCHWriter::WriteBlockInfoBlock() {
+void ASTWriter::WriteBlockInfoBlock() {
   RecordData Record;
   Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
 
-#define BLOCK(X) EmitBlockID(pch::X ## _ID, #X, Stream, Record)
-#define RECORD(X) EmitRecordID(pch::X, #X, Stream, Record)
+#define BLOCK(X) EmitBlockID(X ## _ID, #X, Stream, Record)
+#define RECORD(X) EmitRecordID(X, #X, Stream, Record)
 
-  // PCH Top-Level Block.
-  BLOCK(PCH_BLOCK);
+  // AST Top-Level Block.
+  BLOCK(AST_BLOCK);
   RECORD(ORIGINAL_FILE_NAME);
   RECORD(TYPE_OFFSET);
   RECORD(DECL_OFFSET);
@@ -617,7 +630,7 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(SPECIAL_TYPES);
   RECORD(STATISTICS);
   RECORD(TENTATIVE_DEFINITIONS);
-  RECORD(UNUSED_STATIC_FUNCS);
+  RECORD(UNUSED_FILESCOPED_DECLS);
   RECORD(LOCALLY_SCOPED_EXTERNAL_DECLS);
   RECORD(SELECTOR_OFFSETS);
   RECORD(METHOD_POOL);
@@ -627,9 +640,9 @@ void PCHWriter::WriteBlockInfoBlock() {
   RECORD(STAT_CACHE);
   RECORD(EXT_VECTOR_DECLS);
   RECORD(VERSION_CONTROL_BRANCH_REVISION);
-  RECORD(UNUSED_STATIC_FUNCS);
   RECORD(MACRO_DEFINITION_OFFSETS);
   RECORD(CHAINED_METADATA);
+  RECORD(REFERENCED_SELECTOR_POOL);
   
   // SourceManager Block.
   BLOCK(SOURCE_MANAGER_BLOCK);
@@ -742,17 +755,17 @@ adjustFilenameForRelocatablePCH(const char *Filename, const char *isysroot) {
   return Filename + Pos;
 }
 
-/// \brief Write the PCH metadata (e.g., i686-apple-darwin9).
-void PCHWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
+/// \brief Write the AST metadata (e.g., i686-apple-darwin9).
+void ASTWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
   using namespace llvm;
 
   // Metadata
   const TargetInfo &Target = Context.Target;
   BitCodeAbbrev *MetaAbbrev = new BitCodeAbbrev();
   MetaAbbrev->Add(BitCodeAbbrevOp(
-                    Chain ? pch::CHAINED_METADATA : pch::METADATA));
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // PCH major
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // PCH minor
+                    Chain ? CHAINED_METADATA : METADATA));
+  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // AST major
+  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // AST minor
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang major
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang minor
   MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Relocatable
@@ -761,9 +774,9 @@ void PCHWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
   unsigned MetaAbbrevCode = Stream.EmitAbbrev(MetaAbbrev);
 
   RecordData Record;
-  Record.push_back(Chain ? pch::CHAINED_METADATA : pch::METADATA);
-  Record.push_back(pch::VERSION_MAJOR);
-  Record.push_back(pch::VERSION_MINOR);
+  Record.push_back(Chain ? CHAINED_METADATA : METADATA);
+  Record.push_back(VERSION_MAJOR);
+  Record.push_back(VERSION_MINOR);
   Record.push_back(CLANG_VERSION_MAJOR);
   Record.push_back(CLANG_VERSION_MINOR);
   Record.push_back(isysroot != 0);
@@ -775,7 +788,7 @@ void PCHWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
   SourceManager &SM = Context.getSourceManager();
   if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
     BitCodeAbbrev *FileAbbrev = new BitCodeAbbrev();
-    FileAbbrev->Add(BitCodeAbbrevOp(pch::ORIGINAL_FILE_NAME));
+    FileAbbrev->Add(BitCodeAbbrevOp(ORIGINAL_FILE_NAME));
     FileAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
     unsigned FileAbbrevCode = Stream.EmitAbbrev(FileAbbrev);
 
@@ -787,23 +800,23 @@ void PCHWriter::WriteMetadata(ASTContext &Context, const char *isysroot) {
     MainFileNameStr = adjustFilenameForRelocatablePCH(MainFileNameStr,
                                                       isysroot);
     RecordData Record;
-    Record.push_back(pch::ORIGINAL_FILE_NAME);
+    Record.push_back(ORIGINAL_FILE_NAME);
     Stream.EmitRecordWithBlob(FileAbbrevCode, Record, MainFileNameStr);
   }
 
   // Repository branch/version information.
   BitCodeAbbrev *RepoAbbrev = new BitCodeAbbrev();
-  RepoAbbrev->Add(BitCodeAbbrevOp(pch::VERSION_CONTROL_BRANCH_REVISION));
+  RepoAbbrev->Add(BitCodeAbbrevOp(VERSION_CONTROL_BRANCH_REVISION));
   RepoAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // SVN branch/tag
   unsigned RepoAbbrevCode = Stream.EmitAbbrev(RepoAbbrev);
   Record.clear();
-  Record.push_back(pch::VERSION_CONTROL_BRANCH_REVISION);
+  Record.push_back(VERSION_CONTROL_BRANCH_REVISION);
   Stream.EmitRecordWithBlob(RepoAbbrevCode, Record,
                             getClangFullRepositoryVersion());
 }
 
 /// \brief Write the LangOptions structure.
-void PCHWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
+void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
   RecordData Record;
   Record.push_back(LangOpts.Trigraphs);
   Record.push_back(LangOpts.BCPLComment);  // BCPL-style '//' comments.
@@ -874,7 +887,7 @@ void PCHWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
   Record.push_back(LangOpts.CatchUndefined);
   Record.push_back(LangOpts.ElideConstructors);
   Record.push_back(LangOpts.SpellChecking);
-  Stream.EmitRecord(pch::LANGUAGE_OPTIONS, Record);
+  Stream.EmitRecord(LANGUAGE_OPTIONS, Record);
 }
 
 //===----------------------------------------------------------------------===//
@@ -883,7 +896,7 @@ void PCHWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
 
 namespace {
 // Trait used for the on-disk hash table of stat cache results.
-class PCHStatCacheTrait {
+class ASTStatCacheTrait {
 public:
   typedef const char * key_type;
   typedef key_type key_type_ref;
@@ -932,11 +945,11 @@ public:
 };
 } // end anonymous namespace
 
-/// \brief Write the stat() system call cache to the PCH file.
-void PCHWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
+/// \brief Write the stat() system call cache to the AST file.
+void ASTWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
   // Build the on-disk hash table containing information about every
   // stat() call.
-  OnDiskChainedHashTableGenerator<PCHStatCacheTrait> Generator;
+  OnDiskChainedHashTableGenerator<ASTStatCacheTrait> Generator;
   unsigned NumStatEntries = 0;
   for (MemorizeStatCalls::iterator Stat = StatCalls.begin(),
                                 StatEnd = StatCalls.end();
@@ -958,7 +971,7 @@ void PCHWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
   // Create a blob abbreviation
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::STAT_CACHE));
+  Abbrev->Add(BitCodeAbbrevOp(STAT_CACHE));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
@@ -966,7 +979,7 @@ void PCHWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
 
   // Write the stat cache
   RecordData Record;
-  Record.push_back(pch::STAT_CACHE);
+  Record.push_back(STAT_CACHE);
   Record.push_back(BucketOffset);
   Record.push_back(NumStatEntries);
   Stream.EmitRecordWithBlob(StatCacheAbbrev, Record, StatCacheData.str());
@@ -981,7 +994,7 @@ void PCHWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
 static unsigned CreateSLocFileAbbrev(llvm::BitstreamWriter &Stream) {
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_FILE_ENTRY));
+  Abbrev->Add(BitCodeAbbrevOp(SM_SLOC_FILE_ENTRY));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Offset
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Include location
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // Characteristic
@@ -1003,7 +1016,7 @@ static unsigned CreateSLocFileAbbrev(llvm::BitstreamWriter &Stream) {
 static unsigned CreateSLocBufferAbbrev(llvm::BitstreamWriter &Stream) {
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_BUFFER_ENTRY));
+  Abbrev->Add(BitCodeAbbrevOp(SM_SLOC_BUFFER_ENTRY));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Offset
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Include location
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // Characteristic
@@ -1017,7 +1030,7 @@ static unsigned CreateSLocBufferAbbrev(llvm::BitstreamWriter &Stream) {
 static unsigned CreateSLocBufferBlobAbbrev(llvm::BitstreamWriter &Stream) {
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_BUFFER_BLOB));
+  Abbrev->Add(BitCodeAbbrevOp(SM_SLOC_BUFFER_BLOB));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Blob
   return Stream.EmitAbbrev(Abbrev);
 }
@@ -1027,7 +1040,7 @@ static unsigned CreateSLocBufferBlobAbbrev(llvm::BitstreamWriter &Stream) {
 static unsigned CreateSLocInstantiationAbbrev(llvm::BitstreamWriter &Stream) {
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::SM_SLOC_INSTANTIATION_ENTRY));
+  Abbrev->Add(BitCodeAbbrevOp(SM_SLOC_INSTANTIATION_ENTRY));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Offset
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Spelling location
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Start location
@@ -1044,13 +1057,13 @@ static unsigned CreateSLocInstantiationAbbrev(llvm::BitstreamWriter &Stream) {
 /// entries for files that we actually need. In the common case (no
 /// errors), we probably won't have to create file entries for any of
 /// the files in the AST.
-void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
+void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
                                         const Preprocessor &PP,
                                         const char *isysroot) {
   RecordData Record;
 
   // Enter the source manager block.
-  Stream.EnterSubblock(pch::SOURCE_MANAGER_BLOCK_ID, 3);
+  Stream.EnterSubblock(SOURCE_MANAGER_BLOCK_ID, 3);
 
   // Abbreviations for the various kinds of source-location entries.
   unsigned SLocFileAbbrv = CreateSLocFileAbbrev(Stream);
@@ -1092,15 +1105,17 @@ void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
         Record.push_back(LE->IncludeOffset);
       }
     }
-    Stream.EmitRecord(pch::SM_LINE_TABLE, Record);
+    Stream.EmitRecord(SM_LINE_TABLE, Record);
   }
 
   // Write out the source location entry table. We skip the first
   // entry, which is always the same dummy entry.
   std::vector<uint32_t> SLocEntryOffsets;
   RecordData PreloadSLocs;
-  SLocEntryOffsets.reserve(SourceMgr.sloc_entry_size() - 1);
-  for (unsigned I = 1, N = SourceMgr.sloc_entry_size(); I != N; ++I) {
+  unsigned BaseSLocID = Chain ? Chain->getTotalNumSLocs() : 0;
+  SLocEntryOffsets.reserve(SourceMgr.sloc_entry_size() - 1 - BaseSLocID);
+  for (unsigned I = BaseSLocID + 1, N = SourceMgr.sloc_entry_size();
+       I != N; ++I) {
     // Get this source location entry.
     const SrcMgr::SLocEntry *SLoc = &SourceMgr.getSLocEntry(I);
 
@@ -1111,11 +1126,11 @@ void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
     unsigned Code;
     if (SLoc->isFile()) {
       if (SLoc->getFile().getContentCache()->Entry)
-        Code = pch::SM_SLOC_FILE_ENTRY;
+        Code = SM_SLOC_FILE_ENTRY;
       else
-        Code = pch::SM_SLOC_BUFFER_ENTRY;
+        Code = SM_SLOC_BUFFER_ENTRY;
     } else
-      Code = pch::SM_SLOC_INSTANTIATION_ENTRY;
+      Code = SM_SLOC_INSTANTIATION_ENTRY;
     Record.clear();
     Record.push_back(Code);
 
@@ -1157,7 +1172,7 @@ void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
         // FIXME: For now, preload all file source locations, so that
         // we get the appropriate File entries in the reader. This is
         // a temporary measure.
-        PreloadSLocs.push_back(SLocEntryOffsets.size());
+        PreloadSLocs.push_back(BaseSLocID + SLocEntryOffsets.size());
       } else {
         // The source location entry is a buffer. The blob associated
         // with this entry contains the contents of the buffer.
@@ -1171,13 +1186,13 @@ void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
         Stream.EmitRecordWithBlob(SLocBufferAbbrv, Record,
                                   llvm::StringRef(Name, strlen(Name) + 1));
         Record.clear();
-        Record.push_back(pch::SM_SLOC_BUFFER_BLOB);
+        Record.push_back(SM_SLOC_BUFFER_BLOB);
         Stream.EmitRecordWithBlob(SLocBufferBlobAbbrv, Record,
                                   llvm::StringRef(Buffer->getBufferStart(),
                                                   Buffer->getBufferSize() + 1));
 
         if (strcmp(Name, "<built-in>") == 0)
-          PreloadSLocs.push_back(SLocEntryOffsets.size());
+          PreloadSLocs.push_back(BaseSLocID + SLocEntryOffsets.size());
       }
     } else {
       // The source location entry is an instantiation.
@@ -1200,27 +1215,27 @@ void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   if (SLocEntryOffsets.empty())
     return;
 
-  // Write the source-location offsets table into the PCH block. This
+  // Write the source-location offsets table into the AST block. This
   // table is used for lazily loading source-location information.
   using namespace llvm;
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::SOURCE_LOCATION_OFFSETS));
+  Abbrev->Add(BitCodeAbbrevOp(SOURCE_LOCATION_OFFSETS));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // next offset
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // offsets
   unsigned SLocOffsetsAbbrev = Stream.EmitAbbrev(Abbrev);
 
   Record.clear();
-  Record.push_back(pch::SOURCE_LOCATION_OFFSETS);
+  Record.push_back(SOURCE_LOCATION_OFFSETS);
   Record.push_back(SLocEntryOffsets.size());
   Record.push_back(SourceMgr.getNextOffset());
   Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record,
-                            (const char *)&SLocEntryOffsets.front(),
+                            (const char *)data(SLocEntryOffsets),
                            SLocEntryOffsets.size()*sizeof(SLocEntryOffsets[0]));
 
-  // Write the source location entry preloads array, telling the PCH
+  // Write the source location entry preloads array, telling the AST
   // reader which source locations entries it should load eagerly.
-  Stream.EmitRecord(pch::SOURCE_LOCATION_PRELOADS, PreloadSLocs);
+  Stream.EmitRecord(SOURCE_LOCATION_PRELOADS, PreloadSLocs);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1230,20 +1245,20 @@ void PCHWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
 /// \brief Writes the block containing the serialized form of the
 /// preprocessor.
 ///
-void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
+void ASTWriter::WritePreprocessor(const Preprocessor &PP) {
   RecordData Record;
 
   // If the preprocessor __COUNTER__ value has been bumped, remember it.
   if (PP.getCounterValue() != 0) {
     Record.push_back(PP.getCounterValue());
-    Stream.EmitRecord(pch::PP_COUNTER_VALUE, Record);
+    Stream.EmitRecord(PP_COUNTER_VALUE, Record);
     Record.clear();
   }
 
   // Enter the preprocessor block.
-  Stream.EnterSubblock(pch::PREPROCESSOR_BLOCK_ID, 2);
+  Stream.EnterSubblock(PREPROCESSOR_BLOCK_ID, 2);
 
-  // If the PCH file contains __DATE__ or __TIME__ emit a warning about this.
+  // If the AST file contains __DATE__ or __TIME__ emit a warning about this.
   // FIXME: use diagnostics subsystem for localization etc.
   if (PP.SawDateOrTime())
     fprintf(stderr, "warning: precompiled header used __DATE__ or __TIME__.\n");
@@ -1257,9 +1272,10 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
     // order so that output is reproducible.
     MacroInfo *MI = I->second;
 
-    // Don't emit builtin macros like __LINE__ to the PCH file unless they have
+    // Don't emit builtin macros like __LINE__ to the AST file unless they have
     // been redefined by the header (in which case they are not isBuiltinMacro).
-    if (MI->isBuiltinMacro())
+    // Also skip macros from a AST file if we're chaining.
+    if (MI->isBuiltinMacro() || (Chain && MI->isFromAST()))
       continue;
 
     AddIdentifierRef(I->first, Record);
@@ -1269,9 +1285,9 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
 
     unsigned Code;
     if (MI->isObjectLike()) {
-      Code = pch::PP_MACRO_OBJECT_LIKE;
+      Code = PP_MACRO_OBJECT_LIKE;
     } else {
-      Code = pch::PP_MACRO_FUNCTION_LIKE;
+      Code = PP_MACRO_FUNCTION_LIKE;
 
       Record.push_back(MI->isC99Varargs());
       Record.push_back(MI->isGNUVarargs());
@@ -1308,7 +1324,7 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
       // FIXME: Should translate token flags to a stable encoding.
       Record.push_back(Tok.getFlags());
 
-      Stream.EmitRecord(pch::PP_TOKEN, Record);
+      Stream.EmitRecord(PP_TOKEN, Record);
       Record.clear();
     }
     ++NumMacros;
@@ -1327,13 +1343,13 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
         AddSourceLocation(MI->getSourceRange().getEnd(), Record);
         AddIdentifierRef(MI->getName(), Record);
         Record.push_back(getMacroDefinitionID(MI->getDefinition()));
-        Stream.EmitRecord(pch::PP_MACRO_INSTANTIATION, Record);
+        Stream.EmitRecord(PP_MACRO_INSTANTIATION, Record);
         continue;
       }
       
       if (MacroDefinition *MD = dyn_cast<MacroDefinition>(*E)) {
         // Record this macro definition's location.
-        pch::IdentID ID = getMacroDefinitionID(MD);
+        IdentID ID = getMacroDefinitionID(MD);
         if (ID != MacroDefinitionOffsets.size()) {
           if (ID > MacroDefinitionOffsets.size())
             MacroDefinitionOffsets.resize(ID + 1);
@@ -1348,7 +1364,7 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
         AddSourceLocation(MD->getSourceRange().getEnd(), Record);
         AddIdentifierRef(MD->getName(), Record);
         AddSourceLocation(MD->getLocation(), Record);
-        Stream.EmitRecord(pch::PP_MACRO_DEFINITION, Record);
+        Stream.EmitRecord(PP_MACRO_DEFINITION, Record);
         continue;
       }
     }
@@ -1361,18 +1377,18 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
     // Write the offsets table for identifier IDs.
     using namespace llvm;
     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-    Abbrev->Add(BitCodeAbbrevOp(pch::MACRO_DEFINITION_OFFSETS));
+    Abbrev->Add(BitCodeAbbrevOp(MACRO_DEFINITION_OFFSETS));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of records
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macro defs
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned MacroDefOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
     
     Record.clear();
-    Record.push_back(pch::MACRO_DEFINITION_OFFSETS);
+    Record.push_back(MACRO_DEFINITION_OFFSETS);
     Record.push_back(NumPreprocessingRecords);
     Record.push_back(MacroDefinitionOffsets.size());
     Stream.EmitRecordWithBlob(MacroDefOffsetAbbrev, Record,
-                              (const char *)&MacroDefinitionOffsets.front(),
+                              (const char *)data(MacroDefinitionOffsets),
                               MacroDefinitionOffsets.size() * sizeof(uint32_t));
   }
 }
@@ -1381,30 +1397,31 @@ void PCHWriter::WritePreprocessor(const Preprocessor &PP) {
 // Type Serialization
 //===----------------------------------------------------------------------===//
 
-/// \brief Write the representation of a type to the PCH stream.
-void PCHWriter::WriteType(QualType T) {
-  pch::TypeID &ID = TypeIDs[T];
-  if (ID == 0) // we haven't seen this type before.
-    ID = NextTypeID++;
+/// \brief Write the representation of a type to the AST stream.
+void ASTWriter::WriteType(QualType T) {
+  TypeIdx &Idx = TypeIdxs[T];
+  if (Idx.getIndex() == 0) // we haven't seen this type before.
+    Idx = TypeIdx(NextTypeID++);
 
   // Record the offset for this type.
-  if (TypeOffsets.size() == ID - pch::NUM_PREDEF_TYPE_IDS)
+  unsigned Index = Idx.getIndex() - FirstTypeID;
+  if (TypeOffsets.size() == Index)
     TypeOffsets.push_back(Stream.GetCurrentBitNo());
-  else if (TypeOffsets.size() < ID - pch::NUM_PREDEF_TYPE_IDS) {
-    TypeOffsets.resize(ID + 1 - pch::NUM_PREDEF_TYPE_IDS);
-    TypeOffsets[ID - pch::NUM_PREDEF_TYPE_IDS] = Stream.GetCurrentBitNo();
+  else if (TypeOffsets.size() < Index) {
+    TypeOffsets.resize(Index + 1);
+    TypeOffsets[Index] = Stream.GetCurrentBitNo();
   }
 
   RecordData Record;
 
   // Emit the type's representation.
-  PCHTypeWriter W(*this, Record);
+  ASTTypeWriter W(*this, Record);
 
   if (T.hasLocalNonFastQualifiers()) {
     Qualifiers Qs = T.getLocalQualifiers();
     AddTypeRef(T.getLocalUnqualifiedType(), Record);
     Record.push_back(Qs.getAsOpaqueValue());
-    W.Code = pch::TYPE_EXT_QUAL;
+    W.Code = TYPE_EXT_QUAL;
   } else {
     switch (T->getTypeClass()) {
       // For all of the concrete, non-dependent types, call the
@@ -1432,73 +1449,54 @@ void PCHWriter::WriteType(QualType T) {
 ///
 /// \returns the offset of the DECL_CONTEXT_LEXICAL block within the
 /// bistream, or 0 if no block was written.
-uint64_t PCHWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
+uint64_t ASTWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
                                                  DeclContext *DC) {
   if (DC->decls_empty())
     return 0;
 
   uint64_t Offset = Stream.GetCurrentBitNo();
   RecordData Record;
+  Record.push_back(DECL_CONTEXT_LEXICAL);
+  llvm::SmallVector<DeclID, 64> Decls;
   for (DeclContext::decl_iterator D = DC->decls_begin(), DEnd = DC->decls_end();
          D != DEnd; ++D)
-    AddDeclRef(*D, Record);
+    Decls.push_back(GetDeclRef(*D));
 
   ++NumLexicalDeclContexts;
-  Stream.EmitRecord(pch::DECL_CONTEXT_LEXICAL, Record);
+  Stream.EmitRecordWithBlob(DeclContextLexicalAbbrev, Record,
+     reinterpret_cast<char*>(Decls.data()), Decls.size() * sizeof(DeclID));
   return Offset;
 }
 
-/// \brief Write the block containing all of the declaration IDs
-/// visible from the given DeclContext.
-///
-/// \returns the offset of the DECL_CONTEXT_VISIBLE block within the
-/// bistream, or 0 if no block was written.
-uint64_t PCHWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
-                                                 DeclContext *DC) {
-  if (DC->getPrimaryContext() != DC)
-    return 0;
-
-  // Since there is no name lookup into functions or methods, don't bother to
-  // build a visible-declarations table for these entities.
-  if (DC->isFunctionOrMethod())
-    return 0;
-
-  // If not in C++, we perform name lookup for the translation unit via the
-  // IdentifierInfo chains, don't bother to build a visible-declarations table.
-  // FIXME: In C++ we need the visible declarations in order to "see" the
-  // friend declarations, is there a way to do this without writing the table ?
-  if (DC->isTranslationUnit() && !Context.getLangOptions().CPlusPlus)
-    return 0;
-
-  // Force the DeclContext to build a its name-lookup table.
-  DC->lookup(DeclarationName());
-
-  // Serialize the contents of the mapping used for lookup. Note that,
-  // although we have two very different code paths, the serialized
-  // representation is the same for both cases: a declaration name,
-  // followed by a size, followed by references to the visible
-  // declarations that have that name.
-  uint64_t Offset = Stream.GetCurrentBitNo();
+void ASTWriter::WriteTypeDeclOffsets() {
+  using namespace llvm;
   RecordData Record;
-  StoredDeclsMap *Map = static_cast<StoredDeclsMap*>(DC->getLookupPtr());
-  if (!Map)
-    return 0;
 
-  for (StoredDeclsMap::iterator D = Map->begin(), DEnd = Map->end();
-       D != DEnd; ++D) {
-    AddDeclarationName(D->first, Record);
-    DeclContext::lookup_result Result = D->second.getLookupResult(Context);
-    Record.push_back(Result.second - Result.first);
-    for (; Result.first != Result.second; ++Result.first)
-      AddDeclRef(*Result.first, Record);
-  }
-
-  if (Record.size() == 0)
-    return 0;
+  // Write the type offsets array
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(TYPE_OFFSET));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of types
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // types block
+  unsigned TypeOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
+  Record.clear();
+  Record.push_back(TYPE_OFFSET);
+  Record.push_back(TypeOffsets.size());
+  Stream.EmitRecordWithBlob(TypeOffsetAbbrev, Record,
+                            (const char *)data(TypeOffsets),
+                            TypeOffsets.size() * sizeof(TypeOffsets[0]));
 
-  Stream.EmitRecord(pch::DECL_CONTEXT_VISIBLE, Record);
-  ++NumVisibleDeclContexts;
-  return Offset;
+  // Write the declaration offsets array
+  Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(DECL_OFFSET));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block
+  unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
+  Record.clear();
+  Record.push_back(DECL_OFFSET);
+  Record.push_back(DeclOffsets.size());
+  Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record,
+                            (const char *)data(DeclOffsets),
+                            DeclOffsets.size() * sizeof(DeclOffsets[0]));
 }
 
 //===----------------------------------------------------------------------===//
@@ -1507,27 +1505,23 @@ uint64_t PCHWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
 
 namespace {
 // Trait used for the on-disk hash table used in the method pool.
-class PCHMethodPoolTrait {
-  PCHWriter &Writer;
+class ASTMethodPoolTrait {
+  ASTWriter &Writer;
 
 public:
   typedef Selector key_type;
   typedef key_type key_type_ref;
 
-  typedef std::pair<ObjCMethodList, ObjCMethodList> data_type;
+  struct data_type {
+    SelectorID ID;
+    ObjCMethodList Instance, Factory;
+  };
   typedef const data_type& data_type_ref;
 
-  explicit PCHMethodPoolTrait(PCHWriter &Writer) : Writer(Writer) { }
+  explicit ASTMethodPoolTrait(ASTWriter &Writer) : Writer(Writer) { }
 
   static unsigned ComputeHash(Selector Sel) {
-    unsigned N = Sel.getNumArgs();
-    if (N == 0)
-      ++N;
-    unsigned R = 5381;
-    for (unsigned I = 0; I != N; ++I)
-      if (IdentifierInfo *II = Sel.getIdentifierInfoForSlot(I))
-        R = llvm::HashString(II->getName(), R);
-    return R;
+    return serialization::ComputeHash(Sel);
   }
 
   std::pair<unsigned,unsigned>
@@ -1535,12 +1529,12 @@ public:
                       data_type_ref Methods) {
     unsigned KeyLen = 2 + (Sel.getNumArgs()? Sel.getNumArgs() * 4 : 4);
     clang::io::Emit16(Out, KeyLen);
-    unsigned DataLen = 2 + 2; // 2 bytes for each of the method counts
-    for (const ObjCMethodList *Method = &Methods.first; Method;
+    unsigned DataLen = 4 + 2 + 2; // 2 bytes for each of the method counts
+    for (const ObjCMethodList *Method = &Methods.Instance; Method;
          Method = Method->Next)
       if (Method->Method)
         DataLen += 4;
-    for (const ObjCMethodList *Method = &Methods.second; Method;
+    for (const ObjCMethodList *Method = &Methods.Factory; Method;
          Method = Method->Next)
       if (Method->Method)
         DataLen += 4;
@@ -1564,25 +1558,26 @@ public:
   void EmitData(llvm::raw_ostream& Out, key_type_ref,
                 data_type_ref Methods, unsigned DataLen) {
     uint64_t Start = Out.tell(); (void)Start;
+    clang::io::Emit32(Out, Methods.ID);
     unsigned NumInstanceMethods = 0;
-    for (const ObjCMethodList *Method = &Methods.first; Method;
+    for (const ObjCMethodList *Method = &Methods.Instance; Method;
          Method = Method->Next)
       if (Method->Method)
         ++NumInstanceMethods;
 
     unsigned NumFactoryMethods = 0;
-    for (const ObjCMethodList *Method = &Methods.second; Method;
+    for (const ObjCMethodList *Method = &Methods.Factory; Method;
          Method = Method->Next)
       if (Method->Method)
         ++NumFactoryMethods;
 
     clang::io::Emit16(Out, NumInstanceMethods);
     clang::io::Emit16(Out, NumFactoryMethods);
-    for (const ObjCMethodList *Method = &Methods.first; Method;
+    for (const ObjCMethodList *Method = &Methods.Instance; Method;
          Method = Method->Next)
       if (Method->Method)
         clang::io::Emit32(Out, Writer.getDeclID(Method->Method));
-    for (const ObjCMethodList *Method = &Methods.second; Method;
+    for (const ObjCMethodList *Method = &Methods.Factory; Method;
          Method = Method->Next)
       if (Method->Method)
         clang::io::Emit32(Out, Writer.getDeclID(Method->Method));
@@ -1592,89 +1587,78 @@ public:
 };
 } // end anonymous namespace
 
-/// \brief Write the method pool into the PCH file.
+/// \brief Write ObjC data: selectors and the method pool.
 ///
 /// The method pool contains both instance and factory methods, stored
-/// in an on-disk hash table indexed by the selector.
-void PCHWriter::WriteMethodPool(Sema &SemaRef) {
+/// in an on-disk hash table indexed by the selector. The hash table also
+/// contains an empty entry for every other selector known to Sema.
+void ASTWriter::WriteSelectors(Sema &SemaRef) {
   using namespace llvm;
 
-  // Create and write out the blob that contains the instance and
-  // factor method pools.
-  bool Empty = true;
+  // Do we have to do anything at all?
+  if (SemaRef.MethodPool.empty() && SelectorIDs.empty())
+    return;
+  unsigned NumTableEntries = 0;
+  // Create and write out the blob that contains selectors and the method pool.
   {
-    OnDiskChainedHashTableGenerator<PCHMethodPoolTrait> Generator;
-
-    // Create the on-disk hash table representation. Start by
-    // iterating through the instance method pool.
-    PCHMethodPoolTrait::key_type Key;
-    unsigned NumSelectorsInMethodPool = 0;
-    for (llvm::DenseMap<Selector, ObjCMethodList>::iterator
-           Instance = SemaRef.InstanceMethodPool.begin(),
-           InstanceEnd = SemaRef.InstanceMethodPool.end();
-         Instance != InstanceEnd; ++Instance) {
-      // Check whether there is a factory method with the same
-      // selector.
-      llvm::DenseMap<Selector, ObjCMethodList>::iterator Factory
-        = SemaRef.FactoryMethodPool.find(Instance->first);
-
-      if (Factory == SemaRef.FactoryMethodPool.end())
-        Generator.insert(Instance->first,
-                         std::make_pair(Instance->second,
-                                        ObjCMethodList()));
-      else
-        Generator.insert(Instance->first,
-                         std::make_pair(Instance->second, Factory->second));
-
-      ++NumSelectorsInMethodPool;
-      Empty = false;
-    }
-
-    // Now iterate through the factory method pool, to pick up any
-    // selectors that weren't already in the instance method pool.
-    for (llvm::DenseMap<Selector, ObjCMethodList>::iterator
-           Factory = SemaRef.FactoryMethodPool.begin(),
-           FactoryEnd = SemaRef.FactoryMethodPool.end();
-         Factory != FactoryEnd; ++Factory) {
-      // Check whether there is an instance method with the same
-      // selector. If so, there is no work to do here.
-      llvm::DenseMap<Selector, ObjCMethodList>::iterator Instance
-        = SemaRef.InstanceMethodPool.find(Factory->first);
-
-      if (Instance == SemaRef.InstanceMethodPool.end()) {
-        Generator.insert(Factory->first,
-                         std::make_pair(ObjCMethodList(), Factory->second));
-        ++NumSelectorsInMethodPool;
+    OnDiskChainedHashTableGenerator<ASTMethodPoolTrait> Generator;
+    ASTMethodPoolTrait Trait(*this);
+
+    // Create the on-disk hash table representation. We walk through every
+    // selector we've seen and look it up in the method pool.
+    SelectorOffsets.resize(NextSelectorID - FirstSelectorID);
+    for (llvm::DenseMap<Selector, SelectorID>::iterator
+             I = SelectorIDs.begin(), E = SelectorIDs.end();
+         I != E; ++I) {
+      Selector S = I->first;
+      Sema::GlobalMethodPool::iterator F = SemaRef.MethodPool.find(S);
+      ASTMethodPoolTrait::data_type Data = {
+        I->second,
+        ObjCMethodList(),
+        ObjCMethodList()
+      };
+      if (F != SemaRef.MethodPool.end()) {
+        Data.Instance = F->second.first;
+        Data.Factory = F->second.second;
       }
-
-      Empty = false;
+      // Only write this selector if it's not in an existing AST or something
+      // changed.
+      if (Chain && I->second < FirstSelectorID) {
+        // Selector already exists. Did it change?
+        bool changed = false;
+        for (ObjCMethodList *M = &Data.Instance; !changed && M && M->Method;
+             M = M->Next) {
+          if (M->Method->getPCHLevel() == 0)
+            changed = true;
+        }
+        for (ObjCMethodList *M = &Data.Factory; !changed && M && M->Method;
+             M = M->Next) {
+          if (M->Method->getPCHLevel() == 0)
+            changed = true;
+        }
+        if (!changed)
+          continue;
+      } else if (Data.Instance.Method || Data.Factory.Method) {
+        // A new method pool entry.
+        ++NumTableEntries;
+      }
+      Generator.insert(S, Data, Trait);
     }
 
-    if (Empty && SelectorOffsets.empty())
-      return;
-
     // Create the on-disk hash table in a buffer.
     llvm::SmallString<4096> MethodPool;
     uint32_t BucketOffset;
-    SelectorOffsets.resize(SelVector.size());
     {
-      PCHMethodPoolTrait Trait(*this);
+      ASTMethodPoolTrait Trait(*this);
       llvm::raw_svector_ostream Out(MethodPool);
       // Make sure that no bucket is at offset 0
       clang::io::Emit32(Out, 0);
       BucketOffset = Generator.Emit(Out, Trait);
-
-      // For every selector that we have seen but which was not
-      // written into the hash table, write the selector itself and
-      // record it's offset.
-      for (unsigned I = 0, N = SelVector.size(); I != N; ++I)
-        if (SelectorOffsets[I] == 0)
-          Trait.EmitKey(Out, SelVector[I], 0);
     }
 
     // Create a blob abbreviation
     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-    Abbrev->Add(BitCodeAbbrevOp(pch::METHOD_POOL));
+    Abbrev->Add(BitCodeAbbrevOp(METHOD_POOL));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
@@ -1682,35 +1666,57 @@ void PCHWriter::WriteMethodPool(Sema &SemaRef) {
 
     // Write the method pool
     RecordData Record;
-    Record.push_back(pch::METHOD_POOL);
+    Record.push_back(METHOD_POOL);
     Record.push_back(BucketOffset);
-    Record.push_back(NumSelectorsInMethodPool);
+    Record.push_back(NumTableEntries);
     Stream.EmitRecordWithBlob(MethodPoolAbbrev, Record, MethodPool.str());
 
     // Create a blob abbreviation for the selector table offsets.
     Abbrev = new BitCodeAbbrev();
-    Abbrev->Add(BitCodeAbbrevOp(pch::SELECTOR_OFFSETS));
+    Abbrev->Add(BitCodeAbbrevOp(SELECTOR_OFFSETS));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // index
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned SelectorOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
 
     // Write the selector offsets table.
     Record.clear();
-    Record.push_back(pch::SELECTOR_OFFSETS);
+    Record.push_back(SELECTOR_OFFSETS);
     Record.push_back(SelectorOffsets.size());
     Stream.EmitRecordWithBlob(SelectorOffsetAbbrev, Record,
-                              (const char *)&SelectorOffsets.front(),
+                              (const char *)data(SelectorOffsets),
                               SelectorOffsets.size() * 4);
   }
 }
 
+/// \brief Write the selectors referenced in @selector expression into AST file.
+void ASTWriter::WriteReferencedSelectorsPool(Sema &SemaRef) {
+  using namespace llvm;
+  if (SemaRef.ReferencedSelectors.empty())
+    return;
+
+  RecordData Record;
+
+  // Note: this writes out all references even for a dependent AST. But it is
+  // very tricky to fix, and given that @selector shouldn't really appear in
+  // headers, probably not worth it. It's not a correctness issue.
+  for (DenseMap<Selector, SourceLocation>::iterator S =
+       SemaRef.ReferencedSelectors.begin(),
+       E = SemaRef.ReferencedSelectors.end(); S != E; ++S) {
+    Selector Sel = (*S).first;
+    SourceLocation Loc = (*S).second;
+    AddSelectorRef(Sel, Record);
+    AddSourceLocation(Loc, Record);
+  }
+  Stream.EmitRecord(REFERENCED_SELECTOR_POOL, Record);
+}
+
 //===----------------------------------------------------------------------===//
 // Identifier Table Serialization
 //===----------------------------------------------------------------------===//
 
 namespace {
-class PCHIdentifierTableTrait {
-  PCHWriter &Writer;
+class ASTIdentifierTableTrait {
+  ASTWriter &Writer;
   Preprocessor &PP;
 
   /// \brief Determines whether this is an "interesting" identifier
@@ -1728,10 +1734,10 @@ public:
   typedef const IdentifierInfo* key_type;
   typedef key_type  key_type_ref;
 
-  typedef pch::IdentID data_type;
+  typedef IdentID data_type;
   typedef data_type data_type_ref;
 
-  PCHIdentifierTableTrait(PCHWriter &Writer, Preprocessor &PP)
+  ASTIdentifierTableTrait(ASTWriter &Writer, Preprocessor &PP)
     : Writer(Writer), PP(PP) { }
 
   static unsigned ComputeHash(const IdentifierInfo* II) {
@@ -1740,7 +1746,7 @@ public:
 
   std::pair<unsigned,unsigned>
     EmitKeyDataLength(llvm::raw_ostream& Out, const IdentifierInfo* II,
-                      pch::IdentID ID) {
+                      IdentID ID) {
     unsigned KeyLen = II->getLength() + 1;
     unsigned DataLen = 4; // 4 bytes for the persistent ID << 1
     if (isInterestingIdentifier(II)) {
@@ -1751,7 +1757,7 @@ public:
       for (IdentifierResolver::iterator D = IdentifierResolver::begin(II),
                                      DEnd = IdentifierResolver::end();
            D != DEnd; ++D)
-        DataLen += sizeof(pch::DeclID);
+        DataLen += sizeof(DeclID);
     }
     clang::io::Emit16(Out, DataLen);
     // We emit the key length after the data length so that every
@@ -1770,7 +1776,7 @@ public:
   }
 
   void EmitData(llvm::raw_ostream& Out, const IdentifierInfo* II,
-                pch::IdentID ID, unsigned) {
+                IdentID ID, unsigned) {
     if (!isInterestingIdentifier(II)) {
       clang::io::Emit32(Out, ID << 1);
       return;
@@ -1785,6 +1791,7 @@ public:
     Bits = (Bits << 1) | unsigned(hasMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
+    Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     clang::io::Emit16(Out, Bits);
 
@@ -1797,6 +1804,7 @@ public:
     // "stat"), but IdentifierResolver::AddDeclToIdentifierChain()
     // adds declarations to the end of the list (so we need to see the
     // struct "status" before the function "status").
+    // Only emit declarations that aren't from a chained PCH, though.
     llvm::SmallVector<Decl *, 16> Decls(IdentifierResolver::begin(II),
                                         IdentifierResolver::end());
     for (llvm::SmallVector<Decl *, 16>::reverse_iterator D = Decls.rbegin(),
@@ -1807,43 +1815,46 @@ public:
 };
 } // end anonymous namespace
 
-/// \brief Write the identifier table into the PCH file.
+/// \brief Write the identifier table into the AST file.
 ///
 /// The identifier table consists of a blob containing string data
 /// (the actual identifiers themselves) and a separate "offsets" index
 /// that maps identifier IDs to locations within the blob.
-void PCHWriter::WriteIdentifierTable(Preprocessor &PP) {
+void ASTWriter::WriteIdentifierTable(Preprocessor &PP) {
   using namespace llvm;
 
   // Create and write out the blob that contains the identifier
   // strings.
   {
-    OnDiskChainedHashTableGenerator<PCHIdentifierTableTrait> Generator;
+    OnDiskChainedHashTableGenerator<ASTIdentifierTableTrait> Generator;
+    ASTIdentifierTableTrait Trait(*this, PP);
 
     // Look for any identifiers that were named while processing the
     // headers, but are otherwise not needed. We add these to the hash
     // table to enable checking of the predefines buffer in the case
-    // where the user adds new macro definitions when building the PCH
+    // where the user adds new macro definitions when building the AST
     // file.
     for (IdentifierTable::iterator ID = PP.getIdentifierTable().begin(),
                                 IDEnd = PP.getIdentifierTable().end();
          ID != IDEnd; ++ID)
       getIdentifierRef(ID->second);
 
-    // Create the on-disk hash table representation.
-    IdentifierOffsets.resize(IdentifierIDs.size());
-    for (llvm::DenseMap<const IdentifierInfo *, pch::IdentID>::iterator
+    // Create the on-disk hash table representation. We only store offsets
+    // for identifiers that appear here for the first time.
+    IdentifierOffsets.resize(NextIdentID - FirstIdentID);
+    for (llvm::DenseMap<const IdentifierInfo *, IdentID>::iterator
            ID = IdentifierIDs.begin(), IDEnd = IdentifierIDs.end();
          ID != IDEnd; ++ID) {
       assert(ID->first && "NULL identifier in identifier table");
-      Generator.insert(ID->first, ID->second);
+      if (!Chain || !ID->first->isFromAST())
+        Generator.insert(ID->first, ID->second, Trait);
     }
 
     // Create the on-disk hash table in a buffer.
     llvm::SmallString<4096> IdentifierTable;
     uint32_t BucketOffset;
     {
-      PCHIdentifierTableTrait Trait(*this, PP);
+      ASTIdentifierTableTrait Trait(*this, PP);
       llvm::raw_svector_ostream Out(IdentifierTable);
       // Make sure that no bucket is at offset 0
       clang::io::Emit32(Out, 0);
@@ -1852,236 +1863,345 @@ void PCHWriter::WriteIdentifierTable(Preprocessor &PP) {
 
     // Create a blob abbreviation
     BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-    Abbrev->Add(BitCodeAbbrevOp(pch::IDENTIFIER_TABLE));
+    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_TABLE));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
 
     // Write the identifier table
     RecordData Record;
-    Record.push_back(pch::IDENTIFIER_TABLE);
+    Record.push_back(IDENTIFIER_TABLE);
     Record.push_back(BucketOffset);
     Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
   }
 
   // Write the offsets table for identifier IDs.
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::IDENTIFIER_OFFSET));
+  Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of identifiers
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
   unsigned IdentifierOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
 
   RecordData Record;
-  Record.push_back(pch::IDENTIFIER_OFFSET);
+  Record.push_back(IDENTIFIER_OFFSET);
   Record.push_back(IdentifierOffsets.size());
   Stream.EmitRecordWithBlob(IdentifierOffsetAbbrev, Record,
-                            (const char *)&IdentifierOffsets.front(),
+                            (const char *)data(IdentifierOffsets),
                             IdentifierOffsets.size() * sizeof(uint32_t));
 }
 
 //===----------------------------------------------------------------------===//
-// General Serialization Routines
+// DeclContext's Name Lookup Table Serialization
 //===----------------------------------------------------------------------===//
 
-/// \brief Write a record containing the given attributes.
-void PCHWriter::WriteAttributeRecord(const Attr *Attr) {
-  RecordData Record;
-  for (; Attr; Attr = Attr->getNext()) {
-    Record.push_back(Attr->getKind()); // FIXME: stable encoding, target attrs
-    Record.push_back(Attr->isInherited());
-    switch (Attr->getKind()) {
-    default:
-      assert(0 && "Does not support PCH writing for this attribute yet!");
+namespace {
+// Trait used for the on-disk hash table used in the method pool.
+class ASTDeclContextNameLookupTrait {
+  ASTWriter &Writer;
+
+public:
+  typedef DeclarationName key_type;
+  typedef key_type key_type_ref;
+
+  typedef DeclContext::lookup_result data_type;
+  typedef const data_type& data_type_ref;
+
+  explicit ASTDeclContextNameLookupTrait(ASTWriter &Writer) : Writer(Writer) { }
+
+  unsigned ComputeHash(DeclarationName Name) {
+    llvm::FoldingSetNodeID ID;
+    ID.AddInteger(Name.getNameKind());
+
+    switch (Name.getNameKind()) {
+    case DeclarationName::Identifier:
+      ID.AddString(Name.getAsIdentifierInfo()->getName());
       break;
-    case attr::Alias:
-      AddString(cast<AliasAttr>(Attr)->getAliasee(), Record);
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      ID.AddInteger(serialization::ComputeHash(Name.getObjCSelector()));
       break;
-
-    case attr::AlignMac68k:
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+      ID.AddInteger(Writer.GetOrCreateTypeID(Name.getCXXNameType()));
       break;
-
-    case attr::Aligned:
-      Record.push_back(cast<AlignedAttr>(Attr)->getAlignment());
+    case DeclarationName::CXXOperatorName:
+      ID.AddInteger(Name.getCXXOverloadedOperator());
       break;
-
-    case attr::AlwaysInline:
+    case DeclarationName::CXXLiteralOperatorName:
+      ID.AddString(Name.getCXXLiteralIdentifier()->getName());
+    case DeclarationName::CXXUsingDirective:
       break;
+    }
 
-    case attr::AnalyzerNoReturn:
-      break;
+    return ID.ComputeHash();
+  }
 
-    case attr::Annotate:
-      AddString(cast<AnnotateAttr>(Attr)->getAnnotation(), Record);
+  std::pair<unsigned,unsigned>
+    EmitKeyDataLength(llvm::raw_ostream& Out, DeclarationName Name,
+                      data_type_ref Lookup) {
+    unsigned KeyLen = 1;
+    switch (Name.getNameKind()) {
+    case DeclarationName::Identifier:
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+    case DeclarationName::CXXLiteralOperatorName:
+      KeyLen += 4;
       break;
-
-    case attr::AsmLabel:
-      AddString(cast<AsmLabelAttr>(Attr)->getLabel(), Record);
+    case DeclarationName::CXXOperatorName:
+      KeyLen += 1;
       break;
-
-    case attr::BaseCheck:
+    case DeclarationName::CXXUsingDirective:
       break;
+    }
+    clang::io::Emit16(Out, KeyLen);
 
-    case attr::Blocks:
-      Record.push_back(cast<BlocksAttr>(Attr)->getType()); // FIXME: stable
-      break;
+    // 2 bytes for num of decls and 4 for each DeclID.
+    unsigned DataLen = 2 + 4 * (Lookup.second - Lookup.first);
+    clang::io::Emit16(Out, DataLen);
 
-    case attr::CDecl:
-      break;
+    return std::make_pair(KeyLen, DataLen);
+  }
 
-    case attr::Cleanup:
-      AddDeclRef(cast<CleanupAttr>(Attr)->getFunctionDecl(), Record);
-      break;
+  void EmitKey(llvm::raw_ostream& Out, DeclarationName Name, unsigned) {
+    using namespace clang::io;
 
-    case attr::Const:
+    assert(Name.getNameKind() < 0x100 && "Invalid name kind ?");
+    Emit8(Out, Name.getNameKind());
+    switch (Name.getNameKind()) {
+    case DeclarationName::Identifier:
+      Emit32(Out, Writer.getIdentifierRef(Name.getAsIdentifierInfo()));
       break;
-
-    case attr::Constructor:
-      Record.push_back(cast<ConstructorAttr>(Attr)->getPriority());
+    case DeclarationName::ObjCZeroArgSelector:
+    case DeclarationName::ObjCOneArgSelector:
+    case DeclarationName::ObjCMultiArgSelector:
+      Emit32(Out, Writer.getSelectorRef(Name.getObjCSelector()));
       break;
-
-    case attr::DLLExport:
-    case attr::DLLImport:
-    case attr::Deprecated:
+    case DeclarationName::CXXConstructorName:
+    case DeclarationName::CXXDestructorName:
+    case DeclarationName::CXXConversionFunctionName:
+      Emit32(Out, Writer.getTypeID(Name.getCXXNameType()));
       break;
-
-    case attr::Destructor:
-      Record.push_back(cast<DestructorAttr>(Attr)->getPriority());
+    case DeclarationName::CXXOperatorName:
+      assert(Name.getCXXOverloadedOperator() < 0x100 && "Invalid operator ?");
+      Emit8(Out, Name.getCXXOverloadedOperator());
       break;
-
-    case attr::FastCall:
-    case attr::Final:
+    case DeclarationName::CXXLiteralOperatorName:
+      Emit32(Out, Writer.getIdentifierRef(Name.getCXXLiteralIdentifier()));
       break;
-
-    case attr::Format: {
-      const FormatAttr *Format = cast<FormatAttr>(Attr);
-      AddString(Format->getType(), Record);
-      Record.push_back(Format->getFormatIdx());
-      Record.push_back(Format->getFirstArg());
+    case DeclarationName::CXXUsingDirective:
       break;
     }
+  }
 
-    case attr::FormatArg: {
-      const FormatArgAttr *Format = cast<FormatArgAttr>(Attr);
-      Record.push_back(Format->getFormatIdx());
-      break;
-    }
+  void EmitData(llvm::raw_ostream& Out, key_type_ref,
+                data_type Lookup, unsigned DataLen) {
+    uint64_t Start = Out.tell(); (void)Start;
+    clang::io::Emit16(Out, Lookup.second - Lookup.first);
+    for (; Lookup.first != Lookup.second; ++Lookup.first)
+      clang::io::Emit32(Out, Writer.GetDeclRef(*Lookup.first));
 
-    case attr::Sentinel : {
-      const SentinelAttr *Sentinel = cast<SentinelAttr>(Attr);
-      Record.push_back(Sentinel->getSentinel());
-      Record.push_back(Sentinel->getNullPos());
-      break;
-    }
+    assert(Out.tell() - Start == DataLen && "Data length is wrong");
+  }
+};
+} // end anonymous namespace
 
-    case attr::GNUInline:
-    case attr::Hiding:
-    case attr::IBAction:
-    case attr::IBOutlet:
-    case attr::Malloc:
-    case attr::NoDebug:
-    case attr::NoInline:
-    case attr::NoReturn:
-    case attr::NoThrow:
-      break;
+/// \brief Write the block containing all of the declaration IDs
+/// visible from the given DeclContext.
+///
+/// \returns the offset of the DECL_CONTEXT_VISIBLE block within the
+/// bitstream, or 0 if no block was written.
+uint64_t ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
+                                                 DeclContext *DC) {
+  if (DC->getPrimaryContext() != DC)
+    return 0;
 
-    case attr::IBOutletCollection: {
-      const IBOutletCollectionAttr *ICA = cast<IBOutletCollectionAttr>(Attr);
-      AddDeclRef(ICA->getClass(), Record);
-      break;
-    }
+  // Since there is no name lookup into functions or methods, don't bother to
+  // build a visible-declarations table for these entities.
+  if (DC->isFunctionOrMethod())
+    return 0;
 
-    case attr::NonNull: {
-      const NonNullAttr *NonNull = cast<NonNullAttr>(Attr);
-      Record.push_back(NonNull->size());
-      Record.insert(Record.end(), NonNull->begin(), NonNull->end());
-      break;
-    }
+  // If not in C++, we perform name lookup for the translation unit via the
+  // IdentifierInfo chains, don't bother to build a visible-declarations table.
+  // FIXME: In C++ we need the visible declarations in order to "see" the
+  // friend declarations, is there a way to do this without writing the table ?
+  if (DC->isTranslationUnit() && !Context.getLangOptions().CPlusPlus)
+    return 0;
 
-    case attr::CFReturnsNotRetained:
-    case attr::CFReturnsRetained:
-    case attr::NSReturnsNotRetained:
-    case attr::NSReturnsRetained:
-    case attr::ObjCException:
-    case attr::ObjCNSObject:
-    case attr::Overloadable:
-    case attr::Override:
-      break;
+  // Force the DeclContext to build a its name-lookup table.
+  if (DC->hasExternalVisibleStorage())
+    DC->MaterializeVisibleDeclsFromExternalStorage();
+  else
+    DC->lookup(DeclarationName());
 
-    case attr::MaxFieldAlignment:
-      Record.push_back(cast<MaxFieldAlignmentAttr>(Attr)->getAlignment());
-      break;
+  // Serialize the contents of the mapping used for lookup. Note that,
+  // although we have two very different code paths, the serialized
+  // representation is the same for both cases: a declaration name,
+  // followed by a size, followed by references to the visible
+  // declarations that have that name.
+  uint64_t Offset = Stream.GetCurrentBitNo();
+  StoredDeclsMap *Map = static_cast<StoredDeclsMap*>(DC->getLookupPtr());
+  if (!Map || Map->empty())
+    return 0;
 
-    case attr::Packed:
-      break;
+  OnDiskChainedHashTableGenerator<ASTDeclContextNameLookupTrait> Generator;
+  ASTDeclContextNameLookupTrait Trait(*this);
 
-    case attr::Pure:
-      break;
+  // Create the on-disk hash table representation.
+  for (StoredDeclsMap::iterator D = Map->begin(), DEnd = Map->end();
+       D != DEnd; ++D) {
+    DeclarationName Name = D->first;
+    DeclContext::lookup_result Result = D->second.getLookupResult();
+    Generator.insert(Name, Result, Trait);
+  }
 
-    case attr::Regparm:
-      Record.push_back(cast<RegparmAttr>(Attr)->getNumParams());
-      break;
+  // Create the on-disk hash table in a buffer.
+  llvm::SmallString<4096> LookupTable;
+  uint32_t BucketOffset;
+  {
+    llvm::raw_svector_ostream Out(LookupTable);
+    // Make sure that no bucket is at offset 0
+    clang::io::Emit32(Out, 0);
+    BucketOffset = Generator.Emit(Out, Trait);
+  }
 
-    case attr::ReqdWorkGroupSize:
-      Record.push_back(cast<ReqdWorkGroupSizeAttr>(Attr)->getXDim());
-      Record.push_back(cast<ReqdWorkGroupSizeAttr>(Attr)->getYDim());
-      Record.push_back(cast<ReqdWorkGroupSizeAttr>(Attr)->getZDim());
-      break;
+  // Write the lookup table
+  RecordData Record;
+  Record.push_back(DECL_CONTEXT_VISIBLE);
+  Record.push_back(BucketOffset);
+  Stream.EmitRecordWithBlob(DeclContextVisibleLookupAbbrev, Record,
+                            LookupTable.str());
 
-    case attr::Section:
-      AddString(cast<SectionAttr>(Attr)->getName(), Record);
-      break;
+  Stream.EmitRecord(DECL_CONTEXT_VISIBLE, Record);
+  ++NumVisibleDeclContexts;
+  return Offset;
+}
 
-    case attr::StdCall:
-    case attr::TransparentUnion:
-    case attr::Unavailable:
-    case attr::Unused:
-    case attr::Used:
-      break;
+/// \brief Write an UPDATE_VISIBLE block for the given context.
+///
+/// UPDATE_VISIBLE blocks contain the declarations that are added to an existing
+/// DeclContext in a dependent AST file. As such, they only exist for the TU
+/// (in C++) and for namespaces.
+void ASTWriter::WriteDeclContextVisibleUpdate(const DeclContext *DC) {
+  assert((DC->isTranslationUnit() || DC->isNamespace()) &&
+         "Only TU and namespaces should have visible decl updates.");
+
+  // Make the context build its lookup table, but don't make it load external
+  // decls.
+  DC->lookup(DeclarationName());
 
-    case attr::Visibility:
-      // FIXME: stable encoding
-      Record.push_back(cast<VisibilityAttr>(Attr)->getVisibility());
-      break;
+  StoredDeclsMap *Map = static_cast<StoredDeclsMap*>(DC->getLookupPtr());
+  if (!Map || Map->empty())
+    return;
 
-    case attr::WarnUnusedResult:
-    case attr::Weak:
-    case attr::WeakRef:
-    case attr::WeakImport:
-      break;
-    }
+  OnDiskChainedHashTableGenerator<ASTDeclContextNameLookupTrait> Generator;
+  ASTDeclContextNameLookupTrait Trait(*this);
+
+  // Create the hash table.
+  for (StoredDeclsMap::iterator D = Map->begin(), DEnd = Map->end();
+       D != DEnd; ++D) {
+    DeclarationName Name = D->first;
+    DeclContext::lookup_result Result = D->second.getLookupResult();
+    // For any name that appears in this table, the results are complete, i.e.
+    // they overwrite results from previous PCHs. Merging is always a mess.
+    Generator.insert(Name, Result, Trait);
+  }
+
+  // Create the on-disk hash table in a buffer.
+  llvm::SmallString<4096> LookupTable;
+  uint32_t BucketOffset;
+  {
+    llvm::raw_svector_ostream Out(LookupTable);
+    // Make sure that no bucket is at offset 0
+    clang::io::Emit32(Out, 0);
+    BucketOffset = Generator.Emit(Out, Trait);
   }
 
-  Stream.EmitRecord(pch::DECL_ATTR, Record);
+  // Write the lookup table
+  RecordData Record;
+  Record.push_back(UPDATE_VISIBLE);
+  Record.push_back(getDeclID(cast<Decl>(DC)));
+  Record.push_back(BucketOffset);
+  Stream.EmitRecordWithBlob(UpdateVisibleAbbrev, Record, LookupTable.str());
+}
+
+/// \brief Write ADDITIONAL_TEMPLATE_SPECIALIZATIONS blocks for all templates
+/// that have new specializations in the current AST file.
+void ASTWriter::WriteAdditionalTemplateSpecializations() {
+  RecordData Record;
+  for (AdditionalTemplateSpecializationsMap::iterator
+           I = AdditionalTemplateSpecializations.begin(),
+           E = AdditionalTemplateSpecializations.end();
+       I != E; ++I) {
+    Record.clear();
+    Record.push_back(I->first);
+    Record.insert(Record.end(), I->second.begin(), I->second.end());
+    Stream.EmitRecord(ADDITIONAL_TEMPLATE_SPECIALIZATIONS, Record);
+  }
 }
 
-void PCHWriter::AddString(const std::string &Str, RecordData &Record) {
+//===----------------------------------------------------------------------===//
+// General Serialization Routines
+//===----------------------------------------------------------------------===//
+
+/// \brief Write a record containing the given attributes.
+void ASTWriter::WriteAttributeRecord(const AttrVec &Attrs) {
+  RecordData Record;
+  for (AttrVec::const_iterator i = Attrs.begin(), e = Attrs.end(); i != e; ++i){
+    const Attr * A = *i;
+    Record.push_back(A->getKind()); // FIXME: stable encoding, target attrs
+    AddSourceLocation(A->getLocation(), Record);
+    Record.push_back(A->isInherited());
+
+#include "clang/Serialization/AttrPCHWrite.inc"
+
+  }
+
+  Stream.EmitRecord(DECL_ATTR, Record);
+}
+
+void ASTWriter::AddString(llvm::StringRef Str, RecordData &Record) {
   Record.push_back(Str.size());
   Record.insert(Record.end(), Str.begin(), Str.end());
 }
 
 /// \brief Note that the identifier II occurs at the given offset
 /// within the identifier table.
-void PCHWriter::SetIdentifierOffset(const IdentifierInfo *II, uint32_t Offset) {
-  IdentifierOffsets[IdentifierIDs[II] - 1] = Offset;
+void ASTWriter::SetIdentifierOffset(const IdentifierInfo *II, uint32_t Offset) {
+  IdentID ID = IdentifierIDs[II];
+  // Only store offsets new to this AST file. Other identifier names are looked
+  // up earlier in the chain and thus don't need an offset.
+  if (ID >= FirstIdentID)
+    IdentifierOffsets[ID - FirstIdentID] = Offset;
 }
 
 /// \brief Note that the selector Sel occurs at the given offset
 /// within the method pool/selector table.
-void PCHWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
+void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
   unsigned ID = SelectorIDs[Sel];
   assert(ID && "Unknown selector");
-  SelectorOffsets[ID - 1] = Offset;
+  // Don't record offsets for selectors that are also available in a different
+  // file.
+  if (ID < FirstSelectorID)
+    return;
+  SelectorOffsets[ID - FirstSelectorID] = Offset;
 }
 
-PCHWriter::PCHWriter(llvm::BitstreamWriter &Stream, PCHReader *Chain)
-  : Stream(Stream), Chain(Chain), NextTypeID(pch::NUM_PREDEF_TYPE_IDS),
-    CollectedStmts(&StmtsToEmit), NumStatements(0), NumMacros(0),
-    NumLexicalDeclContexts(0), NumVisibleDeclContexts(0) {
-  if (Chain)
-    Chain->setDeserializationListener(this);
+ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream)
+  : Stream(Stream), Chain(0), FirstDeclID(1), NextDeclID(FirstDeclID),
+    FirstTypeID(NUM_PREDEF_TYPE_IDS), NextTypeID(FirstTypeID),
+    FirstIdentID(1), NextIdentID(FirstIdentID), FirstSelectorID(1),
+    NextSelectorID(FirstSelectorID), CollectedStmts(&StmtsToEmit),
+    NumStatements(0), NumMacros(0), NumLexicalDeclContexts(0),
+    NumVisibleDeclContexts(0) {
 }
 
-void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
+void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
                          const char *isysroot) {
   // Emit the file header.
   Stream.Emit((unsigned)'C', 8);
@@ -2092,12 +2212,12 @@ void PCHWriter::WritePCH(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   WriteBlockInfoBlock();
 
   if (Chain)
-    WritePCHChain(SemaRef, StatCalls, isysroot);
+    WriteASTChain(SemaRef, StatCalls, isysroot);
   else
-    WritePCHCore(SemaRef, StatCalls, isysroot);
+    WriteASTCore(SemaRef, StatCalls, isysroot);
 }
 
-void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
+void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
                              const char *isysroot) {
   using namespace llvm;
 
@@ -2106,6 +2226,7 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   // The translation unit is the first declaration we'll emit.
   DeclIDs[Context.getTranslationUnitDecl()] = 1;
+  ++NextDeclID;
   DeclTypesToEmit.push(Context.getTranslationUnitDecl());
 
   // Make sure that we emit IdentifierInfos (and any attached
@@ -2127,16 +2248,30 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
     AddDeclRef(SemaRef.TentativeDefinitions[i], TentativeDefinitions);
   }
 
-  // Build a record containing all of the static unused functions in this file.
-  RecordData UnusedStaticFuncs;
-  for (unsigned i=0, e = SemaRef.UnusedStaticFuncs.size(); i !=e; ++i)
-    AddDeclRef(SemaRef.UnusedStaticFuncs[i], UnusedStaticFuncs);
+  // Build a record containing all of the file scoped decls in this file.
+  RecordData UnusedFileScopedDecls;
+  for (unsigned i=0, e = SemaRef.UnusedFileScopedDecls.size(); i !=e; ++i)
+    AddDeclRef(SemaRef.UnusedFileScopedDecls[i], UnusedFileScopedDecls);
+
+  RecordData WeakUndeclaredIdentifiers;
+  if (!SemaRef.WeakUndeclaredIdentifiers.empty()) {
+    WeakUndeclaredIdentifiers.push_back(
+                                      SemaRef.WeakUndeclaredIdentifiers.size());
+    for (llvm::DenseMap<IdentifierInfo*,Sema::WeakInfo>::iterator
+         I = SemaRef.WeakUndeclaredIdentifiers.begin(),
+         E = SemaRef.WeakUndeclaredIdentifiers.end(); I != E; ++I) {
+      AddIdentifierRef(I->first, WeakUndeclaredIdentifiers);
+      AddIdentifierRef(I->second.getAlias(), WeakUndeclaredIdentifiers);
+      AddSourceLocation(I->second.getLocation(), WeakUndeclaredIdentifiers);
+      WeakUndeclaredIdentifiers.push_back(I->second.getUsed());
+    }
+  }
 
   // Build a record containing all of the locally-scoped external
   // declarations in this header file. Generally, this record will be
   // empty.
   RecordData LocallyScopedExternalDecls;
-  // FIXME: This is filling in the PCH file in densemap order which is
+  // FIXME: This is filling in the AST file in densemap order which is
   // nondeterminstic!
   for (llvm::DenseMap<DeclarationName, NamedDecl *>::iterator
          TD = SemaRef.LocallyScopedExternalDecls.begin(),
@@ -2151,11 +2286,13 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   // Build a record containing all of the VTable uses information.
   RecordData VTableUses;
-  VTableUses.push_back(SemaRef.VTableUses.size());
-  for (unsigned I = 0, N = SemaRef.VTableUses.size(); I != N; ++I) {
-    AddDeclRef(SemaRef.VTableUses[I].first, VTableUses);
-    AddSourceLocation(SemaRef.VTableUses[I].second, VTableUses);
-    VTableUses.push_back(SemaRef.VTablesUsed[SemaRef.VTableUses[I].first]);
+  if (!SemaRef.VTableUses.empty()) {
+    VTableUses.push_back(SemaRef.VTableUses.size());
+    for (unsigned I = 0, N = SemaRef.VTableUses.size(); I != N; ++I) {
+      AddDeclRef(SemaRef.VTableUses[I].first, VTableUses);
+      AddSourceLocation(SemaRef.VTableUses[I].second, VTableUses);
+      VTableUses.push_back(SemaRef.VTablesUsed[SemaRef.VTableUses[I].first]);
+    }
   }
 
   // Build a record containing all of dynamic classes declarations.
@@ -2163,9 +2300,27 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   for (unsigned I = 0, N = SemaRef.DynamicClasses.size(); I != N; ++I)
     AddDeclRef(SemaRef.DynamicClasses[I], DynamicClasses);
 
-  // Write the remaining PCH contents.
+  // Build a record containing all of pending implicit instantiations.
+  RecordData PendingInstantiations;
+  for (std::deque<Sema::PendingImplicitInstantiation>::iterator
+         I = SemaRef.PendingInstantiations.begin(),
+         N = SemaRef.PendingInstantiations.end(); I != N; ++I) {
+    AddDeclRef(I->first, PendingInstantiations);
+    AddSourceLocation(I->second, PendingInstantiations);
+  }
+  assert(SemaRef.PendingLocalImplicitInstantiations.empty() &&
+         "There are local ones at end of translation unit!");
+
+  // Build a record containing some declaration references.
+  RecordData SemaDeclRefs;
+  if (SemaRef.StdNamespace || SemaRef.StdBadAlloc) {
+    AddDeclRef(SemaRef.getStdNamespace(), SemaDeclRefs);
+    AddDeclRef(SemaRef.getStdBadAlloc(), SemaDeclRefs);
+  }
+
+  // Write the remaining AST contents.
   RecordData Record;
-  Stream.EnterSubblock(pch::PCH_BLOCK_ID, 5);
+  Stream.EnterSubblock(AST_BLOCK_ID, 5);
   WriteMetadata(Context, isysroot);
   WriteLanguageOptions(Context.getLangOptions());
   if (StatCalls && !isysroot)
@@ -2191,11 +2346,11 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   AddTypeRef(Context.ObjCSelRedefinitionType, Record);
   AddTypeRef(Context.getRawNSConstantStringType(), Record);
   Record.push_back(Context.isInt128Installed());
-  Stream.EmitRecord(pch::SPECIAL_TYPES, Record);
+  Stream.EmitRecord(SPECIAL_TYPES, Record);
 
   // Keep writing types and declarations until all types and
   // declarations have been written.
-  Stream.EnterSubblock(pch::DECLTYPES_BLOCK_ID, 3);
+  Stream.EnterSubblock(DECLTYPES_BLOCK_ID, 3);
   WriteDeclsBlockAbbrevs();
   while (!DeclTypesToEmit.empty()) {
     DeclOrType DOT = DeclTypesToEmit.front();
@@ -2208,63 +2363,53 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Stream.ExitBlock();
 
   WritePreprocessor(PP);
-  WriteMethodPool(SemaRef);
+  WriteSelectors(SemaRef);
+  WriteReferencedSelectorsPool(SemaRef);
   WriteIdentifierTable(PP);
 
-  // Write the type offsets array
-  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::TYPE_OFFSET));
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of types
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // types block
-  unsigned TypeOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
-  Record.clear();
-  Record.push_back(pch::TYPE_OFFSET);
-  Record.push_back(TypeOffsets.size());
-  Stream.EmitRecordWithBlob(TypeOffsetAbbrev, Record,
-                            (const char *)&TypeOffsets.front(),
-                            TypeOffsets.size() * sizeof(TypeOffsets[0]));
-
-  // Write the declaration offsets array
-  Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(pch::DECL_OFFSET));
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of declarations
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // declarations block
-  unsigned DeclOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
-  Record.clear();
-  Record.push_back(pch::DECL_OFFSET);
-  Record.push_back(DeclOffsets.size());
-  Stream.EmitRecordWithBlob(DeclOffsetAbbrev, Record,
-                            (const char *)&DeclOffsets.front(),
-                            DeclOffsets.size() * sizeof(DeclOffsets[0]));
+  WriteTypeDeclOffsets();
 
   // Write the record containing external, unnamed definitions.
   if (!ExternalDefinitions.empty())
-    Stream.EmitRecord(pch::EXTERNAL_DEFINITIONS, ExternalDefinitions);
+    Stream.EmitRecord(EXTERNAL_DEFINITIONS, ExternalDefinitions);
 
   // Write the record containing tentative definitions.
   if (!TentativeDefinitions.empty())
-    Stream.EmitRecord(pch::TENTATIVE_DEFINITIONS, TentativeDefinitions);
+    Stream.EmitRecord(TENTATIVE_DEFINITIONS, TentativeDefinitions);
+
+  // Write the record containing unused file scoped decls.
+  if (!UnusedFileScopedDecls.empty())
+    Stream.EmitRecord(UNUSED_FILESCOPED_DECLS, UnusedFileScopedDecls);
 
-  // Write the record containing unused static functions.
-  if (!UnusedStaticFuncs.empty())
-    Stream.EmitRecord(pch::UNUSED_STATIC_FUNCS, UnusedStaticFuncs);
+  // Write the record containing weak undeclared identifiers.
+  if (!WeakUndeclaredIdentifiers.empty())
+    Stream.EmitRecord(WEAK_UNDECLARED_IDENTIFIERS,
+                      WeakUndeclaredIdentifiers);
 
   // Write the record containing locally-scoped external definitions.
   if (!LocallyScopedExternalDecls.empty())
-    Stream.EmitRecord(pch::LOCALLY_SCOPED_EXTERNAL_DECLS,
+    Stream.EmitRecord(LOCALLY_SCOPED_EXTERNAL_DECLS,
                       LocallyScopedExternalDecls);
 
   // Write the record containing ext_vector type names.
   if (!ExtVectorDecls.empty())
-    Stream.EmitRecord(pch::EXT_VECTOR_DECLS, ExtVectorDecls);
+    Stream.EmitRecord(EXT_VECTOR_DECLS, ExtVectorDecls);
 
   // Write the record containing VTable uses information.
   if (!VTableUses.empty())
-    Stream.EmitRecord(pch::VTABLE_USES, VTableUses);
+    Stream.EmitRecord(VTABLE_USES, VTableUses);
 
   // Write the record containing dynamic classes declarations.
   if (!DynamicClasses.empty())
-    Stream.EmitRecord(pch::DYNAMIC_CLASSES, DynamicClasses);
+    Stream.EmitRecord(DYNAMIC_CLASSES, DynamicClasses);
+
+  // Write the record containing pending implicit instantiations.
+  if (!PendingInstantiations.empty())
+    Stream.EmitRecord(PENDING_IMPLICIT_INSTANTIATIONS, PendingInstantiations);
+
+  // Write the record containing declaration references of Sema.
+  if (!SemaDeclRefs.empty())
+    Stream.EmitRecord(SEMA_DECL_REFS, SemaDeclRefs);
 
   // Some simple statistics
   Record.clear();
@@ -2272,42 +2417,162 @@ void PCHWriter::WritePCHCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Record.push_back(NumMacros);
   Record.push_back(NumLexicalDeclContexts);
   Record.push_back(NumVisibleDeclContexts);
-  Stream.EmitRecord(pch::STATISTICS, Record);
+  Stream.EmitRecord(STATISTICS, Record);
   Stream.ExitBlock();
 }
 
-void PCHWriter::WritePCHChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
+void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
                               const char *isysroot) {
   using namespace llvm;
 
+  FirstDeclID += Chain->getTotalNumDecls();
+  FirstTypeID += Chain->getTotalNumTypes();
+  FirstIdentID += Chain->getTotalNumIdentifiers();
+  FirstSelectorID += Chain->getTotalNumSelectors();
+  NextDeclID = FirstDeclID;
+  NextTypeID = FirstTypeID;
+  NextIdentID = FirstIdentID;
+  NextSelectorID = FirstSelectorID;
+
   ASTContext &Context = SemaRef.Context;
   Preprocessor &PP = SemaRef.PP;
-  (void)PP;
-  
+
   RecordData Record;
-  Stream.EnterSubblock(pch::PCH_BLOCK_ID, 5);
+  Stream.EnterSubblock(AST_BLOCK_ID, 5);
   WriteMetadata(Context, isysroot);
-  // FIXME: StatCache
-  // FIXME: Source manager block
+  if (StatCalls && !isysroot)
+    WriteStatCache(*StatCalls);
+  // FIXME: Source manager block should only write new stuff, which could be
+  // done by tracking the largest ID in the chain
+  WriteSourceManagerBlock(Context.getSourceManager(), PP, isysroot);
 
   // The special types are in the chained PCH.
 
   // We don't start with the translation unit, but with its decls that
-  // don't come from the other PCH.
+  // don't come from the chained PCH.
   const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
-  // FIXME: We don't want to iterate over everything here, because it needlessly
-  // deserializes the entire original PCH. Instead we only want to iterate over
-  // the stuff that's already there.
-  // All in good time, though.
-  for (DeclContext::decl_iterator I = TU->decls_begin(), E = TU->decls_end();
+  llvm::SmallVector<DeclID, 64> NewGlobalDecls;
+  for (DeclContext::decl_iterator I = TU->noload_decls_begin(),
+                                  E = TU->noload_decls_end();
        I != E; ++I) {
-    if ((*I)->getPCHLevel() == 0) {
-      (*I)->dump();
-      DeclTypesToEmit.push(*I);
+    if ((*I)->getPCHLevel() == 0)
+      NewGlobalDecls.push_back(GetDeclRef(*I));
+    else if ((*I)->isChangedSinceDeserialization())
+      (void)GetDeclRef(*I); // Make sure it's written, but don't record it.
+  }
+  // We also need to write a lexical updates block for the TU.
+  llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev();
+  Abv->Add(llvm::BitCodeAbbrevOp(TU_UPDATE_LEXICAL));
+  Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+  unsigned TuUpdateLexicalAbbrev = Stream.EmitAbbrev(Abv);
+  Record.clear();
+  Record.push_back(TU_UPDATE_LEXICAL);
+  Stream.EmitRecordWithBlob(TuUpdateLexicalAbbrev, Record,
+                          reinterpret_cast<const char*>(NewGlobalDecls.data()),
+                          NewGlobalDecls.size() * sizeof(DeclID));
+  // And in C++, a visible updates block for the TU.
+  if (Context.getLangOptions().CPlusPlus) {
+    Abv = new llvm::BitCodeAbbrev();
+    Abv->Add(llvm::BitCodeAbbrevOp(UPDATE_VISIBLE));
+    Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
+    Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Fixed, 32));
+    Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+    UpdateVisibleAbbrev = Stream.EmitAbbrev(Abv);
+    WriteDeclContextVisibleUpdate(TU);
+  }
+
+  // Build a record containing all of the new tentative definitions in this
+  // file, in TentativeDefinitions order.
+  RecordData TentativeDefinitions;
+  for (unsigned i = 0, e = SemaRef.TentativeDefinitions.size(); i != e; ++i) {
+    if (SemaRef.TentativeDefinitions[i]->getPCHLevel() == 0)
+      AddDeclRef(SemaRef.TentativeDefinitions[i], TentativeDefinitions);
+  }
+
+  // Build a record containing all of the file scoped decls in this file.
+  RecordData UnusedFileScopedDecls;
+  for (unsigned i=0, e = SemaRef.UnusedFileScopedDecls.size(); i !=e; ++i) {
+    if (SemaRef.UnusedFileScopedDecls[i]->getPCHLevel() == 0)
+      AddDeclRef(SemaRef.UnusedFileScopedDecls[i], UnusedFileScopedDecls);
+  }
+
+  // We write the entire table, overwriting the tables from the chain.
+  RecordData WeakUndeclaredIdentifiers;
+  if (!SemaRef.WeakUndeclaredIdentifiers.empty()) {
+    WeakUndeclaredIdentifiers.push_back(
+                                      SemaRef.WeakUndeclaredIdentifiers.size());
+    for (llvm::DenseMap<IdentifierInfo*,Sema::WeakInfo>::iterator
+         I = SemaRef.WeakUndeclaredIdentifiers.begin(),
+         E = SemaRef.WeakUndeclaredIdentifiers.end(); I != E; ++I) {
+      AddIdentifierRef(I->first, WeakUndeclaredIdentifiers);
+      AddIdentifierRef(I->second.getAlias(), WeakUndeclaredIdentifiers);
+      AddSourceLocation(I->second.getLocation(), WeakUndeclaredIdentifiers);
+      WeakUndeclaredIdentifiers.push_back(I->second.getUsed());
+    }
+  }
+
+  // Build a record containing all of the locally-scoped external
+  // declarations in this header file. Generally, this record will be
+  // empty.
+  RecordData LocallyScopedExternalDecls;
+  // FIXME: This is filling in the AST file in densemap order which is
+  // nondeterminstic!
+  for (llvm::DenseMap<DeclarationName, NamedDecl *>::iterator
+         TD = SemaRef.LocallyScopedExternalDecls.begin(),
+         TDEnd = SemaRef.LocallyScopedExternalDecls.end();
+       TD != TDEnd; ++TD) {
+    if (TD->second->getPCHLevel() == 0)
+      AddDeclRef(TD->second, LocallyScopedExternalDecls);
+  }
+
+  // Build a record containing all of the ext_vector declarations.
+  RecordData ExtVectorDecls;
+  for (unsigned I = 0, N = SemaRef.ExtVectorDecls.size(); I != N; ++I) {
+    if (SemaRef.ExtVectorDecls[I]->getPCHLevel() == 0)
+      AddDeclRef(SemaRef.ExtVectorDecls[I], ExtVectorDecls);
+  }
+
+  // Build a record containing all of the VTable uses information.
+  // We write everything here, because it's too hard to determine whether
+  // a use is new to this part.
+  RecordData VTableUses;
+  if (!SemaRef.VTableUses.empty()) {
+    VTableUses.push_back(SemaRef.VTableUses.size());
+    for (unsigned I = 0, N = SemaRef.VTableUses.size(); I != N; ++I) {
+      AddDeclRef(SemaRef.VTableUses[I].first, VTableUses);
+      AddSourceLocation(SemaRef.VTableUses[I].second, VTableUses);
+      VTableUses.push_back(SemaRef.VTablesUsed[SemaRef.VTableUses[I].first]);
+    }
+  }
+
+  // Build a record containing all of dynamic classes declarations.
+  RecordData DynamicClasses;
+  for (unsigned I = 0, N = SemaRef.DynamicClasses.size(); I != N; ++I)
+    if (SemaRef.DynamicClasses[I]->getPCHLevel() == 0)
+      AddDeclRef(SemaRef.DynamicClasses[I], DynamicClasses);
+
+  // Build a record containing all of pending implicit instantiations.
+  RecordData PendingInstantiations;
+  for (std::deque<Sema::PendingImplicitInstantiation>::iterator
+         I = SemaRef.PendingInstantiations.begin(),
+         N = SemaRef.PendingInstantiations.end(); I != N; ++I) {
+    if (I->first->getPCHLevel() == 0) {
+      AddDeclRef(I->first, PendingInstantiations);
+      AddSourceLocation(I->second, PendingInstantiations);
     }
   }
+  assert(SemaRef.PendingLocalImplicitInstantiations.empty() &&
+         "There are local ones at end of translation unit!");
+
+  // Build a record containing some declaration references.
+  // It's not worth the effort to avoid duplication here.
+  RecordData SemaDeclRefs;
+  if (SemaRef.StdNamespace || SemaRef.StdBadAlloc) {
+    AddDeclRef(SemaRef.getStdNamespace(), SemaDeclRefs);
+    AddDeclRef(SemaRef.getStdBadAlloc(), SemaDeclRefs);
+  }
 
-  Stream.EnterSubblock(pch::DECLTYPES_BLOCK_ID, 3);
+  Stream.EnterSubblock(DECLTYPES_BLOCK_ID, 3);
   WriteDeclsBlockAbbrevs();
   while (!DeclTypesToEmit.empty()) {
     DeclOrType DOT = DeclTypesToEmit.front();
@@ -2319,31 +2584,111 @@ void PCHWriter::WritePCHChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   }
   Stream.ExitBlock();
 
-  // FIXME: Preprocessor
-  // FIXME: Method pool
-  // FIXME: Identifier table
-  // FIXME: Type offsets
-  // FIXME: Declaration offsets
-  // FIXME: External unnamed definitions
-  // FIXME: Tentative definitions
-  // FIXME: Unused static functions
-  // FIXME: Locally-scoped external definitions
-  // FIXME: ext_vector type names
-  // FIXME: Dynamic classes declarations
-  // FIXME: Statistics
+  WritePreprocessor(PP);
+  WriteSelectors(SemaRef);
+  WriteReferencedSelectorsPool(SemaRef);
+  WriteIdentifierTable(PP);
+  WriteTypeDeclOffsets();
+
+  /// Build a record containing first declarations from a chained PCH and the
+  /// most recent declarations in this AST that they point to.
+  RecordData FirstLatestDeclIDs;
+  for (FirstLatestDeclMap::iterator
+        I = FirstLatestDecls.begin(), E = FirstLatestDecls.end(); I != E; ++I) {
+    assert(I->first->getPCHLevel() > I->second->getPCHLevel() &&
+           "Expected first & second to be in different PCHs");
+    AddDeclRef(I->first, FirstLatestDeclIDs);
+    AddDeclRef(I->second, FirstLatestDeclIDs);
+  }
+  if (!FirstLatestDeclIDs.empty())
+    Stream.EmitRecord(REDECLS_UPDATE_LATEST, FirstLatestDeclIDs);
+
+  // Write the record containing external, unnamed definitions.
+  if (!ExternalDefinitions.empty())
+    Stream.EmitRecord(EXTERNAL_DEFINITIONS, ExternalDefinitions);
+
+  // Write the record containing tentative definitions.
+  if (!TentativeDefinitions.empty())
+    Stream.EmitRecord(TENTATIVE_DEFINITIONS, TentativeDefinitions);
+
+  // Write the record containing unused file scoped decls.
+  if (!UnusedFileScopedDecls.empty())
+    Stream.EmitRecord(UNUSED_FILESCOPED_DECLS, UnusedFileScopedDecls);
+
+  // Write the record containing weak undeclared identifiers.
+  if (!WeakUndeclaredIdentifiers.empty())
+    Stream.EmitRecord(WEAK_UNDECLARED_IDENTIFIERS,
+                      WeakUndeclaredIdentifiers);
+
+  // Write the record containing locally-scoped external definitions.
+  if (!LocallyScopedExternalDecls.empty())
+    Stream.EmitRecord(LOCALLY_SCOPED_EXTERNAL_DECLS,
+                      LocallyScopedExternalDecls);
+
+  // Write the record containing ext_vector type names.
+  if (!ExtVectorDecls.empty())
+    Stream.EmitRecord(EXT_VECTOR_DECLS, ExtVectorDecls);
+
+  // Write the record containing VTable uses information.
+  if (!VTableUses.empty())
+    Stream.EmitRecord(VTABLE_USES, VTableUses);
+
+  // Write the record containing dynamic classes declarations.
+  if (!DynamicClasses.empty())
+    Stream.EmitRecord(DYNAMIC_CLASSES, DynamicClasses);
+
+  // Write the record containing pending implicit instantiations.
+  if (!PendingInstantiations.empty())
+    Stream.EmitRecord(PENDING_IMPLICIT_INSTANTIATIONS, PendingInstantiations);
+
+  // Write the record containing declaration references of Sema.
+  if (!SemaDeclRefs.empty())
+    Stream.EmitRecord(SEMA_DECL_REFS, SemaDeclRefs);
+
+  // Write the updates to C++ namespaces.
+  for (llvm::SmallPtrSet<const NamespaceDecl *, 16>::iterator
+           I = UpdatedNamespaces.begin(),
+           E = UpdatedNamespaces.end();
+         I != E; ++I)
+    WriteDeclContextVisibleUpdate(*I);
+
+  // Write the updates to C++ template specialization lists.
+  if (!AdditionalTemplateSpecializations.empty())
+    WriteAdditionalTemplateSpecializations();
+
+  Record.clear();
+  Record.push_back(NumStatements);
+  Record.push_back(NumMacros);
+  Record.push_back(NumLexicalDeclContexts);
+  Record.push_back(NumVisibleDeclContexts);
+  WriteDeclUpdateBlock();
+  Stream.EmitRecord(STATISTICS, Record);
   Stream.ExitBlock();
 }
 
-void PCHWriter::AddSourceLocation(SourceLocation Loc, RecordData &Record) {
+void ASTWriter::WriteDeclUpdateBlock() {
+  if (ReplacedDecls.empty())
+    return;
+
+  RecordData Record;
+  for (llvm::SmallVector<std::pair<DeclID, uint64_t>, 16>::iterator
+           I = ReplacedDecls.begin(), E = ReplacedDecls.end(); I != E; ++I) {
+    Record.push_back(I->first);
+    Record.push_back(I->second);
+  }
+  Stream.EmitRecord(DECL_REPLACEMENTS, Record);
+}
+
+void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordData &Record) {
   Record.push_back(Loc.getRawEncoding());
 }
 
-void PCHWriter::AddSourceRange(SourceRange Range, RecordData &Record) {
+void ASTWriter::AddSourceRange(SourceRange Range, RecordData &Record) {
   AddSourceLocation(Range.getBegin(), Record);
   AddSourceLocation(Range.getEnd(), Record);
 }
 
-void PCHWriter::AddAPInt(const llvm::APInt &Value, RecordData &Record) {
+void ASTWriter::AddAPInt(const llvm::APInt &Value, RecordData &Record) {
   Record.push_back(Value.getBitWidth());
   unsigned N = Value.getNumWords();
   const uint64_t* Words = Value.getRawData();
@@ -2351,58 +2696,65 @@ void PCHWriter::AddAPInt(const llvm::APInt &Value, RecordData &Record) {
     Record.push_back(Words[I]);
 }
 
-void PCHWriter::AddAPSInt(const llvm::APSInt &Value, RecordData &Record) {
+void ASTWriter::AddAPSInt(const llvm::APSInt &Value, RecordData &Record) {
   Record.push_back(Value.isUnsigned());
   AddAPInt(Value, Record);
 }
 
-void PCHWriter::AddAPFloat(const llvm::APFloat &Value, RecordData &Record) {
+void ASTWriter::AddAPFloat(const llvm::APFloat &Value, RecordData &Record) {
   AddAPInt(Value.bitcastToAPInt(), Record);
 }
 
-void PCHWriter::AddIdentifierRef(const IdentifierInfo *II, RecordData &Record) {
+void ASTWriter::AddIdentifierRef(const IdentifierInfo *II, RecordData &Record) {
   Record.push_back(getIdentifierRef(II));
 }
 
-pch::IdentID PCHWriter::getIdentifierRef(const IdentifierInfo *II) {
+IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   if (II == 0)
     return 0;
 
-  pch::IdentID &ID = IdentifierIDs[II];
+  IdentID &ID = IdentifierIDs[II];
   if (ID == 0)
-    ID = IdentifierIDs.size();
+    ID = NextIdentID++;
   return ID;
 }
 
-pch::IdentID PCHWriter::getMacroDefinitionID(MacroDefinition *MD) {
+IdentID ASTWriter::getMacroDefinitionID(MacroDefinition *MD) {
   if (MD == 0)
     return 0;
   
-  pch::IdentID &ID = MacroDefinitions[MD];
+  IdentID &ID = MacroDefinitions[MD];
   if (ID == 0)
     ID = MacroDefinitions.size();
   return ID;
 }
 
-void PCHWriter::AddSelectorRef(const Selector SelRef, RecordData &Record) {
-  if (SelRef.getAsOpaquePtr() == 0) {
-    Record.push_back(0);
-    return;
+void ASTWriter::AddSelectorRef(const Selector SelRef, RecordData &Record) {
+  Record.push_back(getSelectorRef(SelRef));
+}
+
+SelectorID ASTWriter::getSelectorRef(Selector Sel) {
+  if (Sel.getAsOpaquePtr() == 0) {
+    return 0;
   }
 
-  pch::SelectorID &SID = SelectorIDs[SelRef];
+  SelectorID &SID = SelectorIDs[Sel];
+  if (SID == 0 && Chain) {
+    // This might trigger a ReadSelector callback, which will set the ID for
+    // this selector.
+    Chain->LoadSelector(Sel);
+  }
   if (SID == 0) {
-    SID = SelectorIDs.size();
-    SelVector.push_back(SelRef);
+    SID = NextSelectorID++;
   }
-  Record.push_back(SID);
+  return SID;
 }
 
-void PCHWriter::AddCXXTemporary(const CXXTemporary *Temp, RecordData &Record) {
+void ASTWriter::AddCXXTemporary(const CXXTemporary *Temp, RecordData &Record) {
   AddDeclRef(Temp->getDestructor(), Record);
 }
 
-void PCHWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
+void ASTWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
                                            const TemplateArgumentLocInfo &Arg,
                                            RecordData &Record) {
   switch (Kind) {
@@ -2424,7 +2776,7 @@ void PCHWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
   }
 }
 
-void PCHWriter::AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg,
+void ASTWriter::AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg,
                                        RecordData &Record) {
   AddTemplateArgument(Arg.getArgument(), Record);
 
@@ -2439,7 +2791,7 @@ void PCHWriter::AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg,
                              Record);
 }
 
-void PCHWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo, RecordData &Record) {
+void ASTWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo, RecordData &Record) {
   if (TInfo == 0) {
     AddTypeRef(QualType(), Record);
     return;
@@ -2451,102 +2803,72 @@ void PCHWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo, RecordData &Record) {
     TLW.Visit(TL);
 }
 
-void PCHWriter::AddTypeRef(QualType T, RecordData &Record) {
-  if (T.isNull()) {
-    Record.push_back(pch::PREDEF_TYPE_NULL_ID);
-    return;
-  }
-
-  unsigned FastQuals = T.getLocalFastQualifiers();
-  T.removeFastQualifiers();
-
-  if (T.hasLocalNonFastQualifiers()) {
-    pch::TypeID &ID = TypeIDs[T];
-    if (ID == 0) {
-      // We haven't seen these qualifiers applied to this type before.
-      // Assign it a new ID.  This is the only time we enqueue a
-      // qualified type, and it has no CV qualifiers.
-      ID = NextTypeID++;
-      DeclTypesToEmit.push(T);
-    }
+void ASTWriter::AddTypeRef(QualType T, RecordData &Record) {
+  Record.push_back(GetOrCreateTypeID(T));
+}
 
-    // Encode the type qualifiers in the type reference.
-    Record.push_back((ID << Qualifiers::FastWidth) | FastQuals);
-    return;
-  }
+TypeID ASTWriter::GetOrCreateTypeID(QualType T) {
+  return MakeTypeID(T,
+              std::bind1st(std::mem_fun(&ASTWriter::GetOrCreateTypeIdx), this));
+}
 
-  assert(!T.hasLocalQualifiers());
-
-  if (const BuiltinType *BT = dyn_cast<BuiltinType>(T.getTypePtr())) {
-    pch::TypeID ID = 0;
-    switch (BT->getKind()) {
-    case BuiltinType::Void:       ID = pch::PREDEF_TYPE_VOID_ID;       break;
-    case BuiltinType::Bool:       ID = pch::PREDEF_TYPE_BOOL_ID;       break;
-    case BuiltinType::Char_U:     ID = pch::PREDEF_TYPE_CHAR_U_ID;     break;
-    case BuiltinType::UChar:      ID = pch::PREDEF_TYPE_UCHAR_ID;      break;
-    case BuiltinType::UShort:     ID = pch::PREDEF_TYPE_USHORT_ID;     break;
-    case BuiltinType::UInt:       ID = pch::PREDEF_TYPE_UINT_ID;       break;
-    case BuiltinType::ULong:      ID = pch::PREDEF_TYPE_ULONG_ID;      break;
-    case BuiltinType::ULongLong:  ID = pch::PREDEF_TYPE_ULONGLONG_ID;  break;
-    case BuiltinType::UInt128:    ID = pch::PREDEF_TYPE_UINT128_ID;    break;
-    case BuiltinType::Char_S:     ID = pch::PREDEF_TYPE_CHAR_S_ID;     break;
-    case BuiltinType::SChar:      ID = pch::PREDEF_TYPE_SCHAR_ID;      break;
-    case BuiltinType::WChar:      ID = pch::PREDEF_TYPE_WCHAR_ID;      break;
-    case BuiltinType::Short:      ID = pch::PREDEF_TYPE_SHORT_ID;      break;
-    case BuiltinType::Int:        ID = pch::PREDEF_TYPE_INT_ID;        break;
-    case BuiltinType::Long:       ID = pch::PREDEF_TYPE_LONG_ID;       break;
-    case BuiltinType::LongLong:   ID = pch::PREDEF_TYPE_LONGLONG_ID;   break;
-    case BuiltinType::Int128:     ID = pch::PREDEF_TYPE_INT128_ID;     break;
-    case BuiltinType::Float:      ID = pch::PREDEF_TYPE_FLOAT_ID;      break;
-    case BuiltinType::Double:     ID = pch::PREDEF_TYPE_DOUBLE_ID;     break;
-    case BuiltinType::LongDouble: ID = pch::PREDEF_TYPE_LONGDOUBLE_ID; break;
-    case BuiltinType::NullPtr:    ID = pch::PREDEF_TYPE_NULLPTR_ID;    break;
-    case BuiltinType::Char16:     ID = pch::PREDEF_TYPE_CHAR16_ID;     break;
-    case BuiltinType::Char32:     ID = pch::PREDEF_TYPE_CHAR32_ID;     break;
-    case BuiltinType::Overload:   ID = pch::PREDEF_TYPE_OVERLOAD_ID;   break;
-    case BuiltinType::Dependent:  ID = pch::PREDEF_TYPE_DEPENDENT_ID;  break;
-    case BuiltinType::ObjCId:     ID = pch::PREDEF_TYPE_OBJC_ID;       break;
-    case BuiltinType::ObjCClass:  ID = pch::PREDEF_TYPE_OBJC_CLASS;    break;
-    case BuiltinType::ObjCSel:    ID = pch::PREDEF_TYPE_OBJC_SEL;      break;
-    case BuiltinType::UndeducedAuto:
-      assert(0 && "Should not see undeduced auto here");
-      break;
-    }
+TypeID ASTWriter::getTypeID(QualType T) const {
+  return MakeTypeID(T,
+              std::bind1st(std::mem_fun(&ASTWriter::getTypeIdx), this));
+}
 
-    Record.push_back((ID << Qualifiers::FastWidth) | FastQuals);
-    return;
-  }
+TypeIdx ASTWriter::GetOrCreateTypeIdx(QualType T) {
+  if (T.isNull())
+    return TypeIdx();
+  assert(!T.getLocalFastQualifiers());
 
-  pch::TypeID &ID = TypeIDs[T];
-  if (ID == 0) {
+  TypeIdx &Idx = TypeIdxs[T];
+  if (Idx.getIndex() == 0) {
     // We haven't seen this type before. Assign it a new ID and put it
     // into the queue of types to emit.
-    ID = NextTypeID++;
+    Idx = TypeIdx(NextTypeID++);
     DeclTypesToEmit.push(T);
   }
+  return Idx;
+}
+
+TypeIdx ASTWriter::getTypeIdx(QualType T) const {
+  if (T.isNull())
+    return TypeIdx();
+  assert(!T.getLocalFastQualifiers());
 
-  // Encode the type qualifiers in the type reference.
-  Record.push_back((ID << Qualifiers::FastWidth) | FastQuals);
+  TypeIdxMap::const_iterator I = TypeIdxs.find(T);
+  assert(I != TypeIdxs.end() && "Type not emitted!");
+  return I->second;
 }
 
-void PCHWriter::AddDeclRef(const Decl *D, RecordData &Record) {
+void ASTWriter::AddDeclRef(const Decl *D, RecordData &Record) {
+  Record.push_back(GetDeclRef(D));
+}
+
+DeclID ASTWriter::GetDeclRef(const Decl *D) {
   if (D == 0) {
-    Record.push_back(0);
-    return;
+    return 0;
   }
 
-  pch::DeclID &ID = DeclIDs[D];
+  DeclID &ID = DeclIDs[D];
   if (ID == 0) {
     // We haven't seen this declaration before. Give it a new ID and
     // enqueue it in the list of declarations to emit.
-    ID = DeclIDs.size();
+    ID = NextDeclID++;
+    DeclTypesToEmit.push(const_cast<Decl *>(D));
+  } else if (ID < FirstDeclID && D->isChangedSinceDeserialization()) {
+    // We don't add it to the replacement collection here, because we don't
+    // have the offset yet.
     DeclTypesToEmit.push(const_cast<Decl *>(D));
+    // Reset the flag, so that we don't add this decl multiple times.
+    const_cast<Decl *>(D)->setChangedSinceDeserialization(false);
   }
 
-  Record.push_back(ID);
+  return ID;
 }
 
-pch::DeclID PCHWriter::getDeclID(const Decl *D) {
+DeclID ASTWriter::getDeclID(const Decl *D) {
   if (D == 0)
     return 0;
 
@@ -2554,7 +2876,7 @@ pch::DeclID PCHWriter::getDeclID(const Decl *D) {
   return DeclIDs[D];
 }
 
-void PCHWriter::AddDeclarationName(DeclarationName Name, RecordData &Record) {
+void ASTWriter::AddDeclarationName(DeclarationName Name, RecordData &Record) {
   // FIXME: Emit a stable enum for NameKind.  0 = Identifier etc.
   Record.push_back(Name.getNameKind());
   switch (Name.getNameKind()) {
@@ -2588,7 +2910,7 @@ void PCHWriter::AddDeclarationName(DeclarationName Name, RecordData &Record) {
   }
 }
 
-void PCHWriter::AddNestedNameSpecifier(NestedNameSpecifier *NNS,
+void ASTWriter::AddNestedNameSpecifier(NestedNameSpecifier *NNS,
                                        RecordData &Record) {
   // Nested name specifiers usually aren't too long. I think that 8 would
   // typically accomodate the vast majority.
@@ -2627,7 +2949,7 @@ void PCHWriter::AddNestedNameSpecifier(NestedNameSpecifier *NNS,
   }
 }
 
-void PCHWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
+void ASTWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
   TemplateName::NameKind Kind = Name.getKind(); 
   Record.push_back(Kind);
   switch (Kind) {
@@ -2665,7 +2987,7 @@ void PCHWriter::AddTemplateName(TemplateName Name, RecordData &Record) {
   }
 }
 
-void PCHWriter::AddTemplateArgument(const TemplateArgument &Arg, 
+void ASTWriter::AddTemplateArgument(const TemplateArgument &Arg, 
                                     RecordData &Record) {
   Record.push_back(Arg.getKind());
   switch (Arg.getKind()) {
@@ -2697,7 +3019,7 @@ void PCHWriter::AddTemplateArgument(const TemplateArgument &Arg,
 }
 
 void
-PCHWriter::AddTemplateParameterList(const TemplateParameterList *TemplateParams,
+ASTWriter::AddTemplateParameterList(const TemplateParameterList *TemplateParams,
                                     RecordData &Record) {
   assert(TemplateParams && "No TemplateParams!");
   AddSourceLocation(TemplateParams->getTemplateLoc(), Record);
@@ -2712,7 +3034,7 @@ PCHWriter::AddTemplateParameterList(const TemplateParameterList *TemplateParams,
 
 /// \brief Emit a template argument list.
 void
-PCHWriter::AddTemplateArgumentList(const TemplateArgumentList *TemplateArgs,
+ASTWriter::AddTemplateArgumentList(const TemplateArgumentList *TemplateArgs,
                                    RecordData &Record) {
   assert(TemplateArgs && "No TemplateArgs!");
   Record.push_back(TemplateArgs->flat_size());
@@ -2722,7 +3044,7 @@ PCHWriter::AddTemplateArgumentList(const TemplateArgumentList *TemplateArgs,
 
 
 void
-PCHWriter::AddUnresolvedSet(const UnresolvedSetImpl &Set, RecordData &Record) {
+ASTWriter::AddUnresolvedSet(const UnresolvedSetImpl &Set, RecordData &Record) {
   Record.push_back(Set.size());
   for (UnresolvedSetImpl::const_iterator
          I = Set.begin(), E = Set.end(); I != E; ++I) {
@@ -2731,18 +3053,67 @@ PCHWriter::AddUnresolvedSet(const UnresolvedSetImpl &Set, RecordData &Record) {
   }
 }
 
-void PCHWriter::AddCXXBaseSpecifier(const CXXBaseSpecifier &Base,
+void ASTWriter::AddCXXBaseSpecifier(const CXXBaseSpecifier &Base,
                                     RecordData &Record) {
   Record.push_back(Base.isVirtual());
   Record.push_back(Base.isBaseOfClass());
   Record.push_back(Base.getAccessSpecifierAsWritten());
-  AddTypeRef(Base.getType(), Record);
+  AddTypeSourceInfo(Base.getTypeSourceInfo(), Record);
   AddSourceRange(Base.getSourceRange(), Record);
 }
 
-void PCHWriter::TypeRead(pch::TypeID ID, QualType T) {
+void ASTWriter::AddCXXBaseOrMemberInitializers(
+                        const CXXBaseOrMemberInitializer * const *BaseOrMembers,
+                        unsigned NumBaseOrMembers, RecordData &Record) {
+  Record.push_back(NumBaseOrMembers);
+  for (unsigned i=0; i != NumBaseOrMembers; ++i) {
+    const CXXBaseOrMemberInitializer *Init = BaseOrMembers[i];
+
+    Record.push_back(Init->isBaseInitializer());
+    if (Init->isBaseInitializer()) {
+      AddTypeSourceInfo(Init->getBaseClassInfo(), Record);
+      Record.push_back(Init->isBaseVirtual());
+    } else {
+      AddDeclRef(Init->getMember(), Record);
+    }
+    AddSourceLocation(Init->getMemberLocation(), Record);
+    AddStmt(Init->getInit());
+    AddDeclRef(Init->getAnonUnionMember(), Record);
+    AddSourceLocation(Init->getLParenLoc(), Record);
+    AddSourceLocation(Init->getRParenLoc(), Record);
+    Record.push_back(Init->isWritten());
+    if (Init->isWritten()) {
+      Record.push_back(Init->getSourceOrder());
+    } else {
+      Record.push_back(Init->getNumArrayIndices());
+      for (unsigned i=0, e=Init->getNumArrayIndices(); i != e; ++i)
+        AddDeclRef(Init->getArrayIndex(i), Record);
+    }
+  }
+}
+
+void ASTWriter::SetReader(ASTReader *Reader) {
+  assert(Reader && "Cannot remove chain");
+  assert(FirstDeclID == NextDeclID &&
+         FirstTypeID == NextTypeID &&
+         FirstIdentID == NextIdentID &&
+         FirstSelectorID == NextSelectorID &&
+         "Setting chain after writing has started.");
+  Chain = Reader;
 }
 
-void PCHWriter::DeclRead(pch::DeclID ID, const Decl *D) {
+void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) {
+  IdentifierIDs[II] = ID;
 }
 
+void ASTWriter::TypeRead(TypeIdx Idx, QualType T) {
+  TypeIdxs[T] = Idx;
+}
+
+void ASTWriter::DeclRead(DeclID ID, const Decl *D) {
+  DeclIDs[D] = ID;
+}
+
+void ASTWriter::SelectorRead(SelectorID ID, Selector S) {
+  SelectorIDs[S] = ID;
+}
diff --git a/lib/Frontend/PCHWriterDecl.cpp b/lib/Serialization/ASTWriterDecl.cpp
index bc4452ed7f4d..ce39a1076b8e 100644
--- a/lib/Frontend/PCHWriterDecl.cpp
+++ b/lib/Serialization/ASTWriterDecl.cpp
@@ -1,4 +1,4 @@
-//===--- PCHWriterDecl.cpp - Declaration Serialization --------------------===//
+//===--- ASTWriterDecl.cpp - Declaration Serialization --------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/PCHWriter.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
@@ -26,18 +26,18 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 namespace clang {
-  class PCHDeclWriter : public DeclVisitor<PCHDeclWriter, void> {
+  class ASTDeclWriter : public DeclVisitor<ASTDeclWriter, void> {
 
-    PCHWriter &Writer;
+    ASTWriter &Writer;
     ASTContext &Context;
-    PCHWriter::RecordData &Record;
+    ASTWriter::RecordData &Record;
 
   public:
-    pch::DeclCode Code;
+    serialization::DeclCode Code;
     unsigned AbbrevToUse;
 
-    PCHDeclWriter(PCHWriter &Writer, ASTContext &Context,
-                  PCHWriter::RecordData &Record)
+    ASTDeclWriter(ASTWriter &Writer, ASTContext &Context,
+                  ASTWriter::RecordData &Record)
       : Writer(Writer), Context(Context), Record(Record) {
     }
     
@@ -76,6 +76,7 @@ namespace clang {
     void VisitParmVarDecl(ParmVarDecl *D);
     void VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D);
     void VisitTemplateDecl(TemplateDecl *D);
+    void VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D);
     void VisitClassTemplateDecl(ClassTemplateDecl *D);
     void VisitFunctionTemplateDecl(FunctionTemplateDecl *D);
     void VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D);
@@ -91,6 +92,7 @@ namespace clang {
 
     void VisitDeclContext(DeclContext *DC, uint64_t LexicalOffset,
                           uint64_t VisibleOffset);
+    template <typename T> void VisitRedeclarable(Redeclarable<T> *D);
 
 
     // FIXME: Put in the same order is DeclNodes.td?
@@ -112,12 +114,12 @@ namespace clang {
   };
 }
 
-void PCHDeclWriter::Visit(Decl *D) {
-  DeclVisitor<PCHDeclWriter>::Visit(D);
+void ASTDeclWriter::Visit(Decl *D) {
+  DeclVisitor<ASTDeclWriter>::Visit(D);
 
   // Handle FunctionDecl's body here and write it after all other Stmts/Exprs
   // have been written. We want it last because we will not read it back when
-  // retrieving it from the PCH, we'll just lazily set the offset. 
+  // retrieving it from the AST, we'll just lazily set the offset. 
   if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     Record.push_back(FD->isThisDeclarationADefinition());
     if (FD->isThisDeclarationADefinition())
@@ -125,7 +127,7 @@ void PCHDeclWriter::Visit(Decl *D) {
   }
 }
 
-void PCHDeclWriter::VisitDecl(Decl *D) {
+void ASTDeclWriter::VisitDecl(Decl *D) {
   Writer.AddDeclRef(cast_or_null<Decl>(D->getDeclContext()), Record);
   Writer.AddDeclRef(cast_or_null<Decl>(D->getLexicalDeclContext()), Record);
   Writer.AddSourceLocation(D->getLocation(), Record);
@@ -137,32 +139,32 @@ void PCHDeclWriter::VisitDecl(Decl *D) {
   Record.push_back(D->getPCHLevel());
 }
 
-void PCHDeclWriter::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
+void ASTDeclWriter::VisitTranslationUnitDecl(TranslationUnitDecl *D) {
   VisitDecl(D);
   Writer.AddDeclRef(D->getAnonymousNamespace(), Record);
-  Code = pch::DECL_TRANSLATION_UNIT;
+  Code = serialization::DECL_TRANSLATION_UNIT;
 }
 
-void PCHDeclWriter::VisitNamedDecl(NamedDecl *D) {
+void ASTDeclWriter::VisitNamedDecl(NamedDecl *D) {
   VisitDecl(D);
   Writer.AddDeclarationName(D->getDeclName(), Record);
 }
 
-void PCHDeclWriter::VisitTypeDecl(TypeDecl *D) {
+void ASTDeclWriter::VisitTypeDecl(TypeDecl *D) {
   VisitNamedDecl(D);
   Writer.AddTypeRef(QualType(D->getTypeForDecl(), 0), Record);
 }
 
-void PCHDeclWriter::VisitTypedefDecl(TypedefDecl *D) {
+void ASTDeclWriter::VisitTypedefDecl(TypedefDecl *D) {
   VisitTypeDecl(D);
   Writer.AddTypeSourceInfo(D->getTypeSourceInfo(), Record);
-  Code = pch::DECL_TYPEDEF;
+  Code = serialization::DECL_TYPEDEF;
 }
 
-void PCHDeclWriter::VisitTagDecl(TagDecl *D) {
+void ASTDeclWriter::VisitTagDecl(TagDecl *D) {
   VisitTypeDecl(D);
   Record.push_back(D->getIdentifierNamespace());
-  Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
+  VisitRedeclarable(D);
   Record.push_back((unsigned)D->getTagKind()); // FIXME: stable encoding
   Record.push_back(D->isDefinition());
   Record.push_back(D->isEmbeddedInDeclarator());
@@ -172,46 +174,47 @@ void PCHDeclWriter::VisitTagDecl(TagDecl *D) {
   Writer.AddDeclRef(D->getTypedefForAnonDecl(), Record);
 }
 
-void PCHDeclWriter::VisitEnumDecl(EnumDecl *D) {
+void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) {
   VisitTagDecl(D);
   Writer.AddTypeRef(D->getIntegerType(), Record);
   Writer.AddTypeRef(D->getPromotionType(), Record);
   Record.push_back(D->getNumPositiveBits());
   Record.push_back(D->getNumNegativeBits());
   Writer.AddDeclRef(D->getInstantiatedFromMemberEnum(), Record);
-  Code = pch::DECL_ENUM;
+  Code = serialization::DECL_ENUM;
 }
 
-void PCHDeclWriter::VisitRecordDecl(RecordDecl *D) {
+void ASTDeclWriter::VisitRecordDecl(RecordDecl *D) {
   VisitTagDecl(D);
   Record.push_back(D->hasFlexibleArrayMember());
   Record.push_back(D->isAnonymousStructOrUnion());
   Record.push_back(D->hasObjectMember());
-  Code = pch::DECL_RECORD;
+  Code = serialization::DECL_RECORD;
 }
 
-void PCHDeclWriter::VisitValueDecl(ValueDecl *D) {
+void ASTDeclWriter::VisitValueDecl(ValueDecl *D) {
   VisitNamedDecl(D);
   Writer.AddTypeRef(D->getType(), Record);
 }
 
-void PCHDeclWriter::VisitEnumConstantDecl(EnumConstantDecl *D) {
+void ASTDeclWriter::VisitEnumConstantDecl(EnumConstantDecl *D) {
   VisitValueDecl(D);
   Record.push_back(D->getInitExpr()? 1 : 0);
   if (D->getInitExpr())
     Writer.AddStmt(D->getInitExpr());
   Writer.AddAPSInt(D->getInitVal(), Record);
-  Code = pch::DECL_ENUM_CONSTANT;
+  Code = serialization::DECL_ENUM_CONSTANT;
 }
 
-void PCHDeclWriter::VisitDeclaratorDecl(DeclaratorDecl *D) {
+void ASTDeclWriter::VisitDeclaratorDecl(DeclaratorDecl *D) {
   VisitValueDecl(D);
   Writer.AddTypeSourceInfo(D->getTypeSourceInfo(), Record);
   // FIXME: write optional qualifier and its range.
 }
 
-void PCHDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
+void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   VisitDeclaratorDecl(D);
+  // FIXME: write DeclarationNameLoc.
 
   Record.push_back(D->getIdentifierNamespace());
   Record.push_back(D->getTemplatedKind());
@@ -275,10 +278,10 @@ void PCHDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   }
   }
 
-  // FunctionDecl's body is handled last at PCHWriterDecl::Visit,
+  // FunctionDecl's body is handled last at ASTWriterDecl::Visit,
   // after everything else is written.
 
-  Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
+  VisitRedeclarable(D);
   Record.push_back(D->getStorageClass()); // FIXME: stable encoding
   Record.push_back(D->getStorageClassAsWritten());
   Record.push_back(D->isInlineSpecified());
@@ -296,15 +299,17 @@ void PCHDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   for (FunctionDecl::param_iterator P = D->param_begin(), PEnd = D->param_end();
        P != PEnd; ++P)
     Writer.AddDeclRef(*P, Record);
-  Code = pch::DECL_FUNCTION;
+  Code = serialization::DECL_FUNCTION;
 }
 
-void PCHDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
+void ASTDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   VisitNamedDecl(D);
   // FIXME: convert to LazyStmtPtr?
   // Unlike C/C++, method bodies will never be in header files.
-  Record.push_back(D->getBody() != 0);
-  if (D->getBody() != 0) {
+  bool HasBodyStuff = D->getBody() != 0     ||
+                      D->getSelfDecl() != 0 || D->getCmdDecl() != 0;
+  Record.push_back(HasBodyStuff);
+  if (HasBodyStuff) {
     Writer.AddStmt(D->getBody());
     Writer.AddDeclRef(D->getSelfDecl(), Record);
     Writer.AddDeclRef(D->getCmdDecl(), Record);
@@ -312,6 +317,7 @@ void PCHDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   Record.push_back(D->isInstanceMethod());
   Record.push_back(D->isVariadic());
   Record.push_back(D->isSynthesized());
+  Record.push_back(D->isDefined());
   // FIXME: stable encoding for @required/@optional
   Record.push_back(D->getImplementationControl());
   // FIXME: stable encoding for in/out/inout/bycopy/byref/oneway
@@ -324,20 +330,22 @@ void PCHDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   for (ObjCMethodDecl::param_iterator P = D->param_begin(),
                                    PEnd = D->param_end(); P != PEnd; ++P)
     Writer.AddDeclRef(*P, Record);
-  Code = pch::DECL_OBJC_METHOD;
+  Code = serialization::DECL_OBJC_METHOD;
 }
 
-void PCHDeclWriter::VisitObjCContainerDecl(ObjCContainerDecl *D) {
+void ASTDeclWriter::VisitObjCContainerDecl(ObjCContainerDecl *D) {
   VisitNamedDecl(D);
   Writer.AddSourceRange(D->getAtEndRange(), Record);
-  // Abstract class (no need to define a stable pch::DECL code).
+  // Abstract class (no need to define a stable serialization::DECL code).
 }
 
-void PCHDeclWriter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
+void ASTDeclWriter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
   VisitObjCContainerDecl(D);
   Writer.AddTypeRef(QualType(D->getTypeForDecl(), 0), Record);
   Writer.AddDeclRef(D->getSuperClass(), Record);
-  Record.push_back(D->protocol_size());
+
+  // Write out the protocols that are directly referenced by the @interface.
+  Record.push_back(D->ReferencedProtocols.size());
   for (ObjCInterfaceDecl::protocol_iterator P = D->protocol_begin(),
          PEnd = D->protocol_end();
        P != PEnd; ++P)
@@ -346,6 +354,16 @@ void PCHDeclWriter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
          PLEnd = D->protocol_loc_end();
        PL != PLEnd; ++PL)
     Writer.AddSourceLocation(*PL, Record);
+
+  // Write out the protocols that are transitively referenced.
+  Record.push_back(D->AllReferencedProtocols.size());
+  for (ObjCList<ObjCProtocolDecl>::iterator
+        P = D->AllReferencedProtocols.begin(),
+        PEnd = D->AllReferencedProtocols.end();
+       P != PEnd; ++P)
+    Writer.AddDeclRef(*P, Record);
+  
+  // Write out the ivars.
   Record.push_back(D->ivar_size());
   for (ObjCInterfaceDecl::ivar_iterator I = D->ivar_begin(),
                                      IEnd = D->ivar_end(); I != IEnd; ++I)
@@ -356,17 +374,18 @@ void PCHDeclWriter::VisitObjCInterfaceDecl(ObjCInterfaceDecl *D) {
   Writer.AddSourceLocation(D->getClassLoc(), Record);
   Writer.AddSourceLocation(D->getSuperClassLoc(), Record);
   Writer.AddSourceLocation(D->getLocEnd(), Record);
-  Code = pch::DECL_OBJC_INTERFACE;
+  Code = serialization::DECL_OBJC_INTERFACE;
 }
 
-void PCHDeclWriter::VisitObjCIvarDecl(ObjCIvarDecl *D) {
+void ASTDeclWriter::VisitObjCIvarDecl(ObjCIvarDecl *D) {
   VisitFieldDecl(D);
   // FIXME: stable encoding for @public/@private/@protected/@package
   Record.push_back(D->getAccessControl());
-  Code = pch::DECL_OBJC_IVAR;
+  Record.push_back(D->getSynthesize());
+  Code = serialization::DECL_OBJC_IVAR;
 }
 
-void PCHDeclWriter::VisitObjCProtocolDecl(ObjCProtocolDecl *D) {
+void ASTDeclWriter::VisitObjCProtocolDecl(ObjCProtocolDecl *D) {
   VisitObjCContainerDecl(D);
   Record.push_back(D->isForwardDecl());
   Writer.AddSourceLocation(D->getLocEnd(), Record);
@@ -378,25 +397,25 @@ void PCHDeclWriter::VisitObjCProtocolDecl(ObjCProtocolDecl *D) {
          PLEnd = D->protocol_loc_end();
        PL != PLEnd; ++PL)
     Writer.AddSourceLocation(*PL, Record);
-  Code = pch::DECL_OBJC_PROTOCOL;
+  Code = serialization::DECL_OBJC_PROTOCOL;
 }
 
-void PCHDeclWriter::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *D) {
+void ASTDeclWriter::VisitObjCAtDefsFieldDecl(ObjCAtDefsFieldDecl *D) {
   VisitFieldDecl(D);
-  Code = pch::DECL_OBJC_AT_DEFS_FIELD;
+  Code = serialization::DECL_OBJC_AT_DEFS_FIELD;
 }
 
-void PCHDeclWriter::VisitObjCClassDecl(ObjCClassDecl *D) {
+void ASTDeclWriter::VisitObjCClassDecl(ObjCClassDecl *D) {
   VisitDecl(D);
   Record.push_back(D->size());
   for (ObjCClassDecl::iterator I = D->begin(), IEnd = D->end(); I != IEnd; ++I)
     Writer.AddDeclRef(I->getInterface(), Record);
   for (ObjCClassDecl::iterator I = D->begin(), IEnd = D->end(); I != IEnd; ++I)
     Writer.AddSourceLocation(I->getLocation(), Record);
-  Code = pch::DECL_OBJC_CLASS;
+  Code = serialization::DECL_OBJC_CLASS;
 }
 
-void PCHDeclWriter::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D) {
+void ASTDeclWriter::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D) {
   VisitDecl(D);
   Record.push_back(D->protocol_size());
   for (ObjCForwardProtocolDecl::protocol_iterator
@@ -406,10 +425,10 @@ void PCHDeclWriter::VisitObjCForwardProtocolDecl(ObjCForwardProtocolDecl *D) {
          PL = D->protocol_loc_begin(), PLEnd = D->protocol_loc_end();
        PL != PLEnd; ++PL)
     Writer.AddSourceLocation(*PL, Record);
-  Code = pch::DECL_OBJC_FORWARD_PROTOCOL;
+  Code = serialization::DECL_OBJC_FORWARD_PROTOCOL;
 }
 
-void PCHDeclWriter::VisitObjCCategoryDecl(ObjCCategoryDecl *D) {
+void ASTDeclWriter::VisitObjCCategoryDecl(ObjCCategoryDecl *D) {
   VisitObjCContainerDecl(D);
   Writer.AddDeclRef(D->getClassInterface(), Record);
   Record.push_back(D->protocol_size());
@@ -421,18 +440,19 @@ void PCHDeclWriter::VisitObjCCategoryDecl(ObjCCategoryDecl *D) {
        PL != PLEnd; ++PL)
     Writer.AddSourceLocation(*PL, Record);
   Writer.AddDeclRef(D->getNextClassCategory(), Record);
+  Record.push_back(D->hasSynthBitfield());
   Writer.AddSourceLocation(D->getAtLoc(), Record);
   Writer.AddSourceLocation(D->getCategoryNameLoc(), Record);
-  Code = pch::DECL_OBJC_CATEGORY;
+  Code = serialization::DECL_OBJC_CATEGORY;
 }
 
-void PCHDeclWriter::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *D) {
+void ASTDeclWriter::VisitObjCCompatibleAliasDecl(ObjCCompatibleAliasDecl *D) {
   VisitNamedDecl(D);
   Writer.AddDeclRef(D->getClassInterface(), Record);
-  Code = pch::DECL_OBJC_COMPATIBLE_ALIAS;
+  Code = serialization::DECL_OBJC_COMPATIBLE_ALIAS;
 }
 
-void PCHDeclWriter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
+void ASTDeclWriter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   VisitNamedDecl(D);
   Writer.AddSourceLocation(D->getAtLoc(), Record);
   Writer.AddTypeSourceInfo(D->getTypeSourceInfo(), Record);
@@ -446,38 +466,41 @@ void PCHDeclWriter::VisitObjCPropertyDecl(ObjCPropertyDecl *D) {
   Writer.AddDeclRef(D->getGetterMethodDecl(), Record);
   Writer.AddDeclRef(D->getSetterMethodDecl(), Record);
   Writer.AddDeclRef(D->getPropertyIvarDecl(), Record);
-  Code = pch::DECL_OBJC_PROPERTY;
+  Code = serialization::DECL_OBJC_PROPERTY;
 }
 
-void PCHDeclWriter::VisitObjCImplDecl(ObjCImplDecl *D) {
+void ASTDeclWriter::VisitObjCImplDecl(ObjCImplDecl *D) {
   VisitObjCContainerDecl(D);
   Writer.AddDeclRef(D->getClassInterface(), Record);
-  // Abstract class (no need to define a stable pch::DECL code).
+  // Abstract class (no need to define a stable serialization::DECL code).
 }
 
-void PCHDeclWriter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
+void ASTDeclWriter::VisitObjCCategoryImplDecl(ObjCCategoryImplDecl *D) {
   VisitObjCImplDecl(D);
   Writer.AddIdentifierRef(D->getIdentifier(), Record);
-  Code = pch::DECL_OBJC_CATEGORY_IMPL;
+  Code = serialization::DECL_OBJC_CATEGORY_IMPL;
 }
 
-void PCHDeclWriter::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
+void ASTDeclWriter::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   VisitObjCImplDecl(D);
   Writer.AddDeclRef(D->getSuperClass(), Record);
-  // FIXME add writing of IvarInitializers and NumIvarInitializers.
-  Code = pch::DECL_OBJC_IMPLEMENTATION;
+  Writer.AddCXXBaseOrMemberInitializers(D->IvarInitializers,
+                                        D->NumIvarInitializers, Record);
+  Record.push_back(D->hasSynthBitfield());
+  Code = serialization::DECL_OBJC_IMPLEMENTATION;
 }
 
-void PCHDeclWriter::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
+void ASTDeclWriter::VisitObjCPropertyImplDecl(ObjCPropertyImplDecl *D) {
   VisitDecl(D);
   Writer.AddSourceLocation(D->getLocStart(), Record);
   Writer.AddDeclRef(D->getPropertyDecl(), Record);
   Writer.AddDeclRef(D->getPropertyIvarDecl(), Record);
-  // FIXME. write GetterCXXConstructor and SetterCXXAssignment.
-  Code = pch::DECL_OBJC_PROPERTY_IMPL;
+  Writer.AddStmt(D->getGetterCXXConstructor());
+  Writer.AddStmt(D->getSetterCXXAssignment());
+  Code = serialization::DECL_OBJC_PROPERTY_IMPL;
 }
 
-void PCHDeclWriter::VisitFieldDecl(FieldDecl *D) {
+void ASTDeclWriter::VisitFieldDecl(FieldDecl *D) {
   VisitDeclaratorDecl(D);
   Record.push_back(D->isMutable());
   Record.push_back(D->getBitWidth()? 1 : 0);
@@ -485,19 +508,18 @@ void PCHDeclWriter::VisitFieldDecl(FieldDecl *D) {
     Writer.AddStmt(D->getBitWidth());
   if (!D->getDeclName())
     Writer.AddDeclRef(Context.getInstantiatedFromUnnamedFieldDecl(D), Record);
-  Code = pch::DECL_FIELD;
+  Code = serialization::DECL_FIELD;
 }
 
-void PCHDeclWriter::VisitVarDecl(VarDecl *D) {
+void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
   VisitDeclaratorDecl(D);
   Record.push_back(D->getStorageClass()); // FIXME: stable encoding
   Record.push_back(D->getStorageClassAsWritten());
   Record.push_back(D->isThreadSpecified());
   Record.push_back(D->hasCXXDirectInitializer());
-  Record.push_back(D->isDeclaredInCondition());
   Record.push_back(D->isExceptionVariable());
   Record.push_back(D->isNRVOVariable());
-  Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
+  VisitRedeclarable(D);
   Record.push_back(D->getInit() ? 1 : 0);
   if (D->getInit())
     Writer.AddStmt(D->getInit());
@@ -511,22 +533,22 @@ void PCHDeclWriter::VisitVarDecl(VarDecl *D) {
     Writer.AddSourceLocation(SpecInfo->getPointOfInstantiation(), Record);
   }
 
-  Code = pch::DECL_VAR;
+  Code = serialization::DECL_VAR;
 }
 
-void PCHDeclWriter::VisitImplicitParamDecl(ImplicitParamDecl *D) {
+void ASTDeclWriter::VisitImplicitParamDecl(ImplicitParamDecl *D) {
   VisitVarDecl(D);
-  Code = pch::DECL_IMPLICIT_PARAM;
+  Code = serialization::DECL_IMPLICIT_PARAM;
 }
 
-void PCHDeclWriter::VisitParmVarDecl(ParmVarDecl *D) {
+void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) {
   VisitVarDecl(D);
   Record.push_back(D->getObjCDeclQualifier()); // FIXME: stable encoding
   Record.push_back(D->hasInheritedDefaultArg());
   Record.push_back(D->hasUninstantiatedDefaultArg());
   if (D->hasUninstantiatedDefaultArg())
     Writer.AddStmt(D->getUninstantiatedDefaultArg());
-  Code = pch::DECL_PARM_VAR;
+  Code = serialization::DECL_PARM_VAR;
 
   // If the assumptions about the DECL_PARM_VAR abbrev are true, use it.  Here
   // we dynamically check for the properties that we optimize for, but don't
@@ -550,20 +572,19 @@ void PCHDeclWriter::VisitParmVarDecl(ParmVarDecl *D) {
   assert(!D->isInvalidDecl() && "Shouldn't emit invalid decls");
   assert(!D->isThreadSpecified() && "PARM_VAR_DECL can't be __thread");
   assert(D->getAccess() == AS_none && "PARM_VAR_DECL can't be public/private");
-  assert(!D->isDeclaredInCondition() && "PARM_VAR_DECL can't be in condition");
   assert(!D->isExceptionVariable() && "PARM_VAR_DECL can't be exception var");
   assert(D->getPreviousDeclaration() == 0 && "PARM_VAR_DECL can't be redecl");
   assert(!D->isStaticDataMember() &&
          "PARM_VAR_DECL can't be static data member");
 }
 
-void PCHDeclWriter::VisitFileScopeAsmDecl(FileScopeAsmDecl *D) {
+void ASTDeclWriter::VisitFileScopeAsmDecl(FileScopeAsmDecl *D) {
   VisitDecl(D);
   Writer.AddStmt(D->getAsmString());
-  Code = pch::DECL_FILE_SCOPE_ASM;
+  Code = serialization::DECL_FILE_SCOPE_ASM;
 }
 
-void PCHDeclWriter::VisitBlockDecl(BlockDecl *D) {
+void ASTDeclWriter::VisitBlockDecl(BlockDecl *D) {
   VisitDecl(D);
   Writer.AddStmt(D->getBody());
   Writer.AddTypeSourceInfo(D->getSignatureAsWritten(), Record);
@@ -571,19 +592,19 @@ void PCHDeclWriter::VisitBlockDecl(BlockDecl *D) {
   for (FunctionDecl::param_iterator P = D->param_begin(), PEnd = D->param_end();
        P != PEnd; ++P)
     Writer.AddDeclRef(*P, Record);
-  Code = pch::DECL_BLOCK;
+  Code = serialization::DECL_BLOCK;
 }
 
-void PCHDeclWriter::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
+void ASTDeclWriter::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
   VisitDecl(D);
   // FIXME: It might be nice to serialize the brace locations for this
   // declaration, which don't seem to be readily available in the AST.
   Record.push_back(D->getLanguage());
   Record.push_back(D->hasBraces());
-  Code = pch::DECL_LINKAGE_SPEC;
+  Code = serialization::DECL_LINKAGE_SPEC;
 }
 
-void PCHDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
+void ASTDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
   VisitNamedDecl(D);
   Writer.AddSourceLocation(D->getLBracLoc(), Record);
   Writer.AddSourceLocation(D->getRBracLoc(), Record);
@@ -595,20 +616,25 @@ void PCHDeclWriter::VisitNamespaceDecl(NamespaceDecl *D) {
     Writer.AddDeclRef(D->getAnonymousNamespace(), Record);
   else
     Writer.AddDeclRef(D->getOriginalNamespace(), Record);
-  Code = pch::DECL_NAMESPACE;
+  Code = serialization::DECL_NAMESPACE;
+
+  if (Writer.hasChain() && !D->isOriginalNamespace() &&
+      D->getOriginalNamespace()->getPCHLevel() > 0) {
+    Writer.AddUpdatedNamespace(D->getOriginalNamespace());
+  }
 }
 
-void PCHDeclWriter::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
+void ASTDeclWriter::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
   VisitNamedDecl(D);
-  Writer.AddSourceLocation(D->getAliasLoc(), Record);
+  Writer.AddSourceLocation(D->getNamespaceLoc(), Record);
   Writer.AddSourceRange(D->getQualifierRange(), Record);
   Writer.AddNestedNameSpecifier(D->getQualifier(), Record);
   Writer.AddSourceLocation(D->getTargetNameLoc(), Record);
   Writer.AddDeclRef(D->getNamespace(), Record);
-  Code = pch::DECL_NAMESPACE_ALIAS;
+  Code = serialization::DECL_NAMESPACE_ALIAS;
 }
 
-void PCHDeclWriter::VisitUsingDecl(UsingDecl *D) {
+void ASTDeclWriter::VisitUsingDecl(UsingDecl *D) {
   VisitNamedDecl(D);
   Writer.AddSourceRange(D->getNestedNameRange(), Record);
   Writer.AddSourceLocation(D->getUsingLocation(), Record);
@@ -619,48 +645,48 @@ void PCHDeclWriter::VisitUsingDecl(UsingDecl *D) {
     Writer.AddDeclRef(*P, Record);
   Record.push_back(D->isTypeName());
   Writer.AddDeclRef(Context.getInstantiatedFromUsingDecl(D), Record);
-  Code = pch::DECL_USING;
+  Code = serialization::DECL_USING;
 }
 
-void PCHDeclWriter::VisitUsingShadowDecl(UsingShadowDecl *D) {
+void ASTDeclWriter::VisitUsingShadowDecl(UsingShadowDecl *D) {
   VisitNamedDecl(D);
   Writer.AddDeclRef(D->getTargetDecl(), Record);
   Writer.AddDeclRef(D->getUsingDecl(), Record);
   Writer.AddDeclRef(Context.getInstantiatedFromUsingShadowDecl(D), Record);
-  Code = pch::DECL_USING_SHADOW;
+  Code = serialization::DECL_USING_SHADOW;
 }
 
-void PCHDeclWriter::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
+void ASTDeclWriter::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   VisitNamedDecl(D);
+  Writer.AddSourceLocation(D->getUsingLoc(), Record);
   Writer.AddSourceLocation(D->getNamespaceKeyLocation(), Record);
   Writer.AddSourceRange(D->getQualifierRange(), Record);
   Writer.AddNestedNameSpecifier(D->getQualifier(), Record);
-  Writer.AddSourceLocation(D->getIdentLocation(), Record);
   Writer.AddDeclRef(D->getNominatedNamespace(), Record);
   Writer.AddDeclRef(dyn_cast<Decl>(D->getCommonAncestor()), Record);
-  Code = pch::DECL_USING_DIRECTIVE;
+  Code = serialization::DECL_USING_DIRECTIVE;
 }
 
-void PCHDeclWriter::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
+void ASTDeclWriter::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
   VisitValueDecl(D);
   Writer.AddSourceRange(D->getTargetNestedNameRange(), Record);
   Writer.AddSourceLocation(D->getUsingLoc(), Record);
   Writer.AddNestedNameSpecifier(D->getTargetNestedNameSpecifier(), Record);
-  Code = pch::DECL_UNRESOLVED_USING_VALUE;
+  Code = serialization::DECL_UNRESOLVED_USING_VALUE;
 }
 
-void PCHDeclWriter::VisitUnresolvedUsingTypenameDecl(
+void ASTDeclWriter::VisitUnresolvedUsingTypenameDecl(
                                                UnresolvedUsingTypenameDecl *D) {
   VisitTypeDecl(D);
   Writer.AddSourceRange(D->getTargetNestedNameRange(), Record);
   Writer.AddSourceLocation(D->getUsingLoc(), Record);
   Writer.AddSourceLocation(D->getTypenameLoc(), Record);
   Writer.AddNestedNameSpecifier(D->getTargetNestedNameSpecifier(), Record);
-  Code = pch::DECL_UNRESOLVED_USING_TYPENAME;
+  Code = serialization::DECL_UNRESOLVED_USING_TYPENAME;
 }
 
-void PCHDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
-  // See comments at PCHDeclReader::VisitCXXRecordDecl about why this happens
+void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
+  // See comments at ASTDeclReader::VisitCXXRecordDecl about why this happens
   // before VisitRecordDecl.
   enum { Data_NoDefData, Data_Owner, Data_NotOwner };
   bool OwnsDefinitionData = false;
@@ -735,76 +761,52 @@ void PCHDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
     Record.push_back(CXXRecNotTemplate);
   }
 
-  Code = pch::DECL_CXX_RECORD;
+  Code = serialization::DECL_CXX_RECORD;
 }
 
-void PCHDeclWriter::VisitCXXMethodDecl(CXXMethodDecl *D) {
+void ASTDeclWriter::VisitCXXMethodDecl(CXXMethodDecl *D) {
   VisitFunctionDecl(D);
   Record.push_back(D->size_overridden_methods());
   for (CXXMethodDecl::method_iterator
          I = D->begin_overridden_methods(), E = D->end_overridden_methods();
          I != E; ++I)
     Writer.AddDeclRef(*I, Record);
-  Code = pch::DECL_CXX_METHOD;
+  Code = serialization::DECL_CXX_METHOD;
 }
 
-void PCHDeclWriter::VisitCXXConstructorDecl(CXXConstructorDecl *D) {
+void ASTDeclWriter::VisitCXXConstructorDecl(CXXConstructorDecl *D) {
   VisitCXXMethodDecl(D);
 
   Record.push_back(D->IsExplicitSpecified);
   Record.push_back(D->ImplicitlyDefined);
-  
-  Record.push_back(D->NumBaseOrMemberInitializers);
-  for (unsigned i=0; i != D->NumBaseOrMemberInitializers; ++i) {
-    CXXBaseOrMemberInitializer *Init = D->BaseOrMemberInitializers[i];
-
-    Record.push_back(Init->isBaseInitializer());
-    if (Init->isBaseInitializer()) {
-      Writer.AddTypeSourceInfo(Init->getBaseClassInfo(), Record);
-      Record.push_back(Init->isBaseVirtual());
-    } else {
-      Writer.AddDeclRef(Init->getMember(), Record);
-    }
-    Writer.AddSourceLocation(Init->getMemberLocation(), Record);
-    Writer.AddStmt(Init->getInit());
-    Writer.AddDeclRef(Init->getAnonUnionMember(), Record);
-    Writer.AddSourceLocation(Init->getLParenLoc(), Record);
-    Writer.AddSourceLocation(Init->getRParenLoc(), Record);
-    Record.push_back(Init->isWritten());
-    if (Init->isWritten()) {
-      Record.push_back(Init->getSourceOrder());
-    } else {
-      Record.push_back(Init->getNumArrayIndices());
-      for (unsigned i=0, e=Init->getNumArrayIndices(); i != e; ++i)
-        Writer.AddDeclRef(Init->getArrayIndex(i), Record);
-    }
-  }
+  Writer.AddCXXBaseOrMemberInitializers(D->BaseOrMemberInitializers,
+                                        D->NumBaseOrMemberInitializers, Record);
 
-  Code = pch::DECL_CXX_CONSTRUCTOR;
+  Code = serialization::DECL_CXX_CONSTRUCTOR;
 }
 
-void PCHDeclWriter::VisitCXXDestructorDecl(CXXDestructorDecl *D) {
+void ASTDeclWriter::VisitCXXDestructorDecl(CXXDestructorDecl *D) {
   VisitCXXMethodDecl(D);
 
   Record.push_back(D->ImplicitlyDefined);
   Writer.AddDeclRef(D->OperatorDelete, Record);
 
-  Code = pch::DECL_CXX_DESTRUCTOR;
+  Code = serialization::DECL_CXX_DESTRUCTOR;
 }
 
-void PCHDeclWriter::VisitCXXConversionDecl(CXXConversionDecl *D) {
+void ASTDeclWriter::VisitCXXConversionDecl(CXXConversionDecl *D) {
   VisitCXXMethodDecl(D);
   Record.push_back(D->IsExplicitSpecified);
-  Code = pch::DECL_CXX_CONVERSION;
+  Code = serialization::DECL_CXX_CONVERSION;
 }
 
-void PCHDeclWriter::VisitAccessSpecDecl(AccessSpecDecl *D) {
+void ASTDeclWriter::VisitAccessSpecDecl(AccessSpecDecl *D) {
   VisitDecl(D);
   Writer.AddSourceLocation(D->getColonLoc(), Record);
-  Code = pch::DECL_ACCESS_SPEC;
+  Code = serialization::DECL_ACCESS_SPEC;
 }
 
-void PCHDeclWriter::VisitFriendDecl(FriendDecl *D) {
+void ASTDeclWriter::VisitFriendDecl(FriendDecl *D) {
   VisitDecl(D);
   Record.push_back(D->Friend.is<TypeSourceInfo*>());
   if (D->Friend.is<TypeSourceInfo*>())
@@ -813,38 +815,68 @@ void PCHDeclWriter::VisitFriendDecl(FriendDecl *D) {
     Writer.AddDeclRef(D->Friend.get<NamedDecl*>(), Record);
   Writer.AddDeclRef(D->NextFriend, Record);
   Writer.AddSourceLocation(D->FriendLoc, Record);
-  Code = pch::DECL_FRIEND;
+  Code = serialization::DECL_FRIEND;
 }
 
-void PCHDeclWriter::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
-  assert(false && "cannot write FriendTemplateDecl");
+void ASTDeclWriter::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
+  VisitDecl(D);
+  Record.push_back(D->getNumTemplateParameters());
+  for (unsigned i = 0, e = D->getNumTemplateParameters(); i != e; ++i)
+    Writer.AddTemplateParameterList(D->getTemplateParameterList(i), Record);
+  Record.push_back(D->getFriendDecl() != 0);
+  if (D->getFriendDecl())
+    Writer.AddDeclRef(D->getFriendDecl(), Record);
+  else
+    Writer.AddTypeSourceInfo(D->getFriendType(), Record);
+  Writer.AddSourceLocation(D->getFriendLoc(), Record);
+  Code = serialization::DECL_FRIEND_TEMPLATE;
 }
 
-void PCHDeclWriter::VisitTemplateDecl(TemplateDecl *D) {
+void ASTDeclWriter::VisitTemplateDecl(TemplateDecl *D) {
   VisitNamedDecl(D);
 
   Writer.AddDeclRef(D->getTemplatedDecl(), Record);
   Writer.AddTemplateParameterList(D->getTemplateParameters(), Record);
 }
 
-static bool IsKeptInFoldingSet(ClassTemplateSpecializationDecl *D) {
-  return D->getTypeForDecl()->getAsCXXRecordDecl() == D;
-}
-
-void PCHDeclWriter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
+void ASTDeclWriter::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
   VisitTemplateDecl(D);
 
   Record.push_back(D->getIdentifierNamespace());
   Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
   if (D->getPreviousDeclaration() == 0) {
-    // This ClassTemplateDecl owns the CommonPtr; write it.
+    // This TemplateDecl owns the CommonPtr; write it.
     assert(D->isCanonicalDecl());
 
+    Writer.AddDeclRef(D->getInstantiatedFromMemberTemplate(), Record);
+    if (D->getInstantiatedFromMemberTemplate())
+      Record.push_back(D->isMemberSpecialization());
+
+    Writer.AddDeclRef(D->getCommonPtr()->Latest, Record);
+  } else {
+    RedeclarableTemplateDecl *First = D->getFirstDeclaration();
+    assert(First != D);
+    // If this is a most recent redeclaration that is pointed to by a first decl
+    // in a chained PCH, keep track of the association with the map so we can
+    // update the first decl during AST reading.
+    if (First->getMostRecentDeclaration() == D &&
+        First->getPCHLevel() > D->getPCHLevel()) {
+      assert(Writer.FirstLatestDecls.find(First)==Writer.FirstLatestDecls.end()
+             && "The latest is already set");
+      Writer.FirstLatestDecls[First] = D;
+    }
+  }
+}
+
+void ASTDeclWriter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
+  VisitRedeclarableTemplateDecl(D);
+
+  if (D->getPreviousDeclaration() == 0) {
     typedef llvm::FoldingSet<ClassTemplateSpecializationDecl> CTSDSetTy;
     CTSDSetTy &CTSDSet = D->getSpecializations();
     Record.push_back(CTSDSet.size());
     for (CTSDSetTy::iterator I=CTSDSet.begin(), E = CTSDSet.end(); I!=E; ++I) {
-      assert(IsKeptInFoldingSet(&*I));
+      assert(I->isCanonicalDecl() && "Expected only canonical decls in set");
       Writer.AddDeclRef(&*I, Record);
     }
 
@@ -852,33 +884,37 @@ void PCHDeclWriter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
     CTPSDSetTy &CTPSDSet = D->getPartialSpecializations();
     Record.push_back(CTPSDSet.size());
     for (CTPSDSetTy::iterator I=CTPSDSet.begin(), E=CTPSDSet.end(); I!=E; ++I) {
-      assert(IsKeptInFoldingSet(&*I));
+      assert(I->isCanonicalDecl() && "Expected only canonical decls in set");
       Writer.AddDeclRef(&*I, Record); 
     }
 
     // InjectedClassNameType is computed, no need to write it.
-
-    Writer.AddDeclRef(D->getInstantiatedFromMemberTemplate(), Record);
-    if (D->getInstantiatedFromMemberTemplate())
-      Record.push_back(D->isMemberSpecialization());
   }
-  Code = pch::DECL_CLASS_TEMPLATE;
+  Code = serialization::DECL_CLASS_TEMPLATE;
 }
 
-void PCHDeclWriter::VisitClassTemplateSpecializationDecl(
+void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
                                            ClassTemplateSpecializationDecl *D) {
   VisitCXXRecordDecl(D);
 
   llvm::PointerUnion<ClassTemplateDecl *,
                      ClassTemplatePartialSpecializationDecl *> InstFrom
     = D->getSpecializedTemplateOrPartial();
+  Decl *InstFromD;
   if (InstFrom.is<ClassTemplateDecl *>()) {
-    Writer.AddDeclRef(InstFrom.get<ClassTemplateDecl *>(), Record);
+    InstFromD = InstFrom.get<ClassTemplateDecl *>();
+    Writer.AddDeclRef(InstFromD, Record);
   } else {
-    Writer.AddDeclRef(InstFrom.get<ClassTemplatePartialSpecializationDecl *>(),
-                      Record);
+    InstFromD = InstFrom.get<ClassTemplatePartialSpecializationDecl *>();
+    Writer.AddDeclRef(InstFromD, Record);
     Writer.AddTemplateArgumentList(&D->getTemplateInstantiationArgs(), Record);
+    InstFromD = cast<ClassTemplatePartialSpecializationDecl>(InstFromD)->
+                    getSpecializedTemplate();
   }
+  // Is this a specialization of an already-serialized template?
+  if (InstFromD->getCanonicalDecl()->getPCHLevel() != 0)
+    Writer.AddAdditionalTemplateSpecialization(Writer.getDeclID(InstFromD),
+                                               Writer.getDeclID(D));
 
   // Explicit info.
   Writer.AddTypeSourceInfo(D->getTypeAsWritten(), Record);
@@ -891,17 +927,15 @@ void PCHDeclWriter::VisitClassTemplateSpecializationDecl(
   Writer.AddSourceLocation(D->getPointOfInstantiation(), Record);
   Record.push_back(D->getSpecializationKind());
 
-  bool IsInInFoldingSet = IsKeptInFoldingSet(D);
-  Record.push_back(IsInInFoldingSet);
-  if (IsInInFoldingSet) {
-    // When reading, we'll add it to the folding set of this one. 
+  if (D->isCanonicalDecl()) {
+    // When reading, we'll add it to the folding set of the following template. 
     Writer.AddDeclRef(D->getSpecializedTemplate()->getCanonicalDecl(), Record);
   }
 
-  Code = pch::DECL_CLASS_TEMPLATE_SPECIALIZATION;
+  Code = serialization::DECL_CLASS_TEMPLATE_SPECIALIZATION;
 }
 
-void PCHDeclWriter::VisitClassTemplatePartialSpecializationDecl(
+void ASTDeclWriter::VisitClassTemplatePartialSpecializationDecl(
                                     ClassTemplatePartialSpecializationDecl *D) {
   VisitClassTemplateSpecializationDecl(D);
 
@@ -919,14 +953,12 @@ void PCHDeclWriter::VisitClassTemplatePartialSpecializationDecl(
     Record.push_back(D->isMemberSpecialization());
   }
 
-  Code = pch::DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION;
+  Code = serialization::DECL_CLASS_TEMPLATE_PARTIAL_SPECIALIZATION;
 }
 
-void PCHDeclWriter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
-  VisitTemplateDecl(D);
+void ASTDeclWriter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
+  VisitRedeclarableTemplateDecl(D);
 
-  Record.push_back(D->getIdentifierNamespace());
-  Writer.AddDeclRef(D->getPreviousDeclaration(), Record);
   if (D->getPreviousDeclaration() == 0) {
     // This FunctionTemplateDecl owns the CommonPtr; write it.
 
@@ -934,17 +966,16 @@ void PCHDeclWriter::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
     Record.push_back(D->getSpecializations().size());
     for (llvm::FoldingSet<FunctionTemplateSpecializationInfo>::iterator
            I = D->getSpecializations().begin(),
-           E = D->getSpecializations().end()   ; I != E; ++I)
+           E = D->getSpecializations().end()   ; I != E; ++I) {
+      assert(I->Function->isCanonicalDecl() &&
+             "Expected only canonical decls in set");
       Writer.AddDeclRef(I->Function, Record);
-
-    Writer.AddDeclRef(D->getInstantiatedFromMemberTemplate(), Record);
-    if (D->getInstantiatedFromMemberTemplate())
-      Record.push_back(D->isMemberSpecialization());
+    }
   }
-  Code = pch::DECL_FUNCTION_TEMPLATE;
+  Code = serialization::DECL_FUNCTION_TEMPLATE;
 }
 
-void PCHDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
+void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
   VisitTypeDecl(D);
 
   Record.push_back(D->wasDeclaredWithTypename());
@@ -952,10 +983,10 @@ void PCHDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
   Record.push_back(D->defaultArgumentWasInherited());
   Writer.AddTypeSourceInfo(D->getDefaultArgumentInfo(), Record);
 
-  Code = pch::DECL_TEMPLATE_TYPE_PARM;
+  Code = serialization::DECL_TEMPLATE_TYPE_PARM;
 }
 
-void PCHDeclWriter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
+void ASTDeclWriter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
   VisitVarDecl(D);
   // TemplateParmPosition.
   Record.push_back(D->getDepth());
@@ -966,10 +997,10 @@ void PCHDeclWriter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
     Writer.AddStmt(D->getDefaultArgument());
     Record.push_back(D->defaultArgumentWasInherited());
   }
-  Code = pch::DECL_NON_TYPE_TEMPLATE_PARM;
+  Code = serialization::DECL_NON_TYPE_TEMPLATE_PARM;
 }
 
-void PCHDeclWriter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
+void ASTDeclWriter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   VisitTemplateDecl(D);
   // TemplateParmPosition.
   Record.push_back(D->getDepth());
@@ -977,11 +1008,14 @@ void PCHDeclWriter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   // Rest of TemplateTemplateParmDecl.
   Writer.AddTemplateArgumentLoc(D->getDefaultArgument(), Record);
   Record.push_back(D->defaultArgumentWasInherited());
-  Code = pch::DECL_TEMPLATE_TEMPLATE_PARM;
+  Code = serialization::DECL_TEMPLATE_TEMPLATE_PARM;
 }
 
-void PCHDeclWriter::VisitStaticAssertDecl(StaticAssertDecl *D) {
-  assert(false && "cannot write StaticAssertDecl");
+void ASTDeclWriter::VisitStaticAssertDecl(StaticAssertDecl *D) {
+  VisitDecl(D);
+  Writer.AddStmt(D->getAssertExpr());
+  Writer.AddStmt(D->getMessage());
+  Code = serialization::DECL_STATIC_ASSERT;
 }
 
 /// \brief Emit the DeclContext part of a declaration context decl.
@@ -995,22 +1029,45 @@ void PCHDeclWriter::VisitStaticAssertDecl(StaticAssertDecl *D) {
 /// that there are no declarations visible from this context. Note
 /// that this value will not be emitted for non-primary declaration
 /// contexts.
-void PCHDeclWriter::VisitDeclContext(DeclContext *DC, uint64_t LexicalOffset,
+void ASTDeclWriter::VisitDeclContext(DeclContext *DC, uint64_t LexicalOffset,
                                      uint64_t VisibleOffset) {
   Record.push_back(LexicalOffset);
   Record.push_back(VisibleOffset);
 }
 
+template <typename T>
+void ASTDeclWriter::VisitRedeclarable(Redeclarable<T> *D) {
+  enum { NoRedeclaration = 0, PointsToPrevious, PointsToLatest };
+  if (D->RedeclLink.getNext() == D) {
+    Record.push_back(NoRedeclaration);
+  } else {
+    Record.push_back(D->RedeclLink.NextIsPrevious() ? PointsToPrevious
+                                                    : PointsToLatest);
+    Writer.AddDeclRef(D->RedeclLink.getPointer(), Record);
+  }
+
+  T *First = D->getFirstDeclaration();
+  T *ThisDecl = static_cast<T*>(D);
+  // If this is a most recent redeclaration that is pointed to by a first decl
+  // in a chained PCH, keep track of the association with the map so we can
+  // update the first decl during AST reading.
+  if (ThisDecl != First && First->getMostRecentDeclaration() == ThisDecl &&
+      First->getPCHLevel() > ThisDecl->getPCHLevel()) {
+    assert(Writer.FirstLatestDecls.find(First) == Writer.FirstLatestDecls.end()
+           && "The latest is already set");
+    Writer.FirstLatestDecls[First] = ThisDecl;
+  }
+}
 
 //===----------------------------------------------------------------------===//
-// PCHWriter Implementation
+// ASTWriter Implementation
 //===----------------------------------------------------------------------===//
 
-void PCHWriter::WriteDeclsBlockAbbrevs() {
+void ASTWriter::WriteDeclsBlockAbbrevs() {
   using namespace llvm;
   // Abbreviation for DECL_PARM_VAR.
   BitCodeAbbrev *Abv = new BitCodeAbbrev();
-  Abv->Add(BitCodeAbbrevOp(pch::DECL_PARM_VAR));
+  Abv->Add(BitCodeAbbrevOp(serialization::DECL_PARM_VAR));
 
   // Decl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext
@@ -1029,13 +1086,12 @@ void PCHWriter::WriteDeclsBlockAbbrevs() {
   // ValueDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type
   // DeclaratorDecl
-  Abv->Add(BitCodeAbbrevOp(pch::PREDEF_TYPE_NULL_ID)); // InfoType
+  Abv->Add(BitCodeAbbrevOp(serialization::PREDEF_TYPE_NULL_ID)); // InfoType
   // VarDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // StorageClass
   Abv->Add(BitCodeAbbrevOp(0));                       // StorageClassAsWritten
   Abv->Add(BitCodeAbbrevOp(0));                       // isThreadSpecified
   Abv->Add(BitCodeAbbrevOp(0));                       // hasCXXDirectInitializer
-  Abv->Add(BitCodeAbbrevOp(0));                       // isDeclaredInCondition
   Abv->Add(BitCodeAbbrevOp(0));                       // isExceptionVariable
   Abv->Add(BitCodeAbbrevOp(0));                       // isNRVOVariable
   Abv->Add(BitCodeAbbrevOp(0));                       // PrevDecl
@@ -1047,12 +1103,23 @@ void PCHWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                   // HasUninstantiatedDefaultArg
 
   ParmVarDeclAbbrev = Stream.EmitAbbrev(Abv);
+
+  Abv = new BitCodeAbbrev();
+  Abv->Add(BitCodeAbbrevOp(serialization::DECL_CONTEXT_LEXICAL));
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  DeclContextLexicalAbbrev = Stream.EmitAbbrev(Abv);
+
+  Abv = new BitCodeAbbrev();
+  Abv->Add(BitCodeAbbrevOp(serialization::DECL_CONTEXT_VISIBLE));
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+  DeclContextVisibleLookupAbbrev = Stream.EmitAbbrev(Abv);
 }
 
 /// isRequiredDecl - Check if this is a "required" Decl, which must be seen by
 /// consumers of the AST.
 ///
-/// Such decls will always be deserialized from the PCH file, so we would like
+/// Such decls will always be deserialized from the AST file, so we would like
 /// this to be as restrictive as possible. Currently the predicate is driven by
 /// code generation requirements, if other clients have a different notion of
 /// what is "required" then we may have to consider an alternate scheme where
@@ -1061,60 +1128,17 @@ void PCHWriter::WriteDeclsBlockAbbrevs() {
 /// clients to use a separate API call to "realize" the decl. This should be
 /// relatively painless since they would presumably only do it for top-level
 /// decls.
-//
-// FIXME: This predicate is essentially IRgen's predicate to determine whether a
-// declaration can be deferred. Merge them somehow.
 static bool isRequiredDecl(const Decl *D, ASTContext &Context) {
-  // File scoped assembly must be seen.
-  if (isa<FileScopeAsmDecl>(D))
+  // File scoped assembly or obj-c implementation must be seen.
+  if (isa<FileScopeAsmDecl>(D) || isa<ObjCImplementationDecl>(D))
     return true;
 
-  // Otherwise if this isn't a function or a file scoped variable it doesn't
-  // need to be seen.
-  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
-    if (!VD->isFileVarDecl())
-      return false;
-  } else if (!isa<FunctionDecl>(D))
-    return false;
-
-  // Aliases and used decls must be seen.
-  if (D->hasAttr<AliasAttr>() || D->hasAttr<UsedAttr>())
-    return true;
-
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
-    // Forward declarations don't need to be seen.
-    if (!FD->isThisDeclarationADefinition())
-      return false;
-
-    // Constructors and destructors must be seen.
-    if (FD->hasAttr<ConstructorAttr>() || FD->hasAttr<DestructorAttr>())
-      return true;
-
-    // Otherwise, this is required unless it is static.
-    //
-    // FIXME: Inlines.
-    return FD->getStorageClass() != FunctionDecl::Static;
-  } else {
-    const VarDecl *VD = cast<VarDecl>(D);
-
-    // In C++, this doesn't need to be seen if it is marked "extern".
-    if (Context.getLangOptions().CPlusPlus && !VD->getInit() &&
-        (VD->getStorageClass() == VarDecl::Extern ||
-         VD->isExternC()))
-      return false;
-
-    // In C, this doesn't need to be seen unless it is a definition.
-    if (!Context.getLangOptions().CPlusPlus && !VD->getInit())
-      return false;
-
-    // Otherwise, this is required unless it is static.
-    return VD->getStorageClass() != VarDecl::Static;
-  }
+  return Context.DeclMustBeEmitted(D);
 }
 
-void PCHWriter::WriteDecl(ASTContext &Context, Decl *D) {
+void ASTWriter::WriteDecl(ASTContext &Context, Decl *D) {
   RecordData Record;
-  PCHDeclWriter W(*this, Context, Record);
+  ASTDeclWriter W(*this, Context, Record);
 
   // If this declaration is also a DeclContext, write blocks for the
   // declarations that lexically stored inside its context and those
@@ -1130,23 +1154,29 @@ void PCHWriter::WriteDecl(ASTContext &Context, Decl *D) {
   }
 
   // Determine the ID for this declaration
-  pch::DeclID &ID = DeclIDs[D];
-  if (ID == 0)
-    ID = DeclIDs.size();
-
-  unsigned Index = ID - 1;
-
-  // Record the offset for this declaration
-  if (DeclOffsets.size() == Index)
-    DeclOffsets.push_back(Stream.GetCurrentBitNo());
-  else if (DeclOffsets.size() < Index) {
-    DeclOffsets.resize(Index+1);
-    DeclOffsets[Index] = Stream.GetCurrentBitNo();
+  serialization::DeclID &IDR = DeclIDs[D];
+  if (IDR == 0)
+    IDR = NextDeclID++;
+  serialization::DeclID ID = IDR;
+
+  if (ID < FirstDeclID) {
+    // We're replacing a decl in a previous file.
+    ReplacedDecls.push_back(std::make_pair(ID, Stream.GetCurrentBitNo()));
+  } else {
+    unsigned Index = ID - FirstDeclID;
+
+    // Record the offset for this declaration
+    if (DeclOffsets.size() == Index)
+      DeclOffsets.push_back(Stream.GetCurrentBitNo());
+    else if (DeclOffsets.size() < Index) {
+      DeclOffsets.resize(Index+1);
+      DeclOffsets[Index] = Stream.GetCurrentBitNo();
+    }
   }
 
   // Build and emit a record for this declaration
   Record.clear();
-  W.Code = (pch::DeclCode)0;
+  W.Code = (serialization::DeclCode)0;
   W.AbbrevToUse = 0;
   W.Visit(D);
   if (DC) W.VisitDeclContext(DC, LexicalOffset, VisibleOffset);
@@ -1164,9 +1194,9 @@ void PCHWriter::WriteDecl(ASTContext &Context, Decl *D) {
   FlushStmts();
 
   // Note "external" declarations so that we can add them to a record in the
-  // PCH file later.
+  // AST file later.
   //
   // FIXME: This should be renamed, the predicate is much more complicated.
   if (isRequiredDecl(D, Context))
-    ExternalDefinitions.push_back(Index + 1);
+    ExternalDefinitions.push_back(ID);
 }
diff --git a/lib/Frontend/PCHWriterStmt.cpp b/lib/Serialization/ASTWriterStmt.cpp
index 75377286e5aa..7f2da6c225a3 100644
--- a/lib/Frontend/PCHWriterStmt.cpp
+++ b/lib/Serialization/ASTWriterStmt.cpp
@@ -1,4 +1,4 @@
-//===--- PCHWriterStmt.cpp - Statement and Expression Serialization -------===//
+//===--- ASTWriterStmt.cpp - Statement and Expression Serialization -------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Frontend/PCHWriter.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/StmtVisitor.h"
@@ -23,14 +23,14 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 namespace clang {
-  class PCHStmtWriter : public StmtVisitor<PCHStmtWriter, void> {
-    PCHWriter &Writer;
-    PCHWriter::RecordData &Record;
+  class ASTStmtWriter : public StmtVisitor<ASTStmtWriter, void> {
+    ASTWriter &Writer;
+    ASTWriter::RecordData &Record;
 
   public:
-    pch::StmtCode Code;
+    serialization::StmtCode Code;
 
-    PCHStmtWriter(PCHWriter &Writer, PCHWriter::RecordData &Record)
+    ASTStmtWriter(ASTWriter &Writer, ASTWriter::RecordData &Record)
       : Writer(Writer), Record(Record) { }
     
     void
@@ -115,6 +115,9 @@ namespace clang {
     void VisitObjCAtThrowStmt(ObjCAtThrowStmt *);
 
     // C++ Statements
+    void VisitCXXCatchStmt(CXXCatchStmt *S);
+    void VisitCXXTryStmt(CXXTryStmt *S);
+
     void VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E);
     void VisitCXXMemberCallExpr(CXXMemberCallExpr *E);
     void VisitCXXConstructExpr(CXXConstructExpr *E);
@@ -132,7 +135,6 @@ namespace clang {
     void VisitCXXThrowExpr(CXXThrowExpr *E);
     void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E);
     void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
-    void VisitCXXBindReferenceExpr(CXXBindReferenceExpr *E);
 
     void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
     void VisitCXXNewExpr(CXXNewExpr *E);
@@ -152,7 +154,7 @@ namespace clang {
   };
 }
 
-void PCHStmtWriter::
+void ASTStmtWriter::
 AddExplicitTemplateArgumentList(const ExplicitTemplateArgumentList &Args) {
   Writer.AddSourceLocation(Args.LAngleLoc, Record);
   Writer.AddSourceLocation(Args.RAngleLoc, Record);
@@ -160,16 +162,16 @@ AddExplicitTemplateArgumentList(const ExplicitTemplateArgumentList &Args) {
     Writer.AddTemplateArgumentLoc(Args.getTemplateArgs()[i], Record);
 }
 
-void PCHStmtWriter::VisitStmt(Stmt *S) {
+void ASTStmtWriter::VisitStmt(Stmt *S) {
 }
 
-void PCHStmtWriter::VisitNullStmt(NullStmt *S) {
+void ASTStmtWriter::VisitNullStmt(NullStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getSemiLoc(), Record);
-  Code = pch::STMT_NULL;
+  Code = serialization::STMT_NULL;
 }
 
-void PCHStmtWriter::VisitCompoundStmt(CompoundStmt *S) {
+void ASTStmtWriter::VisitCompoundStmt(CompoundStmt *S) {
   VisitStmt(S);
   Record.push_back(S->size());
   for (CompoundStmt::body_iterator CS = S->body_begin(), CSEnd = S->body_end();
@@ -177,15 +179,15 @@ void PCHStmtWriter::VisitCompoundStmt(CompoundStmt *S) {
     Writer.AddStmt(*CS);
   Writer.AddSourceLocation(S->getLBracLoc(), Record);
   Writer.AddSourceLocation(S->getRBracLoc(), Record);
-  Code = pch::STMT_COMPOUND;
+  Code = serialization::STMT_COMPOUND;
 }
 
-void PCHStmtWriter::VisitSwitchCase(SwitchCase *S) {
+void ASTStmtWriter::VisitSwitchCase(SwitchCase *S) {
   VisitStmt(S);
   Record.push_back(Writer.getSwitchCaseID(S));
 }
 
-void PCHStmtWriter::VisitCaseStmt(CaseStmt *S) {
+void ASTStmtWriter::VisitCaseStmt(CaseStmt *S) {
   VisitSwitchCase(S);
   Writer.AddStmt(S->getLHS());
   Writer.AddStmt(S->getRHS());
@@ -193,27 +195,27 @@ void PCHStmtWriter::VisitCaseStmt(CaseStmt *S) {
   Writer.AddSourceLocation(S->getCaseLoc(), Record);
   Writer.AddSourceLocation(S->getEllipsisLoc(), Record);
   Writer.AddSourceLocation(S->getColonLoc(), Record);
-  Code = pch::STMT_CASE;
+  Code = serialization::STMT_CASE;
 }
 
-void PCHStmtWriter::VisitDefaultStmt(DefaultStmt *S) {
+void ASTStmtWriter::VisitDefaultStmt(DefaultStmt *S) {
   VisitSwitchCase(S);
   Writer.AddStmt(S->getSubStmt());
   Writer.AddSourceLocation(S->getDefaultLoc(), Record);
   Writer.AddSourceLocation(S->getColonLoc(), Record);
-  Code = pch::STMT_DEFAULT;
+  Code = serialization::STMT_DEFAULT;
 }
 
-void PCHStmtWriter::VisitLabelStmt(LabelStmt *S) {
+void ASTStmtWriter::VisitLabelStmt(LabelStmt *S) {
   VisitStmt(S);
   Writer.AddIdentifierRef(S->getID(), Record);
   Writer.AddStmt(S->getSubStmt());
   Writer.AddSourceLocation(S->getIdentLoc(), Record);
   Record.push_back(Writer.GetLabelID(S));
-  Code = pch::STMT_LABEL;
+  Code = serialization::STMT_LABEL;
 }
 
-void PCHStmtWriter::VisitIfStmt(IfStmt *S) {
+void ASTStmtWriter::VisitIfStmt(IfStmt *S) {
   VisitStmt(S);
   Writer.AddDeclRef(S->getConditionVariable(), Record);
   Writer.AddStmt(S->getCond());
@@ -221,10 +223,10 @@ void PCHStmtWriter::VisitIfStmt(IfStmt *S) {
   Writer.AddStmt(S->getElse());
   Writer.AddSourceLocation(S->getIfLoc(), Record);
   Writer.AddSourceLocation(S->getElseLoc(), Record);
-  Code = pch::STMT_IF;
+  Code = serialization::STMT_IF;
 }
 
-void PCHStmtWriter::VisitSwitchStmt(SwitchStmt *S) {
+void ASTStmtWriter::VisitSwitchStmt(SwitchStmt *S) {
   VisitStmt(S);
   Writer.AddDeclRef(S->getConditionVariable(), Record);
   Writer.AddStmt(S->getCond());
@@ -233,29 +235,29 @@ void PCHStmtWriter::VisitSwitchStmt(SwitchStmt *S) {
   for (SwitchCase *SC = S->getSwitchCaseList(); SC;
        SC = SC->getNextSwitchCase())
     Record.push_back(Writer.RecordSwitchCaseID(SC));
-  Code = pch::STMT_SWITCH;
+  Code = serialization::STMT_SWITCH;
 }
 
-void PCHStmtWriter::VisitWhileStmt(WhileStmt *S) {
+void ASTStmtWriter::VisitWhileStmt(WhileStmt *S) {
   VisitStmt(S);
   Writer.AddDeclRef(S->getConditionVariable(), Record);
   Writer.AddStmt(S->getCond());
   Writer.AddStmt(S->getBody());
   Writer.AddSourceLocation(S->getWhileLoc(), Record);
-  Code = pch::STMT_WHILE;
+  Code = serialization::STMT_WHILE;
 }
 
-void PCHStmtWriter::VisitDoStmt(DoStmt *S) {
+void ASTStmtWriter::VisitDoStmt(DoStmt *S) {
   VisitStmt(S);
   Writer.AddStmt(S->getCond());
   Writer.AddStmt(S->getBody());
   Writer.AddSourceLocation(S->getDoLoc(), Record);
   Writer.AddSourceLocation(S->getWhileLoc(), Record);
   Writer.AddSourceLocation(S->getRParenLoc(), Record);
-  Code = pch::STMT_DO;
+  Code = serialization::STMT_DO;
 }
 
-void PCHStmtWriter::VisitForStmt(ForStmt *S) {
+void ASTStmtWriter::VisitForStmt(ForStmt *S) {
   VisitStmt(S);
   Writer.AddStmt(S->getInit());
   Writer.AddStmt(S->getCond());
@@ -265,56 +267,56 @@ void PCHStmtWriter::VisitForStmt(ForStmt *S) {
   Writer.AddSourceLocation(S->getForLoc(), Record);
   Writer.AddSourceLocation(S->getLParenLoc(), Record);
   Writer.AddSourceLocation(S->getRParenLoc(), Record);
-  Code = pch::STMT_FOR;
+  Code = serialization::STMT_FOR;
 }
 
-void PCHStmtWriter::VisitGotoStmt(GotoStmt *S) {
+void ASTStmtWriter::VisitGotoStmt(GotoStmt *S) {
   VisitStmt(S);
   Record.push_back(Writer.GetLabelID(S->getLabel()));
   Writer.AddSourceLocation(S->getGotoLoc(), Record);
   Writer.AddSourceLocation(S->getLabelLoc(), Record);
-  Code = pch::STMT_GOTO;
+  Code = serialization::STMT_GOTO;
 }
 
-void PCHStmtWriter::VisitIndirectGotoStmt(IndirectGotoStmt *S) {
+void ASTStmtWriter::VisitIndirectGotoStmt(IndirectGotoStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getGotoLoc(), Record);
   Writer.AddSourceLocation(S->getStarLoc(), Record);
   Writer.AddStmt(S->getTarget());
-  Code = pch::STMT_INDIRECT_GOTO;
+  Code = serialization::STMT_INDIRECT_GOTO;
 }
 
-void PCHStmtWriter::VisitContinueStmt(ContinueStmt *S) {
+void ASTStmtWriter::VisitContinueStmt(ContinueStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getContinueLoc(), Record);
-  Code = pch::STMT_CONTINUE;
+  Code = serialization::STMT_CONTINUE;
 }
 
-void PCHStmtWriter::VisitBreakStmt(BreakStmt *S) {
+void ASTStmtWriter::VisitBreakStmt(BreakStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getBreakLoc(), Record);
-  Code = pch::STMT_BREAK;
+  Code = serialization::STMT_BREAK;
 }
 
-void PCHStmtWriter::VisitReturnStmt(ReturnStmt *S) {
+void ASTStmtWriter::VisitReturnStmt(ReturnStmt *S) {
   VisitStmt(S);
   Writer.AddStmt(S->getRetValue());
   Writer.AddSourceLocation(S->getReturnLoc(), Record);
   Writer.AddDeclRef(S->getNRVOCandidate(), Record);
-  Code = pch::STMT_RETURN;
+  Code = serialization::STMT_RETURN;
 }
 
-void PCHStmtWriter::VisitDeclStmt(DeclStmt *S) {
+void ASTStmtWriter::VisitDeclStmt(DeclStmt *S) {
   VisitStmt(S);
   Writer.AddSourceLocation(S->getStartLoc(), Record);
   Writer.AddSourceLocation(S->getEndLoc(), Record);
   DeclGroupRef DG = S->getDeclGroup();
   for (DeclGroupRef::iterator D = DG.begin(), DEnd = DG.end(); D != DEnd; ++D)
     Writer.AddDeclRef(*D, Record);
-  Code = pch::STMT_DECL;
+  Code = serialization::STMT_DECL;
 }
 
-void PCHStmtWriter::VisitAsmStmt(AsmStmt *S) {
+void ASTStmtWriter::VisitAsmStmt(AsmStmt *S) {
   VisitStmt(S);
   Record.push_back(S->getNumOutputs());
   Record.push_back(S->getNumInputs());
@@ -344,29 +346,29 @@ void PCHStmtWriter::VisitAsmStmt(AsmStmt *S) {
   for (unsigned I = 0, N = S->getNumClobbers(); I != N; ++I)
     Writer.AddStmt(S->getClobber(I));
 
-  Code = pch::STMT_ASM;
+  Code = serialization::STMT_ASM;
 }
 
-void PCHStmtWriter::VisitExpr(Expr *E) {
+void ASTStmtWriter::VisitExpr(Expr *E) {
   VisitStmt(E);
   Writer.AddTypeRef(E->getType(), Record);
   Record.push_back(E->isTypeDependent());
   Record.push_back(E->isValueDependent());
 }
 
-void PCHStmtWriter::VisitPredefinedExpr(PredefinedExpr *E) {
+void ASTStmtWriter::VisitPredefinedExpr(PredefinedExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Record.push_back(E->getIdentType()); // FIXME: stable encoding
-  Code = pch::EXPR_PREDEFINED;
+  Code = serialization::EXPR_PREDEFINED;
 }
 
-void PCHStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) {
+void ASTStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) {
   VisitExpr(E);
 
   Record.push_back(E->hasQualifier());
   unsigned NumTemplateArgs = E->getNumTemplateArgs();
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgumentList() &&
+  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
          "Template args list with no args ?");
   Record.push_back(NumTemplateArgs);
 
@@ -376,35 +378,36 @@ void PCHStmtWriter::VisitDeclRefExpr(DeclRefExpr *E) {
   }
 
   if (NumTemplateArgs)
-    AddExplicitTemplateArgumentList(*E->getExplicitTemplateArgumentList());
+    AddExplicitTemplateArgumentList(E->getExplicitTemplateArgs());
 
   Writer.AddDeclRef(E->getDecl(), Record);
+  // FIXME: write DeclarationNameLoc.
   Writer.AddSourceLocation(E->getLocation(), Record);
-  Code = pch::EXPR_DECL_REF;
+  Code = serialization::EXPR_DECL_REF;
 }
 
-void PCHStmtWriter::VisitIntegerLiteral(IntegerLiteral *E) {
+void ASTStmtWriter::VisitIntegerLiteral(IntegerLiteral *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Writer.AddAPInt(E->getValue(), Record);
-  Code = pch::EXPR_INTEGER_LITERAL;
+  Code = serialization::EXPR_INTEGER_LITERAL;
 }
 
-void PCHStmtWriter::VisitFloatingLiteral(FloatingLiteral *E) {
+void ASTStmtWriter::VisitFloatingLiteral(FloatingLiteral *E) {
   VisitExpr(E);
   Writer.AddAPFloat(E->getValue(), Record);
   Record.push_back(E->isExact());
   Writer.AddSourceLocation(E->getLocation(), Record);
-  Code = pch::EXPR_FLOATING_LITERAL;
+  Code = serialization::EXPR_FLOATING_LITERAL;
 }
 
-void PCHStmtWriter::VisitImaginaryLiteral(ImaginaryLiteral *E) {
+void ASTStmtWriter::VisitImaginaryLiteral(ImaginaryLiteral *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getSubExpr());
-  Code = pch::EXPR_IMAGINARY_LITERAL;
+  Code = serialization::EXPR_IMAGINARY_LITERAL;
 }
 
-void PCHStmtWriter::VisitStringLiteral(StringLiteral *E) {
+void ASTStmtWriter::VisitStringLiteral(StringLiteral *E) {
   VisitExpr(E);
   Record.push_back(E->getByteLength());
   Record.push_back(E->getNumConcatenated());
@@ -412,49 +415,48 @@ void PCHStmtWriter::VisitStringLiteral(StringLiteral *E) {
   // FIXME: String data should be stored as a blob at the end of the
   // StringLiteral. However, we can't do so now because we have no
   // provision for coping with abbreviations when we're jumping around
-  // the PCH file during deserialization.
-  Record.insert(Record.end(),
-                E->getStrData(), E->getStrData() + E->getByteLength());
+  // the AST file during deserialization.
+  Record.append(E->getString().begin(), E->getString().end());
   for (unsigned I = 0, N = E->getNumConcatenated(); I != N; ++I)
     Writer.AddSourceLocation(E->getStrTokenLoc(I), Record);
-  Code = pch::EXPR_STRING_LITERAL;
+  Code = serialization::EXPR_STRING_LITERAL;
 }
 
-void PCHStmtWriter::VisitCharacterLiteral(CharacterLiteral *E) {
+void ASTStmtWriter::VisitCharacterLiteral(CharacterLiteral *E) {
   VisitExpr(E);
   Record.push_back(E->getValue());
   Writer.AddSourceLocation(E->getLocation(), Record);
   Record.push_back(E->isWide());
-  Code = pch::EXPR_CHARACTER_LITERAL;
+  Code = serialization::EXPR_CHARACTER_LITERAL;
 }
 
-void PCHStmtWriter::VisitParenExpr(ParenExpr *E) {
+void ASTStmtWriter::VisitParenExpr(ParenExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLParen(), Record);
   Writer.AddSourceLocation(E->getRParen(), Record);
   Writer.AddStmt(E->getSubExpr());
-  Code = pch::EXPR_PAREN;
+  Code = serialization::EXPR_PAREN;
 }
 
-void PCHStmtWriter::VisitParenListExpr(ParenListExpr *E) {
+void ASTStmtWriter::VisitParenListExpr(ParenListExpr *E) {
   VisitExpr(E);
   Record.push_back(E->NumExprs);
   for (unsigned i=0; i != E->NumExprs; ++i)
     Writer.AddStmt(E->Exprs[i]);
   Writer.AddSourceLocation(E->LParenLoc, Record);
   Writer.AddSourceLocation(E->RParenLoc, Record);
-  Code = pch::EXPR_PAREN_LIST;
+  Code = serialization::EXPR_PAREN_LIST;
 }
 
-void PCHStmtWriter::VisitUnaryOperator(UnaryOperator *E) {
+void ASTStmtWriter::VisitUnaryOperator(UnaryOperator *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getSubExpr());
   Record.push_back(E->getOpcode()); // FIXME: stable encoding
   Writer.AddSourceLocation(E->getOperatorLoc(), Record);
-  Code = pch::EXPR_UNARY_OPERATOR;
+  Code = serialization::EXPR_UNARY_OPERATOR;
 }
 
-void PCHStmtWriter::VisitOffsetOfExpr(OffsetOfExpr *E) {
+void ASTStmtWriter::VisitOffsetOfExpr(OffsetOfExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumComponents());
   Record.push_back(E->getNumExpressions());
@@ -480,17 +482,16 @@ void PCHStmtWriter::VisitOffsetOfExpr(OffsetOfExpr *E) {
       break;
         
     case OffsetOfExpr::OffsetOfNode::Base:
-      // FIXME: Implement this!
-      llvm_unreachable("PCH for offsetof(base-specifier) not implemented");
+      Writer.AddCXXBaseSpecifier(*ON.getBase(), Record);
       break;
     }
   }
   for (unsigned I = 0, N = E->getNumExpressions(); I != N; ++I)
     Writer.AddStmt(E->getIndexExpr(I));
-  Code = pch::EXPR_OFFSETOF;
+  Code = serialization::EXPR_OFFSETOF;
 }
 
-void PCHStmtWriter::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
+void ASTStmtWriter::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
   VisitExpr(E);
   Record.push_back(E->isSizeOf());
   if (E->isArgumentType())
@@ -501,18 +502,18 @@ void PCHStmtWriter::VisitSizeOfAlignOfExpr(SizeOfAlignOfExpr *E) {
   }
   Writer.AddSourceLocation(E->getOperatorLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_SIZEOF_ALIGN_OF;
+  Code = serialization::EXPR_SIZEOF_ALIGN_OF;
 }
 
-void PCHStmtWriter::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
+void ASTStmtWriter::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getLHS());
   Writer.AddStmt(E->getRHS());
   Writer.AddSourceLocation(E->getRBracketLoc(), Record);
-  Code = pch::EXPR_ARRAY_SUBSCRIPT;
+  Code = serialization::EXPR_ARRAY_SUBSCRIPT;
 }
 
-void PCHStmtWriter::VisitCallExpr(CallExpr *E) {
+void ASTStmtWriter::VisitCallExpr(CallExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
@@ -520,10 +521,10 @@ void PCHStmtWriter::VisitCallExpr(CallExpr *E) {
   for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg)
     Writer.AddStmt(*Arg);
-  Code = pch::EXPR_CALL;
+  Code = serialization::EXPR_CALL;
 }
 
-void PCHStmtWriter::VisitMemberExpr(MemberExpr *E) {
+void ASTStmtWriter::VisitMemberExpr(MemberExpr *E) {
   // Don't call VisitExpr, we'll write everything here.
 
   Record.push_back(E->hasQualifier());
@@ -533,7 +534,7 @@ void PCHStmtWriter::VisitMemberExpr(MemberExpr *E) {
   }
 
   unsigned NumTemplateArgs = E->getNumTemplateArgs();
-  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgumentList() &&
+  assert((NumTemplateArgs != 0) == E->hasExplicitTemplateArgs() &&
          "Template args list with no args ?");
   Record.push_back(NumTemplateArgs);
   if (NumTemplateArgs) {
@@ -550,92 +551,94 @@ void PCHStmtWriter::VisitMemberExpr(MemberExpr *E) {
   Writer.AddTypeRef(E->getType(), Record);
   Writer.AddStmt(E->getBase());
   Writer.AddDeclRef(E->getMemberDecl(), Record);
+  // FIXME: write DeclarationNameLoc.
   Writer.AddSourceLocation(E->getMemberLoc(), Record);
   Record.push_back(E->isArrow());
-  Code = pch::EXPR_MEMBER;
+  Code = serialization::EXPR_MEMBER;
 }
 
-void PCHStmtWriter::VisitObjCIsaExpr(ObjCIsaExpr *E) {
+void ASTStmtWriter::VisitObjCIsaExpr(ObjCIsaExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getBase());
   Writer.AddSourceLocation(E->getIsaMemberLoc(), Record);
   Record.push_back(E->isArrow());
-  Code = pch::EXPR_OBJC_ISA;
+  Code = serialization::EXPR_OBJC_ISA;
 }
 
-void PCHStmtWriter::VisitCastExpr(CastExpr *E) {
+void ASTStmtWriter::VisitCastExpr(CastExpr *E) {
   VisitExpr(E);
+  Record.push_back(E->path_size());
   Writer.AddStmt(E->getSubExpr());
   Record.push_back(E->getCastKind()); // FIXME: stable encoding
-  CXXBaseSpecifierArray &BasePath = E->getBasePath();
-  Record.push_back(BasePath.size());
-  for (CXXBaseSpecifierArray::iterator I = BasePath.begin(), E = BasePath.end();
-       I != E; ++I)
-    Writer.AddCXXBaseSpecifier(**I, Record);
+
+  for (CastExpr::path_iterator
+         PI = E->path_begin(), PE = E->path_end(); PI != PE; ++PI)
+    Writer.AddCXXBaseSpecifier(**PI, Record);
 }
 
-void PCHStmtWriter::VisitBinaryOperator(BinaryOperator *E) {
+void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getLHS());
   Writer.AddStmt(E->getRHS());
   Record.push_back(E->getOpcode()); // FIXME: stable encoding
   Writer.AddSourceLocation(E->getOperatorLoc(), Record);
-  Code = pch::EXPR_BINARY_OPERATOR;
+  Code = serialization::EXPR_BINARY_OPERATOR;
 }
 
-void PCHStmtWriter::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
+void ASTStmtWriter::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
   VisitBinaryOperator(E);
   Writer.AddTypeRef(E->getComputationLHSType(), Record);
   Writer.AddTypeRef(E->getComputationResultType(), Record);
-  Code = pch::EXPR_COMPOUND_ASSIGN_OPERATOR;
+  Code = serialization::EXPR_COMPOUND_ASSIGN_OPERATOR;
 }
 
-void PCHStmtWriter::VisitConditionalOperator(ConditionalOperator *E) {
+void ASTStmtWriter::VisitConditionalOperator(ConditionalOperator *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getCond());
   Writer.AddStmt(E->getLHS());
   Writer.AddStmt(E->getRHS());
+  Writer.AddStmt(E->getSAVE());
   Writer.AddSourceLocation(E->getQuestionLoc(), Record);
   Writer.AddSourceLocation(E->getColonLoc(), Record);
-  Code = pch::EXPR_CONDITIONAL_OPERATOR;
+  Code = serialization::EXPR_CONDITIONAL_OPERATOR;
 }
 
-void PCHStmtWriter::VisitImplicitCastExpr(ImplicitCastExpr *E) {
+void ASTStmtWriter::VisitImplicitCastExpr(ImplicitCastExpr *E) {
   VisitCastExpr(E);
-  Record.push_back(E->isLvalueCast());
-  Code = pch::EXPR_IMPLICIT_CAST;
+  Record.push_back(E->getValueKind());
+  Code = serialization::EXPR_IMPLICIT_CAST;
 }
 
-void PCHStmtWriter::VisitExplicitCastExpr(ExplicitCastExpr *E) {
+void ASTStmtWriter::VisitExplicitCastExpr(ExplicitCastExpr *E) {
   VisitCastExpr(E);
   Writer.AddTypeSourceInfo(E->getTypeInfoAsWritten(), Record);
 }
 
-void PCHStmtWriter::VisitCStyleCastExpr(CStyleCastExpr *E) {
+void ASTStmtWriter::VisitCStyleCastExpr(CStyleCastExpr *E) {
   VisitExplicitCastExpr(E);
   Writer.AddSourceLocation(E->getLParenLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_CSTYLE_CAST;
+  Code = serialization::EXPR_CSTYLE_CAST;
 }
 
-void PCHStmtWriter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
+void ASTStmtWriter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLParenLoc(), Record);
   Writer.AddTypeSourceInfo(E->getTypeSourceInfo(), Record);
   Writer.AddStmt(E->getInitializer());
   Record.push_back(E->isFileScope());
-  Code = pch::EXPR_COMPOUND_LITERAL;
+  Code = serialization::EXPR_COMPOUND_LITERAL;
 }
 
-void PCHStmtWriter::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
+void ASTStmtWriter::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getBase());
   Writer.AddIdentifierRef(&E->getAccessor(), Record);
   Writer.AddSourceLocation(E->getAccessorLoc(), Record);
-  Code = pch::EXPR_EXT_VECTOR_ELEMENT;
+  Code = serialization::EXPR_EXT_VECTOR_ELEMENT;
 }
 
-void PCHStmtWriter::VisitInitListExpr(InitListExpr *E) {
+void ASTStmtWriter::VisitInitListExpr(InitListExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumInits());
   for (unsigned I = 0, N = E->getNumInits(); I != N; ++I)
@@ -645,10 +648,10 @@ void PCHStmtWriter::VisitInitListExpr(InitListExpr *E) {
   Writer.AddSourceLocation(E->getRBraceLoc(), Record);
   Writer.AddDeclRef(E->getInitializedFieldInUnion(), Record);
   Record.push_back(E->hadArrayRangeDesignator());
-  Code = pch::EXPR_INIT_LIST;
+  Code = serialization::EXPR_INIT_LIST;
 }
 
-void PCHStmtWriter::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
+void ASTStmtWriter::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumSubExprs());
   for (unsigned I = 0, N = E->getNumSubExprs(); I != N; ++I)
@@ -660,166 +663,167 @@ void PCHStmtWriter::VisitDesignatedInitExpr(DesignatedInitExpr *E) {
        D != DEnd; ++D) {
     if (D->isFieldDesignator()) {
       if (FieldDecl *Field = D->getField()) {
-        Record.push_back(pch::DESIG_FIELD_DECL);
+        Record.push_back(serialization::DESIG_FIELD_DECL);
         Writer.AddDeclRef(Field, Record);
       } else {
-        Record.push_back(pch::DESIG_FIELD_NAME);
+        Record.push_back(serialization::DESIG_FIELD_NAME);
         Writer.AddIdentifierRef(D->getFieldName(), Record);
       }
       Writer.AddSourceLocation(D->getDotLoc(), Record);
       Writer.AddSourceLocation(D->getFieldLoc(), Record);
     } else if (D->isArrayDesignator()) {
-      Record.push_back(pch::DESIG_ARRAY);
+      Record.push_back(serialization::DESIG_ARRAY);
       Record.push_back(D->getFirstExprIndex());
       Writer.AddSourceLocation(D->getLBracketLoc(), Record);
       Writer.AddSourceLocation(D->getRBracketLoc(), Record);
     } else {
       assert(D->isArrayRangeDesignator() && "Unknown designator");
-      Record.push_back(pch::DESIG_ARRAY_RANGE);
+      Record.push_back(serialization::DESIG_ARRAY_RANGE);
       Record.push_back(D->getFirstExprIndex());
       Writer.AddSourceLocation(D->getLBracketLoc(), Record);
       Writer.AddSourceLocation(D->getEllipsisLoc(), Record);
       Writer.AddSourceLocation(D->getRBracketLoc(), Record);
     }
   }
-  Code = pch::EXPR_DESIGNATED_INIT;
+  Code = serialization::EXPR_DESIGNATED_INIT;
 }
 
-void PCHStmtWriter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
+void ASTStmtWriter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
   VisitExpr(E);
-  Code = pch::EXPR_IMPLICIT_VALUE_INIT;
+  Code = serialization::EXPR_IMPLICIT_VALUE_INIT;
 }
 
-void PCHStmtWriter::VisitVAArgExpr(VAArgExpr *E) {
+void ASTStmtWriter::VisitVAArgExpr(VAArgExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getSubExpr());
+  Writer.AddTypeSourceInfo(E->getWrittenTypeInfo(), Record);
   Writer.AddSourceLocation(E->getBuiltinLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_VA_ARG;
+  Code = serialization::EXPR_VA_ARG;
 }
 
-void PCHStmtWriter::VisitAddrLabelExpr(AddrLabelExpr *E) {
+void ASTStmtWriter::VisitAddrLabelExpr(AddrLabelExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getAmpAmpLoc(), Record);
   Writer.AddSourceLocation(E->getLabelLoc(), Record);
   Record.push_back(Writer.GetLabelID(E->getLabel()));
-  Code = pch::EXPR_ADDR_LABEL;
+  Code = serialization::EXPR_ADDR_LABEL;
 }
 
-void PCHStmtWriter::VisitStmtExpr(StmtExpr *E) {
+void ASTStmtWriter::VisitStmtExpr(StmtExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getSubStmt());
   Writer.AddSourceLocation(E->getLParenLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_STMT;
+  Code = serialization::EXPR_STMT;
 }
 
-void PCHStmtWriter::VisitTypesCompatibleExpr(TypesCompatibleExpr *E) {
+void ASTStmtWriter::VisitTypesCompatibleExpr(TypesCompatibleExpr *E) {
   VisitExpr(E);
-  Writer.AddTypeRef(E->getArgType1(), Record);
-  Writer.AddTypeRef(E->getArgType2(), Record);
+  Writer.AddTypeSourceInfo(E->getArgTInfo1(), Record);
+  Writer.AddTypeSourceInfo(E->getArgTInfo2(), Record);
   Writer.AddSourceLocation(E->getBuiltinLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_TYPES_COMPATIBLE;
+  Code = serialization::EXPR_TYPES_COMPATIBLE;
 }
 
-void PCHStmtWriter::VisitChooseExpr(ChooseExpr *E) {
+void ASTStmtWriter::VisitChooseExpr(ChooseExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getCond());
   Writer.AddStmt(E->getLHS());
   Writer.AddStmt(E->getRHS());
   Writer.AddSourceLocation(E->getBuiltinLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_CHOOSE;
+  Code = serialization::EXPR_CHOOSE;
 }
 
-void PCHStmtWriter::VisitGNUNullExpr(GNUNullExpr *E) {
+void ASTStmtWriter::VisitGNUNullExpr(GNUNullExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getTokenLocation(), Record);
-  Code = pch::EXPR_GNU_NULL;
+  Code = serialization::EXPR_GNU_NULL;
 }
 
-void PCHStmtWriter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
+void ASTStmtWriter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumSubExprs());
   for (unsigned I = 0, N = E->getNumSubExprs(); I != N; ++I)
     Writer.AddStmt(E->getExpr(I));
   Writer.AddSourceLocation(E->getBuiltinLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_SHUFFLE_VECTOR;
+  Code = serialization::EXPR_SHUFFLE_VECTOR;
 }
 
-void PCHStmtWriter::VisitBlockExpr(BlockExpr *E) {
+void ASTStmtWriter::VisitBlockExpr(BlockExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getBlockDecl(), Record);
   Record.push_back(E->hasBlockDeclRefExprs());
-  Code = pch::EXPR_BLOCK;
+  Code = serialization::EXPR_BLOCK;
 }
 
-void PCHStmtWriter::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
+void ASTStmtWriter::VisitBlockDeclRefExpr(BlockDeclRefExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getDecl(), Record);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Record.push_back(E->isByRef());
   Record.push_back(E->isConstQualAdded());
   Writer.AddStmt(E->getCopyConstructorExpr());
-  Code = pch::EXPR_BLOCK_DECL_REF;
+  Code = serialization::EXPR_BLOCK_DECL_REF;
 }
 
 //===----------------------------------------------------------------------===//
 // Objective-C Expressions and Statements.
 //===----------------------------------------------------------------------===//
 
-void PCHStmtWriter::VisitObjCStringLiteral(ObjCStringLiteral *E) {
+void ASTStmtWriter::VisitObjCStringLiteral(ObjCStringLiteral *E) {
   VisitExpr(E);
   Writer.AddStmt(E->getString());
   Writer.AddSourceLocation(E->getAtLoc(), Record);
-  Code = pch::EXPR_OBJC_STRING_LITERAL;
+  Code = serialization::EXPR_OBJC_STRING_LITERAL;
 }
 
-void PCHStmtWriter::VisitObjCEncodeExpr(ObjCEncodeExpr *E) {
+void ASTStmtWriter::VisitObjCEncodeExpr(ObjCEncodeExpr *E) {
   VisitExpr(E);
   Writer.AddTypeSourceInfo(E->getEncodedTypeSourceInfo(), Record);
   Writer.AddSourceLocation(E->getAtLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_OBJC_ENCODE;
+  Code = serialization::EXPR_OBJC_ENCODE;
 }
 
-void PCHStmtWriter::VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
+void ASTStmtWriter::VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
   VisitExpr(E);
   Writer.AddSelectorRef(E->getSelector(), Record);
   Writer.AddSourceLocation(E->getAtLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_OBJC_SELECTOR_EXPR;
+  Code = serialization::EXPR_OBJC_SELECTOR_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
+void ASTStmtWriter::VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getProtocol(), Record);
   Writer.AddSourceLocation(E->getAtLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_OBJC_PROTOCOL_EXPR;
+  Code = serialization::EXPR_OBJC_PROTOCOL_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
+void ASTStmtWriter::VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getDecl(), Record);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Writer.AddStmt(E->getBase());
   Record.push_back(E->isArrow());
   Record.push_back(E->isFreeIvar());
-  Code = pch::EXPR_OBJC_IVAR_REF_EXPR;
+  Code = serialization::EXPR_OBJC_IVAR_REF_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
+void ASTStmtWriter::VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getProperty(), Record);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Writer.AddStmt(E->getBase());
-  Code = pch::EXPR_OBJC_PROPERTY_REF_EXPR;
+  Code = serialization::EXPR_OBJC_PROPERTY_REF_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCImplicitSetterGetterRefExpr(
+void ASTStmtWriter::VisitObjCImplicitSetterGetterRefExpr(
                                   ObjCImplicitSetterGetterRefExpr *E) {
   VisitExpr(E);
   Writer.AddDeclRef(E->getGetterMethod(), Record);
@@ -830,10 +834,10 @@ void PCHStmtWriter::VisitObjCImplicitSetterGetterRefExpr(
   Writer.AddStmt(E->getBase());
   Writer.AddSourceLocation(E->getLocation(), Record);
   Writer.AddSourceLocation(E->getClassLoc(), Record);
-  Code = pch::EXPR_OBJC_KVC_REF_EXPR;
+  Code = serialization::EXPR_OBJC_KVC_REF_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
+void ASTStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
   Record.push_back((unsigned)E->getReceiverKind()); // FIXME: stable encoding
@@ -867,40 +871,40 @@ void PCHStmtWriter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
   for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end();
        Arg != ArgEnd; ++Arg)
     Writer.AddStmt(*Arg);
-  Code = pch::EXPR_OBJC_MESSAGE_EXPR;
+  Code = serialization::EXPR_OBJC_MESSAGE_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCSuperExpr(ObjCSuperExpr *E) {
+void ASTStmtWriter::VisitObjCSuperExpr(ObjCSuperExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLoc(), Record);
-  Code = pch::EXPR_OBJC_SUPER_EXPR;
+  Code = serialization::EXPR_OBJC_SUPER_EXPR;
 }
 
-void PCHStmtWriter::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
+void ASTStmtWriter::VisitObjCForCollectionStmt(ObjCForCollectionStmt *S) {
   VisitStmt(S);
   Writer.AddStmt(S->getElement());
   Writer.AddStmt(S->getCollection());
   Writer.AddStmt(S->getBody());
   Writer.AddSourceLocation(S->getForLoc(), Record);
   Writer.AddSourceLocation(S->getRParenLoc(), Record);
-  Code = pch::STMT_OBJC_FOR_COLLECTION;
+  Code = serialization::STMT_OBJC_FOR_COLLECTION;
 }
 
-void PCHStmtWriter::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
+void ASTStmtWriter::VisitObjCAtCatchStmt(ObjCAtCatchStmt *S) {
   Writer.AddStmt(S->getCatchBody());
   Writer.AddDeclRef(S->getCatchParamDecl(), Record);
   Writer.AddSourceLocation(S->getAtCatchLoc(), Record);
   Writer.AddSourceLocation(S->getRParenLoc(), Record);
-  Code = pch::STMT_OBJC_CATCH;
+  Code = serialization::STMT_OBJC_CATCH;
 }
 
-void PCHStmtWriter::VisitObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
+void ASTStmtWriter::VisitObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
   Writer.AddStmt(S->getFinallyBody());
   Writer.AddSourceLocation(S->getAtFinallyLoc(), Record);
-  Code = pch::STMT_OBJC_FINALLY;
+  Code = serialization::STMT_OBJC_FINALLY;
 }
 
-void PCHStmtWriter::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
+void ASTStmtWriter::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
   Record.push_back(S->getNumCatchStmts());
   Record.push_back(S->getFinallyStmt() != 0);
   Writer.AddStmt(S->getTryBody());
@@ -909,38 +913,56 @@ void PCHStmtWriter::VisitObjCAtTryStmt(ObjCAtTryStmt *S) {
   if (S->getFinallyStmt())
     Writer.AddStmt(S->getFinallyStmt());
   Writer.AddSourceLocation(S->getAtTryLoc(), Record);
-  Code = pch::STMT_OBJC_AT_TRY;
+  Code = serialization::STMT_OBJC_AT_TRY;
 }
 
-void PCHStmtWriter::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
+void ASTStmtWriter::VisitObjCAtSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
   Writer.AddStmt(S->getSynchExpr());
   Writer.AddStmt(S->getSynchBody());
   Writer.AddSourceLocation(S->getAtSynchronizedLoc(), Record);
-  Code = pch::STMT_OBJC_AT_SYNCHRONIZED;
+  Code = serialization::STMT_OBJC_AT_SYNCHRONIZED;
 }
 
-void PCHStmtWriter::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
+void ASTStmtWriter::VisitObjCAtThrowStmt(ObjCAtThrowStmt *S) {
   Writer.AddStmt(S->getThrowExpr());
   Writer.AddSourceLocation(S->getThrowLoc(), Record);
-  Code = pch::STMT_OBJC_AT_THROW;
+  Code = serialization::STMT_OBJC_AT_THROW;
 }
 
 //===----------------------------------------------------------------------===//
 // C++ Expressions and Statements.
 //===----------------------------------------------------------------------===//
 
-void PCHStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
+void ASTStmtWriter::VisitCXXCatchStmt(CXXCatchStmt *S) {
+  VisitStmt(S);
+  Writer.AddSourceLocation(S->getCatchLoc(), Record);
+  Writer.AddDeclRef(S->getExceptionDecl(), Record);
+  Writer.AddStmt(S->getHandlerBlock());
+  Code = serialization::STMT_CXX_CATCH;
+}
+
+void ASTStmtWriter::VisitCXXTryStmt(CXXTryStmt *S) {
+  VisitStmt(S);
+  Record.push_back(S->getNumHandlers());
+  Writer.AddSourceLocation(S->getTryLoc(), Record);
+  Writer.AddStmt(S->getTryBlock());
+  for (unsigned i = 0, e = S->getNumHandlers(); i != e; ++i)
+    Writer.AddStmt(S->getHandler(i));
+  Code = serialization::STMT_CXX_TRY;
+}
+
+void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   Record.push_back(E->getOperator());
-  Code = pch::EXPR_CXX_OPERATOR_CALL;
+  Code = serialization::EXPR_CXX_OPERATOR_CALL;
 }
 
-void PCHStmtWriter::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
+void ASTStmtWriter::VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
   VisitCallExpr(E);
-  Code = pch::EXPR_CXX_MEMBER_CALL;
+  Code = serialization::EXPR_CXX_MEMBER_CALL;
 }
 
-void PCHStmtWriter::VisitCXXConstructExpr(CXXConstructExpr *E) {
+void ASTStmtWriter::VisitCXXConstructExpr(CXXConstructExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getNumArgs());
   for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
@@ -950,88 +972,88 @@ void PCHStmtWriter::VisitCXXConstructExpr(CXXConstructExpr *E) {
   Record.push_back(E->isElidable());
   Record.push_back(E->requiresZeroInitialization());
   Record.push_back(E->getConstructionKind()); // FIXME: stable encoding
-  Code = pch::EXPR_CXX_CONSTRUCT;
+  Code = serialization::EXPR_CXX_CONSTRUCT;
 }
 
-void PCHStmtWriter::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E) {
+void ASTStmtWriter::VisitCXXTemporaryObjectExpr(CXXTemporaryObjectExpr *E) {
   VisitCXXConstructExpr(E);
   Writer.AddSourceLocation(E->getTypeBeginLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_CXX_TEMPORARY_OBJECT;
+  Code = serialization::EXPR_CXX_TEMPORARY_OBJECT;
 }
 
-void PCHStmtWriter::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
+void ASTStmtWriter::VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
   VisitExplicitCastExpr(E);
   Writer.AddSourceLocation(E->getOperatorLoc(), Record);
 }
 
-void PCHStmtWriter::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) {
+void ASTStmtWriter::VisitCXXStaticCastExpr(CXXStaticCastExpr *E) {
   VisitCXXNamedCastExpr(E);
-  Code = pch::EXPR_CXX_STATIC_CAST;
+  Code = serialization::EXPR_CXX_STATIC_CAST;
 }
 
-void PCHStmtWriter::VisitCXXDynamicCastExpr(CXXDynamicCastExpr *E) {
+void ASTStmtWriter::VisitCXXDynamicCastExpr(CXXDynamicCastExpr *E) {
   VisitCXXNamedCastExpr(E);
-  Code = pch::EXPR_CXX_DYNAMIC_CAST;
+  Code = serialization::EXPR_CXX_DYNAMIC_CAST;
 }
 
-void PCHStmtWriter::VisitCXXReinterpretCastExpr(CXXReinterpretCastExpr *E) {
+void ASTStmtWriter::VisitCXXReinterpretCastExpr(CXXReinterpretCastExpr *E) {
   VisitCXXNamedCastExpr(E);
-  Code = pch::EXPR_CXX_REINTERPRET_CAST;
+  Code = serialization::EXPR_CXX_REINTERPRET_CAST;
 }
 
-void PCHStmtWriter::VisitCXXConstCastExpr(CXXConstCastExpr *E) {
+void ASTStmtWriter::VisitCXXConstCastExpr(CXXConstCastExpr *E) {
   VisitCXXNamedCastExpr(E);
-  Code = pch::EXPR_CXX_CONST_CAST;
+  Code = serialization::EXPR_CXX_CONST_CAST;
 }
 
-void PCHStmtWriter::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E) {
+void ASTStmtWriter::VisitCXXFunctionalCastExpr(CXXFunctionalCastExpr *E) {
   VisitExplicitCastExpr(E);
   Writer.AddSourceLocation(E->getTypeBeginLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_CXX_FUNCTIONAL_CAST;
+  Code = serialization::EXPR_CXX_FUNCTIONAL_CAST;
 }
 
-void PCHStmtWriter::VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
+void ASTStmtWriter::VisitCXXBoolLiteralExpr(CXXBoolLiteralExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getValue());
   Writer.AddSourceLocation(E->getLocation(), Record);
-  Code = pch::EXPR_CXX_BOOL_LITERAL;
+  Code = serialization::EXPR_CXX_BOOL_LITERAL;
 }
 
-void PCHStmtWriter::VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E) {
+void ASTStmtWriter::VisitCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLocation(), Record);
-  Code = pch::EXPR_CXX_NULL_PTR_LITERAL;
+  Code = serialization::EXPR_CXX_NULL_PTR_LITERAL;
 }
 
-void PCHStmtWriter::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
+void ASTStmtWriter::VisitCXXTypeidExpr(CXXTypeidExpr *E) {
   VisitExpr(E);
   Writer.AddSourceRange(E->getSourceRange(), Record);
   if (E->isTypeOperand()) {
     Writer.AddTypeSourceInfo(E->getTypeOperandSourceInfo(), Record);
-    Code = pch::EXPR_CXX_TYPEID_TYPE;
+    Code = serialization::EXPR_CXX_TYPEID_TYPE;
   } else {
     Writer.AddStmt(E->getExprOperand());
-    Code = pch::EXPR_CXX_TYPEID_EXPR;
+    Code = serialization::EXPR_CXX_TYPEID_EXPR;
   }
 }
 
-void PCHStmtWriter::VisitCXXThisExpr(CXXThisExpr *E) {
+void ASTStmtWriter::VisitCXXThisExpr(CXXThisExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Record.push_back(E->isImplicit());
-  Code = pch::EXPR_CXX_THIS;
+  Code = serialization::EXPR_CXX_THIS;
 }
 
-void PCHStmtWriter::VisitCXXThrowExpr(CXXThrowExpr *E) {
+void ASTStmtWriter::VisitCXXThrowExpr(CXXThrowExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getThrowLoc(), Record);
   Writer.AddStmt(E->getSubExpr());
-  Code = pch::EXPR_CXX_THROW;
+  Code = serialization::EXPR_CXX_THROW;
 }
 
-void PCHStmtWriter::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
+void ASTStmtWriter::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   VisitExpr(E);
 
   bool HasOtherExprStored = E->Param.getInt();
@@ -1042,32 +1064,24 @@ void PCHStmtWriter::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   Writer.AddDeclRef(E->getParam(), Record);
   Writer.AddSourceLocation(E->getUsedLocation(), Record);
 
-  Code = pch::EXPR_CXX_DEFAULT_ARG;
+  Code = serialization::EXPR_CXX_DEFAULT_ARG;
 }
 
-void PCHStmtWriter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
+void ASTStmtWriter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
   VisitExpr(E);
   Writer.AddCXXTemporary(E->getTemporary(), Record);
   Writer.AddStmt(E->getSubExpr());
-  Code = pch::EXPR_CXX_BIND_TEMPORARY;
-}
-
-void PCHStmtWriter::VisitCXXBindReferenceExpr(CXXBindReferenceExpr *E) {
-  VisitExpr(E);
-  Writer.AddStmt(E->getSubExpr());
-  Record.push_back(E->extendsLifetime());
-  Record.push_back(E->requiresTemporaryCopy());
-  Code = pch::EXPR_CXX_BIND_REFERENCE;
+  Code = serialization::EXPR_CXX_BIND_TEMPORARY;
 }
 
-void PCHStmtWriter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
+void ASTStmtWriter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
   VisitExpr(E);
   Writer.AddSourceLocation(E->getTypeBeginLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_CXX_SCALAR_VALUE_INIT;
+  Code = serialization::EXPR_CXX_SCALAR_VALUE_INIT;
 }
 
-void PCHStmtWriter::VisitCXXNewExpr(CXXNewExpr *E) {
+void ASTStmtWriter::VisitCXXNewExpr(CXXNewExpr *E) {
   VisitExpr(E);
   Record.push_back(E->isGlobalNew());
   Record.push_back(E->hasInitializer());
@@ -1084,10 +1098,10 @@ void PCHStmtWriter::VisitCXXNewExpr(CXXNewExpr *E) {
        I != e; ++I)
     Writer.AddStmt(*I);
   
-  Code = pch::EXPR_CXX_NEW;
+  Code = serialization::EXPR_CXX_NEW;
 }
 
-void PCHStmtWriter::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+void ASTStmtWriter::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   VisitExpr(E);
   Record.push_back(E->isGlobalDelete());
   Record.push_back(E->isArrayForm());
@@ -1095,10 +1109,10 @@ void PCHStmtWriter::VisitCXXDeleteExpr(CXXDeleteExpr *E) {
   Writer.AddStmt(E->getArgument());
   Writer.AddSourceLocation(E->getSourceRange().getBegin(), Record);
   
-  Code = pch::EXPR_CXX_DELETE;
+  Code = serialization::EXPR_CXX_DELETE;
 }
 
-void PCHStmtWriter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
+void ASTStmtWriter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   VisitExpr(E);
 
   Writer.AddStmt(E->getBase());
@@ -1117,30 +1131,29 @@ void PCHStmtWriter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
   else
     Writer.AddTypeSourceInfo(E->getDestroyedTypeInfo(), Record);
 
-  Code = pch::EXPR_CXX_PSEUDO_DESTRUCTOR;
+  Code = serialization::EXPR_CXX_PSEUDO_DESTRUCTOR;
 }
 
-void PCHStmtWriter::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
+void ASTStmtWriter::VisitCXXExprWithTemporaries(CXXExprWithTemporaries *E) {
   VisitExpr(E);
   Record.push_back(E->getNumTemporaries());
   for (unsigned i = 0, e = E->getNumTemporaries(); i != e; ++i)
     Writer.AddCXXTemporary(E->getTemporary(i), Record);
   
   Writer.AddStmt(E->getSubExpr());
-  Code = pch::EXPR_CXX_EXPR_WITH_TEMPORARIES;
+  Code = serialization::EXPR_CXX_EXPR_WITH_TEMPORARIES;
 }
 
 void
-PCHStmtWriter::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
+ASTStmtWriter::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   VisitExpr(E);
   
   // Don't emit anything here, NumTemplateArgs must be emitted first.
 
   if (E->hasExplicitTemplateArgs()) {
-    const ExplicitTemplateArgumentList &Args
-      = *E->getExplicitTemplateArgumentList();
+    const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
     assert(Args.NumTemplateArgs &&
-           "Num of template args was zero! PCH reading will mess up!");
+           "Num of template args was zero! AST reading will mess up!");
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
   } else {
@@ -1157,13 +1170,14 @@ PCHStmtWriter::VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E){
   Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
   Writer.AddSourceRange(E->getQualifierRange(), Record);
   Writer.AddDeclRef(E->getFirstQualifierFoundInScope(), Record);
+  // FIXME: write whole DeclarationNameInfo.
   Writer.AddDeclarationName(E->getMember(), Record);
   Writer.AddSourceLocation(E->getMemberLoc(), Record);
-  Code = pch::EXPR_CXX_DEPENDENT_SCOPE_MEMBER;
+  Code = serialization::EXPR_CXX_DEPENDENT_SCOPE_MEMBER;
 }
 
 void
-PCHStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
+ASTStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
   VisitExpr(E);
   
   // Don't emit anything here, NumTemplateArgs must be emitted first.
@@ -1171,22 +1185,23 @@ PCHStmtWriter::VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
   if (E->hasExplicitTemplateArgs()) {
     const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
     assert(Args.NumTemplateArgs &&
-           "Num of template args was zero! PCH reading will mess up!");
+           "Num of template args was zero! AST reading will mess up!");
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
   } else {
     Record.push_back(0);
   }
 
+  // FIXME: write whole DeclarationNameInfo.
   Writer.AddDeclarationName(E->getDeclName(), Record);
   Writer.AddSourceLocation(E->getLocation(), Record);
   Writer.AddSourceRange(E->getQualifierRange(), Record);
   Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
-  Code = pch::EXPR_CXX_DEPENDENT_SCOPE_DECL_REF;
+  Code = serialization::EXPR_CXX_DEPENDENT_SCOPE_DECL_REF;
 }
 
 void
-PCHStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
+ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
   VisitExpr(E);
   Record.push_back(E->arg_size());
   for (CXXUnresolvedConstructExpr::arg_iterator
@@ -1196,10 +1211,10 @@ PCHStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) {
   Writer.AddTypeRef(E->getTypeAsWritten(), Record);
   Writer.AddSourceLocation(E->getLParenLoc(), Record);
   Writer.AddSourceLocation(E->getRParenLoc(), Record);
-  Code = pch::EXPR_CXX_UNRESOLVED_CONSTRUCT;
+  Code = serialization::EXPR_CXX_UNRESOLVED_CONSTRUCT;
 }
 
-void PCHStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
+void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
   VisitExpr(E);
   
   // Don't emit anything here, NumTemplateArgs must be emitted first.
@@ -1207,7 +1222,7 @@ void PCHStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
   if (E->hasExplicitTemplateArgs()) {
     const ExplicitTemplateArgumentList &Args = E->getExplicitTemplateArgs();
     assert(Args.NumTemplateArgs &&
-           "Num of template args was zero! PCH reading will mess up!");
+           "Num of template args was zero! AST reading will mess up!");
     Record.push_back(Args.NumTemplateArgs);
     AddExplicitTemplateArgumentList(Args);
   } else {
@@ -1221,43 +1236,44 @@ void PCHStmtWriter::VisitOverloadExpr(OverloadExpr *E) {
     Record.push_back(OvI.getAccess());
   }
 
+  // FIXME: write whole DeclarationNameInfo.
   Writer.AddDeclarationName(E->getName(), Record);
   Writer.AddNestedNameSpecifier(E->getQualifier(), Record);
   Writer.AddSourceRange(E->getQualifierRange(), Record);
   Writer.AddSourceLocation(E->getNameLoc(), Record);
 }
 
-void PCHStmtWriter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
+void ASTStmtWriter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
   VisitOverloadExpr(E);
   Record.push_back(E->isArrow());
   Record.push_back(E->hasUnresolvedUsing());
   Writer.AddStmt(!E->isImplicitAccess() ? E->getBase() : 0);
   Writer.AddTypeRef(E->getBaseType(), Record);
   Writer.AddSourceLocation(E->getOperatorLoc(), Record);
-  Code = pch::EXPR_CXX_UNRESOLVED_MEMBER;
+  Code = serialization::EXPR_CXX_UNRESOLVED_MEMBER;
 }
 
-void PCHStmtWriter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
+void ASTStmtWriter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
   VisitOverloadExpr(E);
   Record.push_back(E->requiresADL());
   Record.push_back(E->isOverloaded());
   Writer.AddDeclRef(E->getNamingClass(), Record);
-  Code = pch::EXPR_CXX_UNRESOLVED_LOOKUP;
+  Code = serialization::EXPR_CXX_UNRESOLVED_LOOKUP;
 }
 
-void PCHStmtWriter::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
+void ASTStmtWriter::VisitUnaryTypeTraitExpr(UnaryTypeTraitExpr *E) {
   VisitExpr(E);
   Record.push_back(E->getTrait());
   Writer.AddSourceRange(E->getSourceRange(), Record);
   Writer.AddTypeRef(E->getQueriedType(), Record);
-  Code = pch::EXPR_CXX_UNARY_TYPE_TRAIT;
+  Code = serialization::EXPR_CXX_UNARY_TYPE_TRAIT;
 }
 
 //===----------------------------------------------------------------------===//
-// PCHWriter Implementation
+// ASTWriter Implementation
 //===----------------------------------------------------------------------===//
 
-unsigned PCHWriter::RecordSwitchCaseID(SwitchCase *S) {
+unsigned ASTWriter::RecordSwitchCaseID(SwitchCase *S) {
   assert(SwitchCaseIDs.find(S) == SwitchCaseIDs.end() &&
          "SwitchCase recorded twice");
   unsigned NextID = SwitchCaseIDs.size();
@@ -1265,7 +1281,7 @@ unsigned PCHWriter::RecordSwitchCaseID(SwitchCase *S) {
   return NextID;
 }
 
-unsigned PCHWriter::getSwitchCaseID(SwitchCase *S) {
+unsigned ASTWriter::getSwitchCaseID(SwitchCase *S) {
   assert(SwitchCaseIDs.find(S) != SwitchCaseIDs.end() &&
          "SwitchCase hasn't been seen yet");
   return SwitchCaseIDs[S];
@@ -1273,7 +1289,7 @@ unsigned PCHWriter::getSwitchCaseID(SwitchCase *S) {
 
 /// \brief Retrieve the ID for the given label statement, which may
 /// or may not have been emitted yet.
-unsigned PCHWriter::GetLabelID(LabelStmt *S) {
+unsigned ASTWriter::GetLabelID(LabelStmt *S) {
   std::map<LabelStmt *, unsigned>::iterator Pos = LabelIDs.find(S);
   if (Pos != LabelIDs.end())
     return Pos->second;
@@ -1285,33 +1301,33 @@ unsigned PCHWriter::GetLabelID(LabelStmt *S) {
 
 /// \brief Write the given substatement or subexpression to the
 /// bitstream.
-void PCHWriter::WriteSubStmt(Stmt *S) {
+void ASTWriter::WriteSubStmt(Stmt *S) {
   RecordData Record;
-  PCHStmtWriter Writer(*this, Record);
+  ASTStmtWriter Writer(*this, Record);
   ++NumStatements;
   
   if (!S) {
-    Stream.EmitRecord(pch::STMT_NULL_PTR, Record);
+    Stream.EmitRecord(serialization::STMT_NULL_PTR, Record);
     return;
   }
 
-  // Redirect PCHWriter::AddStmt to collect sub stmts.
+  // Redirect ASTWriter::AddStmt to collect sub stmts.
   llvm::SmallVector<Stmt *, 16> SubStmts;
   CollectedStmts = &SubStmts;
 
-  Writer.Code = pch::STMT_NULL_PTR;
+  Writer.Code = serialization::STMT_NULL_PTR;
   Writer.Visit(S);
   
 #ifndef NDEBUG
-  if (Writer.Code == pch::STMT_NULL_PTR) {
+  if (Writer.Code == serialization::STMT_NULL_PTR) {
     SourceManager &SrcMgr
       = DeclIDs.begin()->first->getASTContext().getSourceManager();
     S->dump(SrcMgr);
-    assert(0 && "Unhandled sub statement writing PCH file");
+    assert(0 && "Unhandled sub statement writing AST file");
   }
 #endif
 
-  // Revert PCHWriter::AddStmt.
+  // Revert ASTWriter::AddStmt.
   CollectedStmts = &StmtsToEmit;
 
   // Write the sub stmts in reverse order, last to first. When reading them back
@@ -1326,7 +1342,7 @@ void PCHWriter::WriteSubStmt(Stmt *S) {
 
 /// \brief Flush all of the statements that have been added to the
 /// queue via AddStmt().
-void PCHWriter::FlushStmts() {
+void ASTWriter::FlushStmts() {
   RecordData Record;
 
   for (unsigned I = 0, N = StmtsToEmit.size(); I != N; ++I) {
@@ -1338,7 +1354,7 @@ void PCHWriter::FlushStmts() {
     // Note that we are at the end of a full expression. Any
     // expression records that follow this one are part of a different
     // expression.
-    Stream.EmitRecord(pch::STMT_STOP, Record);
+    Stream.EmitRecord(serialization::STMT_STOP, Record);
   }
 
   StmtsToEmit.clear();
diff --git a/lib/Serialization/CMakeLists.txt b/lib/Serialization/CMakeLists.txt
new file mode 100644
index 000000000000..d863c179bed2
--- /dev/null
+++ b/lib/Serialization/CMakeLists.txt
@@ -0,0 +1,23 @@
+set(LLVM_NO_RTTI 1)
+
+add_clang_library(clangSerialization
+  GeneratePCH.cpp
+  ASTCommon.cpp
+  ASTReader.cpp
+  ASTReaderDecl.cpp
+  ASTReaderStmt.cpp
+  ASTWriter.cpp
+  ASTWriterDecl.cpp
+  ASTWriterStmt.cpp
+  )
+
+add_dependencies(clangSerialization
+  ClangAttrClasses
+  ClangAttrList
+  ClangAttrPCHRead
+  ClangAttrPCHWrite
+  ClangDiagnosticFrontend
+  ClangDiagnosticLex
+  ClangDiagnosticSema
+  ClangDeclNodes
+  ClangStmtNodes)
diff --git a/lib/Frontend/GeneratePCH.cpp b/lib/Serialization/GeneratePCH.cpp
index 2f3df9479d93..5329b6cbd4d0 100644
--- a/lib/Frontend/GeneratePCH.cpp
+++ b/lib/Serialization/GeneratePCH.cpp
@@ -8,12 +8,12 @@
 //===----------------------------------------------------------------------===//
 //
 //  This file defines the CreatePCHGenerate function, which creates an
-//  ASTConsume that generates a PCH file.
+//  ASTConsumer that generates a PCH file.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/ASTConsumers.h"
-#include "clang/Frontend/PCHWriter.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTConsumer.h"
@@ -25,36 +25,20 @@
 
 using namespace clang;
 
-namespace {
-  class PCHGenerator : public SemaConsumer {
-    const Preprocessor &PP;
-    const char *isysroot;
-    llvm::raw_ostream *Out;
-    Sema *SemaPtr;
-    MemorizeStatCalls *StatCalls; // owned by the FileManager
-    std::vector<unsigned char> Buffer;
-    llvm::BitstreamWriter Stream;
-    PCHWriter Writer;
-
-  public:
-    PCHGenerator(const Preprocessor &PP, PCHReader *Chain,
-                 const char *isysroot, llvm::raw_ostream *Out);
-    virtual void InitializeSema(Sema &S) { SemaPtr = &S; }
-    virtual void HandleTranslationUnit(ASTContext &Ctx);
-  };
-}
-
 PCHGenerator::PCHGenerator(const Preprocessor &PP,
-                           PCHReader *Chain,
+                           bool Chaining,
                            const char *isysroot,
                            llvm::raw_ostream *OS)
-  : PP(PP), isysroot(isysroot), Out(OS), SemaPtr(0), StatCalls(0),
-    Stream(Buffer), Writer(Stream, Chain) {
+  : PP(PP), isysroot(isysroot), Out(OS), SemaPtr(0),
+    StatCalls(0), Stream(Buffer), Writer(Stream) {
 
   // Install a stat() listener to keep track of all of the stat()
   // calls.
   StatCalls = new MemorizeStatCalls;
-  PP.getFileManager().addStatCache(StatCalls, /*AtBeginning=*/true);
+  // If we have a chain, we want new stat calls only, so install the memorizer
+  // *after* the already installed ASTReader's stat cache.
+  PP.getFileManager().addStatCache(StatCalls,
+    /*AtBeginning=*/!Chaining);
 }
 
 void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
@@ -63,7 +47,7 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
 
   // Emit the PCH file
   assert(SemaPtr && "No Sema?");
-  Writer.WritePCH(*SemaPtr, StatCalls, isysroot);
+  Writer.WriteAST(*SemaPtr, StatCalls, isysroot);
 
   // Write the generated bitstream to "Out".
   Out->write((char *)&Buffer.front(), Buffer.size());
@@ -75,9 +59,6 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   Buffer.clear();
 }
 
-ASTConsumer *clang::CreatePCHGenerator(const Preprocessor &PP,
-                                       llvm::raw_ostream *OS,
-                                       PCHReader *Chain,
-                                       const char *isysroot) {
-  return new PCHGenerator(PP, Chain, isysroot, OS);
+ASTDeserializationListener *PCHGenerator::GetASTDeserializationListener() {
+  return &Writer;
 }
diff --git a/lib/Serialization/Makefile b/lib/Serialization/Makefile
new file mode 100644
index 000000000000..e89ddc38ec94
--- /dev/null
+++ b/lib/Serialization/Makefile
@@ -0,0 +1,19 @@
+##===- clang/lib/Serialization/Makefile --------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+#  This implements the semantic analyzer and AST builder library for the 
+#  C-Language front-end.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../..
+LIBRARYNAME := clangSerialization
+
+include $(CLANG_LEVEL)/Makefile
+