Update LLVM to 92395.

46 files changed, 2172 insertions, 936 deletions

diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index bb61682c2328..761fbc6870f0 100644
--- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -127,11 +127,11 @@ AggressiveAntiDepBreaker(MachineFunction& MFi,
       CriticalPathSet |= CPSet;
    }
  
-  DEBUG(errs() << "AntiDep Critical-Path Registers:");
+  DEBUG(dbgs() << "AntiDep Critical-Path Registers:");
   DEBUG(for (int r = CriticalPathSet.find_first(); r != -1; 
              r = CriticalPathSet.find_next(r))
-          errs() << " " << TRI->getName(r));
-  DEBUG(errs() << '\n');
+          dbgs() << " " << TRI->getName(r));
+  DEBUG(dbgs() << '\n');
 }
 
 AggressiveAntiDepBreaker::~AggressiveAntiDepBreaker() {
@@ -218,9 +218,9 @@ void AggressiveAntiDepBreaker::Observe(MachineInstr *MI, unsigned Count,
   PrescanInstruction(MI, Count, PassthruRegs);
   ScanInstruction(MI, Count);
 
-  DEBUG(errs() << "Observe: ");
+  DEBUG(dbgs() << "Observe: ");
   DEBUG(MI->dump());
-  DEBUG(errs() << "\tRegs:");
+  DEBUG(dbgs() << "\tRegs:");
 
   unsigned *DefIndices = State->GetDefIndices();
   for (unsigned Reg = 0; Reg != TRI->getNumRegs(); ++Reg) {
@@ -232,14 +232,14 @@ void AggressiveAntiDepBreaker::Observe(MachineInstr *MI, unsigned Count,
     // schedule region).
     if (State->IsLive(Reg)) {
       DEBUG(if (State->GetGroup(Reg) != 0)
-              errs() << " " << TRI->getName(Reg) << "=g" << 
+              dbgs() << " " << TRI->getName(Reg) << "=g" << 
                 State->GetGroup(Reg) << "->g0(region live-out)");
       State->UnionGroups(Reg, 0);
     } else if ((DefIndices[Reg] < InsertPosIndex) && (DefIndices[Reg] >= Count)) {
       DefIndices[Reg] = Count;
     }
   }
-  DEBUG(errs() << '\n');
+  DEBUG(dbgs() << '\n');
 }
 
 bool AggressiveAntiDepBreaker::IsImplicitDefUse(MachineInstr *MI,
@@ -333,8 +333,8 @@ void AggressiveAntiDepBreaker::HandleLastUse(unsigned Reg, unsigned KillIdx,
     RegRefs.erase(Reg);
     State->LeaveGroup(Reg);
     DEBUG(if (header != NULL) {
-        errs() << header << TRI->getName(Reg); header = NULL; });
-    DEBUG(errs() << "->g" << State->GetGroup(Reg) << tag);
+        dbgs() << header << TRI->getName(Reg); header = NULL; });
+    DEBUG(dbgs() << "->g" << State->GetGroup(Reg) << tag);
   }
   // Repeat for subregisters.
   for (const unsigned *Subreg = TRI->getSubRegisters(Reg);
@@ -346,13 +346,13 @@ void AggressiveAntiDepBreaker::HandleLastUse(unsigned Reg, unsigned KillIdx,
       RegRefs.erase(SubregReg);
       State->LeaveGroup(SubregReg);
       DEBUG(if (header != NULL) {
-          errs() << header << TRI->getName(Reg); header = NULL; });
-      DEBUG(errs() << " " << TRI->getName(SubregReg) << "->g" <<
+          dbgs() << header << TRI->getName(Reg); header = NULL; });
+      DEBUG(dbgs() << " " << TRI->getName(SubregReg) << "->g" <<
             State->GetGroup(SubregReg) << tag);
     }
   }
 
-  DEBUG(if ((header == NULL) && (footer != NULL)) errs() << footer);
+  DEBUG(if ((header == NULL) && (footer != NULL)) dbgs() << footer);
 }
 
 void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Count,
@@ -375,20 +375,20 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou
     HandleLastUse(Reg, Count + 1, "", "\tDead Def: ", "\n");
   }
 
-  DEBUG(errs() << "\tDef Groups:");
+  DEBUG(dbgs() << "\tDef Groups:");
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     MachineOperand &MO = MI->getOperand(i);
     if (!MO.isReg() || !MO.isDef()) continue;
     unsigned Reg = MO.getReg();
     if (Reg == 0) continue;
 
-    DEBUG(errs() << " " << TRI->getName(Reg) << "=g" << State->GetGroup(Reg)); 
+    DEBUG(dbgs() << " " << TRI->getName(Reg) << "=g" << State->GetGroup(Reg)); 
 
     // If MI's defs have a special allocation requirement, don't allow
     // any def registers to be changed. Also assume all registers
     // defined in a call must not be changed (ABI).
     if (MI->getDesc().isCall() || MI->getDesc().hasExtraDefRegAllocReq()) {
-      DEBUG(if (State->GetGroup(Reg) != 0) errs() << "->g0(alloc-req)");
+      DEBUG(if (State->GetGroup(Reg) != 0) dbgs() << "->g0(alloc-req)");
       State->UnionGroups(Reg, 0);
     }
 
@@ -398,7 +398,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou
       unsigned AliasReg = *Alias;
       if (State->IsLive(AliasReg)) {
         State->UnionGroups(Reg, AliasReg);
-        DEBUG(errs() << "->g" << State->GetGroup(Reg) << "(via " << 
+        DEBUG(dbgs() << "->g" << State->GetGroup(Reg) << "(via " << 
               TRI->getName(AliasReg) << ")");
       }
     }
@@ -411,7 +411,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou
     RegRefs.insert(std::make_pair(Reg, RR));
   }
 
-  DEBUG(errs() << '\n');
+  DEBUG(dbgs() << '\n');
 
   // Scan the register defs for this instruction and update
   // live-ranges.
@@ -437,7 +437,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, unsigned Cou
 
 void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI,
                                            unsigned Count) {
-  DEBUG(errs() << "\tUse Groups:");
+  DEBUG(dbgs() << "\tUse Groups:");
   std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>& 
     RegRefs = State->GetRegRefs();
 
@@ -449,7 +449,7 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI,
     unsigned Reg = MO.getReg();
     if (Reg == 0) continue;
     
-    DEBUG(errs() << " " << TRI->getName(Reg) << "=g" << 
+    DEBUG(dbgs() << " " << TRI->getName(Reg) << "=g" << 
           State->GetGroup(Reg)); 
 
     // It wasn't previously live but now it is, this is a kill. Forget
@@ -461,7 +461,7 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI,
     // any use registers to be changed. Also assume all registers
     // used in a call must not be changed (ABI).
     if (MI->getDesc().isCall() || MI->getDesc().hasExtraSrcRegAllocReq()) {
-      DEBUG(if (State->GetGroup(Reg) != 0) errs() << "->g0(alloc-req)");
+      DEBUG(if (State->GetGroup(Reg) != 0) dbgs() << "->g0(alloc-req)");
       State->UnionGroups(Reg, 0);
     }
 
@@ -473,12 +473,12 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI,
     RegRefs.insert(std::make_pair(Reg, RR));
   }
   
-  DEBUG(errs() << '\n');
+  DEBUG(dbgs() << '\n');
 
   // Form a group of all defs and uses of a KILL instruction to ensure
   // that all registers are renamed as a group.
   if (MI->getOpcode() == TargetInstrInfo::KILL) {
-    DEBUG(errs() << "\tKill Group:");
+    DEBUG(dbgs() << "\tKill Group:");
 
     unsigned FirstReg = 0;
     for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
@@ -488,15 +488,15 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI,
       if (Reg == 0) continue;
       
       if (FirstReg != 0) {
-        DEBUG(errs() << "=" << TRI->getName(Reg));
+        DEBUG(dbgs() << "=" << TRI->getName(Reg));
         State->UnionGroups(FirstReg, Reg);
       } else {
-        DEBUG(errs() << " " << TRI->getName(Reg));
+        DEBUG(dbgs() << " " << TRI->getName(Reg));
         FirstReg = Reg;
       }
     }
   
-    DEBUG(errs() << "->g" << State->GetGroup(FirstReg) << '\n');
+    DEBUG(dbgs() << "->g" << State->GetGroup(FirstReg) << '\n');
   }
 }
 
@@ -525,7 +525,7 @@ BitVector AggressiveAntiDepBreaker::GetRenameRegisters(unsigned Reg) {
       BV &= RCBV;
     }
 
-    DEBUG(errs() << " " << RC->getName());
+    DEBUG(dbgs() << " " << RC->getName());
   }
   
   return BV;
@@ -552,7 +552,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
   // Find the "superest" register in the group. At the same time,
   // collect the BitVector of registers that can be used to rename
   // each register.
-  DEBUG(errs() << "\tRename Candidates for Group g" << AntiDepGroupIndex << ":\n");
+  DEBUG(dbgs() << "\tRename Candidates for Group g" << AntiDepGroupIndex << ":\n");
   std::map<unsigned, BitVector> RenameRegisterMap;
   unsigned SuperReg = 0;
   for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
@@ -562,15 +562,15 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
 
     // If Reg has any references, then collect possible rename regs
     if (RegRefs.count(Reg) > 0) {
-      DEBUG(errs() << "\t\t" << TRI->getName(Reg) << ":");
+      DEBUG(dbgs() << "\t\t" << TRI->getName(Reg) << ":");
     
       BitVector BV = GetRenameRegisters(Reg);
       RenameRegisterMap.insert(std::pair<unsigned, BitVector>(Reg, BV));
 
-      DEBUG(errs() << " ::");
+      DEBUG(dbgs() << " ::");
       DEBUG(for (int r = BV.find_first(); r != -1; r = BV.find_next(r))
-              errs() << " " << TRI->getName(r));
-      DEBUG(errs() << "\n");
+              dbgs() << " " << TRI->getName(r));
+      DEBUG(dbgs() << "\n");
     }
   }
 
@@ -591,7 +591,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
     if (renamecnt++ % DebugDiv != DebugMod)
       return false;
     
-    errs() << "*** Performing rename " << TRI->getName(SuperReg) <<
+    dbgs() << "*** Performing rename " << TRI->getName(SuperReg) <<
       " for debug ***\n";
   }
 #endif
@@ -606,11 +606,11 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
   const TargetRegisterClass::iterator RB = SuperRC->allocation_order_begin(MF);
   const TargetRegisterClass::iterator RE = SuperRC->allocation_order_end(MF);
   if (RB == RE) {
-    DEBUG(errs() << "\tEmpty Super Regclass!!\n");
+    DEBUG(dbgs() << "\tEmpty Super Regclass!!\n");
     return false;
   }
 
-  DEBUG(errs() << "\tFind Registers:");
+  DEBUG(dbgs() << "\tFind Registers:");
 
   if (RenameOrder.count(SuperRC) == 0)
     RenameOrder.insert(RenameOrderType::value_type(SuperRC, RE));
@@ -625,7 +625,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
     // Don't replace a register with itself.
     if (NewSuperReg == SuperReg) continue;
     
-    DEBUG(errs() << " [" << TRI->getName(NewSuperReg) << ':');
+    DEBUG(dbgs() << " [" << TRI->getName(NewSuperReg) << ':');
     RenameMap.clear();
 
     // For each referenced group register (which must be a SuperReg or
@@ -642,12 +642,12 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
           NewReg = TRI->getSubReg(NewSuperReg, NewSubRegIdx);
       }
 
-      DEBUG(errs() << " " << TRI->getName(NewReg));
+      DEBUG(dbgs() << " " << TRI->getName(NewReg));
       
       // Check if Reg can be renamed to NewReg.
       BitVector BV = RenameRegisterMap[Reg];
       if (!BV.test(NewReg)) {
-        DEBUG(errs() << "(no rename)");
+        DEBUG(dbgs() << "(no rename)");
         goto next_super_reg;
       }
 
@@ -656,7 +656,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
       // must also check all aliases of NewReg, because we can't define a
       // register when any sub or super is already live.
       if (State->IsLive(NewReg) || (KillIndices[Reg] > DefIndices[NewReg])) {
-        DEBUG(errs() << "(live)");
+        DEBUG(dbgs() << "(live)");
         goto next_super_reg;
       } else {
         bool found = false;
@@ -664,7 +664,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
              *Alias; ++Alias) {
           unsigned AliasReg = *Alias;
           if (State->IsLive(AliasReg) || (KillIndices[Reg] > DefIndices[AliasReg])) {
-            DEBUG(errs() << "(alias " << TRI->getName(AliasReg) << " live)");
+            DEBUG(dbgs() << "(alias " << TRI->getName(AliasReg) << " live)");
             found = true;
             break;
           }
@@ -681,14 +681,14 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
     // renamed, as recorded in RenameMap.
     RenameOrder.erase(SuperRC);
     RenameOrder.insert(RenameOrderType::value_type(SuperRC, R));
-    DEBUG(errs() << "]\n");
+    DEBUG(dbgs() << "]\n");
     return true;
 
   next_super_reg:
-    DEBUG(errs() << ']');
+    DEBUG(dbgs() << ']');
   } while (R != EndR);
 
-  DEBUG(errs() << '\n');
+  DEBUG(dbgs() << '\n');
 
   // No registers are free and available!
   return false;
@@ -740,13 +740,13 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
   }
 
 #ifndef NDEBUG 
-  DEBUG(errs() << "\n===== Aggressive anti-dependency breaking\n");
-  DEBUG(errs() << "Available regs:");
+  DEBUG(dbgs() << "\n===== Aggressive anti-dependency breaking\n");
+  DEBUG(dbgs() << "Available regs:");
   for (unsigned Reg = 0; Reg < TRI->getNumRegs(); ++Reg) {
     if (!State->IsLive(Reg))
-      DEBUG(errs() << " " << TRI->getName(Reg));
+      DEBUG(dbgs() << " " << TRI->getName(Reg));
   }
-  DEBUG(errs() << '\n');
+  DEBUG(dbgs() << '\n');
 #endif
 
   // Attempt to break anti-dependence edges. Walk the instructions
@@ -758,7 +758,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
        I != E; --Count) {
     MachineInstr *MI = --I;
 
-    DEBUG(errs() << "Anti: ");
+    DEBUG(dbgs() << "Anti: ");
     DEBUG(MI->dump());
 
     std::set<unsigned> PassthruRegs;
@@ -795,30 +795,30 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
             (Edge->getKind() != SDep::Output)) continue;
         
         unsigned AntiDepReg = Edge->getReg();
-        DEBUG(errs() << "\tAntidep reg: " << TRI->getName(AntiDepReg));
+        DEBUG(dbgs() << "\tAntidep reg: " << TRI->getName(AntiDepReg));
         assert(AntiDepReg != 0 && "Anti-dependence on reg0?");
         
         if (!AllocatableSet.test(AntiDepReg)) {
           // Don't break anti-dependencies on non-allocatable registers.
-          DEBUG(errs() << " (non-allocatable)\n");
+          DEBUG(dbgs() << " (non-allocatable)\n");
           continue;
         } else if ((ExcludeRegs != NULL) && ExcludeRegs->test(AntiDepReg)) {
           // Don't break anti-dependencies for critical path registers
           // if not on the critical path
-          DEBUG(errs() << " (not critical-path)\n");
+          DEBUG(dbgs() << " (not critical-path)\n");
           continue;
         } else if (PassthruRegs.count(AntiDepReg) != 0) {
           // If the anti-dep register liveness "passes-thru", then
           // don't try to change it. It will be changed along with
           // the use if required to break an earlier antidep.
-          DEBUG(errs() << " (passthru)\n");
+          DEBUG(dbgs() << " (passthru)\n");
           continue;
         } else {
           // No anti-dep breaking for implicit deps
           MachineOperand *AntiDepOp = MI->findRegisterDefOperand(AntiDepReg);
           assert(AntiDepOp != NULL && "Can't find index for defined register operand");
           if ((AntiDepOp == NULL) || AntiDepOp->isImplicit()) {
-            DEBUG(errs() << " (implicit)\n");
+            DEBUG(dbgs() << " (implicit)\n");
             continue;
           }
           
@@ -844,13 +844,13 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
                  PE = PathSU->Preds.end(); P != PE; ++P) {
             if ((P->getSUnit() == NextSU) && (P->getKind() != SDep::Anti) &&
                 (P->getKind() != SDep::Output)) {
-              DEBUG(errs() << " (real dependency)\n");
+              DEBUG(dbgs() << " (real dependency)\n");
               AntiDepReg = 0;
               break;
             } else if ((P->getSUnit() != NextSU) && 
                        (P->getKind() == SDep::Data) && 
                        (P->getReg() == AntiDepReg)) {
-              DEBUG(errs() << " (other dependency)\n");
+              DEBUG(dbgs() << " (other dependency)\n");
               AntiDepReg = 0;
               break;
             }
@@ -865,16 +865,16 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
         // Determine AntiDepReg's register group.
         const unsigned GroupIndex = State->GetGroup(AntiDepReg);
         if (GroupIndex == 0) {
-          DEBUG(errs() << " (zero group)\n");
+          DEBUG(dbgs() << " (zero group)\n");
           continue;
         }
         
-        DEBUG(errs() << '\n');
+        DEBUG(dbgs() << '\n');
         
         // Look for a suitable register to use to break the anti-dependence.
         std::map<unsigned, unsigned> RenameMap;
         if (FindSuitableFreeRegisters(GroupIndex, RenameOrder, RenameMap)) {
-          DEBUG(errs() << "\tBreaking anti-dependence edge on "
+          DEBUG(dbgs() << "\tBreaking anti-dependence edge on "
                 << TRI->getName(AntiDepReg) << ":");
           
           // Handle each group register...
@@ -883,7 +883,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
             unsigned CurrReg = S->first;
             unsigned NewReg = S->second;
             
-            DEBUG(errs() << " " << TRI->getName(CurrReg) << "->" << 
+            DEBUG(dbgs() << " " << TRI->getName(CurrReg) << "->" << 
                   TRI->getName(NewReg) << "(" <<  
                   RegRefs.count(CurrReg) << " refs)");
             
@@ -917,7 +917,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
           }
           
           ++Broken;
-          DEBUG(errs() << '\n');
+          DEBUG(dbgs() << '\n');
         }
       }
     }
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 44fd176a79e5..6b24e24d3716 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -236,7 +236,7 @@ namespace {
     const MCSection *S;
     unsigned Alignment;
     SmallVector<unsigned, 4> CPEs;
-    SectionCPs(const MCSection *s, unsigned a) : S(s), Alignment(a) {};
+    SectionCPs(const MCSection *s, unsigned a) : S(s), Alignment(a) {}
   };
 }
 
@@ -1905,7 +1905,6 @@ void AsmPrinter::EmitComments(const MachineInstr &MI) const {
       if (Newline) O << '\n';
       O.PadToColumn(MAI->getCommentColumn());
       O << MAI->getCommentString() << " Reload Reuse";
-      Newline = true;
     }
   }
 }
diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp
index 0e93b9849ce5..b85e11acc0f1 100644
--- a/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -16,6 +16,7 @@
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Target/TargetData.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
 using namespace llvm;
@@ -93,7 +94,7 @@ void DIEAbbrev::print(raw_ostream &O) {
       << '\n';
   }
 }
-void DIEAbbrev::dump() { print(errs()); }
+void DIEAbbrev::dump() { print(dbgs()); }
 #endif
 
 //===----------------------------------------------------------------------===//
@@ -164,14 +165,14 @@ void DIE::print(raw_ostream &O, unsigned IncIndent) {
 }
 
 void DIE::dump() {
-  print(errs());
+  print(dbgs());
 }
 #endif
 
 
 #ifndef NDEBUG
 void DIEValue::dump() {
-  print(errs());
+  print(dbgs());
 }
 #endif
 
diff --git a/lib/CodeGen/AsmPrinter/DIE.h b/lib/CodeGen/AsmPrinter/DIE.h
index cad8b8955024..a6dc9b690c23 100644
--- a/lib/CodeGen/AsmPrinter/DIE.h
+++ b/lib/CodeGen/AsmPrinter/DIE.h
@@ -68,6 +68,7 @@ namespace llvm {
     /// Data - Raw data bytes for abbreviation.
     ///
     SmallVector<DIEAbbrevData, 8> Data;
+
   public:
     DIEAbbrev(unsigned T, unsigned C) : Tag(T), ChildrenFlag(C), Data() {}
     virtual ~DIEAbbrev() {}
@@ -131,19 +132,18 @@ namespace llvm {
     ///
     std::vector<DIE *> Children;
 
+    DIE *Parent;
+
     /// Attributes values.
     ///
     SmallVector<DIEValue*, 32> Values;
 
-    /// Abstract compile unit.
-    CompileUnit *AbstractCU;
-    
     // Private data for print()
     mutable unsigned IndentCount;
   public:
     explicit DIE(unsigned Tag)
       : Abbrev(Tag, dwarf::DW_CHILDREN_no), Offset(0),
-        Size(0), IndentCount(0) {}
+        Size(0), Parent (0), IndentCount(0) {}
     virtual ~DIE();
 
     // Accessors.
@@ -154,13 +154,12 @@ namespace llvm {
     unsigned getSize() const { return Size; }
     const std::vector<DIE *> &getChildren() const { return Children; }
     SmallVector<DIEValue*, 32> &getValues() { return Values; }
-    CompileUnit *getAbstractCompileUnit() const { return AbstractCU; }
-
+    DIE *getParent() const { return Parent; }
     void setTag(unsigned Tag) { Abbrev.setTag(Tag); }
     void setOffset(unsigned O) { Offset = O; }
     void setSize(unsigned S) { Size = S; }
-    void setAbstractCompileUnit(CompileUnit *CU) { AbstractCU = CU; }
-
+    void setParent(DIE *P) { Parent = P; }
+    
     /// addValue - Add a value and attributes to a DIE.
     ///
     void addValue(unsigned Attribute, unsigned Form, DIEValue *Value) {
@@ -179,8 +178,13 @@ namespace llvm {
     /// addChild - Add a child to the DIE.
     ///
     void addChild(DIE *Child) {
+      if (Child->getParent()) {
+        assert (Child->getParent() == this && "Unexpected DIE Parent!");
+        return;
+      }
       Abbrev.setChildrenFlag(dwarf::DW_CHILDREN_yes);
       Children.push_back(Child);
+      Child->setParent(this);
     }
 
 #ifndef NDEBUG
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index c200a46100c0..8a3ceb631d03 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -30,11 +30,6 @@
 #include "llvm/System/Path.h"
 using namespace llvm;
 
-static TimerGroup &getDwarfTimerGroup() {
-  static TimerGroup DwarfTimerGroup("Dwarf Debugging");
-  return DwarfTimerGroup;
-}
-
 //===----------------------------------------------------------------------===//
 
 /// Configuration values for initial hash set sizes (log2).
@@ -112,7 +107,12 @@ public:
 
   /// getDIEEntry - Returns the debug information entry for the speciefied
   /// debug variable.
-  DIEEntry *getDIEEntry(MDNode *N) { return GVToDIEEntryMap.lookup(N); }
+  DIEEntry *getDIEEntry(MDNode *N) { 
+    ValueMap<MDNode *, DIEEntry *>::iterator I = GVToDIEEntryMap.find(N);
+    if (I == GVToDIEEntryMap.end())
+      return NULL;
+    return I->second;
+  }
 
   /// insertDIEEntry - Insert debug information entry into the map.
   void insertDIEEntry(MDNode *N, DIEEntry *E) {
@@ -234,7 +234,7 @@ public:
 
 #ifndef NDEBUG
 void DbgScope::dump() const {
-  raw_ostream &err = errs();
+  raw_ostream &err = dbgs();
   err.indent(IndentLevel);
   MDNode *N = Desc.getNode();
   N->dump();
@@ -269,8 +269,7 @@ DwarfDebug::DwarfDebug(raw_ostream &OS, AsmPrinter *A, const MCAsmInfo *T)
     SectionSourceLines(), didInitial(false), shouldEmit(false),
     CurrentFnDbgScope(0), DebugTimer(0) {
   if (TimePassesIsEnabled)
-    DebugTimer = new Timer("Dwarf Debug Writer",
-                           getDwarfTimerGroup());
+    DebugTimer = new Timer("Dwarf Debug Writer");
 }
 DwarfDebug::~DwarfDebug() {
   for (unsigned j = 0, M = DIEValues.size(); j < M; ++j)
@@ -446,6 +445,23 @@ void DwarfDebug::addSourceLine(DIE *Die, const DIType *Ty) {
   addUInt(Die, dwarf::DW_AT_decl_line, 0, Line);
 }
 
+/// addSourceLine - Add location information to specified debug information
+/// entry.
+void DwarfDebug::addSourceLine(DIE *Die, const DINameSpace *NS) {
+  // If there is no compile unit specified, don't add a line #.
+  if (NS->getCompileUnit().isNull())
+    return;
+
+  unsigned Line = NS->getLineNumber();
+  StringRef FN = NS->getFilename();
+  StringRef Dir = NS->getDirectory();
+
+  unsigned FileID = GetOrCreateSourceID(Dir, FN);
+  assert(FileID && "Invalid file id");
+  addUInt(Die, dwarf::DW_AT_decl_file, 0, FileID);
+  addUInt(Die, dwarf::DW_AT_decl_line, 0, Line);
+}
+
 /* Byref variables, in Blocks, are declared by the programmer as
    "SomeType VarName;", but the compiler creates a
    __Block_byref_x_VarName struct, and gives the variable VarName
@@ -745,6 +761,9 @@ void DwarfDebug::addToContextOwner(DIE *Die, DIDescriptor Context) {
   else if (Context.isType()) {
     DIE *ContextDIE = getOrCreateTypeDIE(DIType(Context.getNode()));
     ContextDIE->addChild(Die);
+  } else if (Context.isNameSpace()) {
+    DIE *ContextDIE = getOrCreateNameSpace(DINameSpace(Context.getNode()));
+    ContextDIE->addChild(Die);
   } else if (DIE *ContextDIE = ModuleCU->getDIE(Context.getNode()))
     ContextDIE->addChild(Die);
   else 
@@ -781,7 +800,6 @@ void DwarfDebug::addType(DIE *Entity, DIType Ty) {
 
   // Check for pre-existence.
   DIEEntry *Entry = ModuleCU->getDIEEntry(Ty.getNode());
-
   // If it exists then use the existing value.
   if (Entry) {
     Entity->addValue(dwarf::DW_AT_type, dwarf::DW_FORM_ref4, Entry);
@@ -1030,13 +1048,6 @@ DIE *DwarfDebug::createGlobalVariableDIE(const DIGlobalVariable &GV) {
     addUInt(GVDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
   addSourceLine(GVDie, &GV);
 
-  // Add address.
-  DIEBlock *Block = new DIEBlock();
-  addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
-  addObjectLabel(Block, 0, dwarf::DW_FORM_udata,
-                 Asm->Mang->getMangledName(GV.getGlobal()));
-  addBlock(GVDie, dwarf::DW_AT_location, 0, Block);
-
   return GVDie;
 }
 
@@ -1285,7 +1296,6 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(MDNode *SPNode) {
    SPDie = new DIE(dwarf::DW_TAG_subprogram);
    addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4, 
                SPDeclDie);
-   
    ModuleCU->addDie(SPDie);
  }
    
@@ -1559,6 +1569,20 @@ unsigned DwarfDebug::GetOrCreateSourceID(StringRef DirName, StringRef FileName)
   return SrcId;
 }
 
+/// getOrCreateNameSpace - Create a DIE for DINameSpace.
+DIE *DwarfDebug::getOrCreateNameSpace(DINameSpace NS) {
+  DIE *NDie = ModuleCU->getDIE(NS.getNode());
+  if (NDie)
+    return NDie;
+  NDie = new DIE(dwarf::DW_TAG_namespace);
+  ModuleCU->insertDIE(NS.getNode(), NDie);
+  if (!NS.getName().empty())
+    addString(NDie, dwarf::DW_AT_name, dwarf::DW_FORM_string, NS.getName());
+  addSourceLine(NDie, &NS);
+  addToContextOwner(NDie, NS.getContext());
+  return NDie;
+}
+
 CompileUnit *DwarfDebug::constructCompileUnit(MDNode *N) {
   DICompileUnit DIUnit(N);
   StringRef FN = DIUnit.getFilename();
@@ -1620,6 +1644,25 @@ void DwarfDebug::constructGlobalVariableDIE(MDNode *N) {
   ModuleCU->insertDIE(N, VariableDie);
 
   // Add to context owner.
+  if (DI_GV.isDefinition() 
+      && !DI_GV.getContext().isCompileUnit()) {
+    // Create specification DIE.
+    DIE *VariableSpecDIE = new DIE(dwarf::DW_TAG_variable);
+    addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification,
+                dwarf::DW_FORM_ref4, VariableDie);
+    DIEBlock *Block = new DIEBlock();
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
+    addObjectLabel(Block, 0, dwarf::DW_FORM_udata,
+                   Asm->Mang->getMangledName(DI_GV.getGlobal()));
+    addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block);
+    ModuleCU->addDie(VariableSpecDIE);
+  } else {
+    DIEBlock *Block = new DIEBlock();
+    addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_addr);
+    addObjectLabel(Block, 0, dwarf::DW_FORM_udata,
+                   Asm->Mang->getMangledName(DI_GV.getGlobal()));
+    addBlock(VariableDie, dwarf::DW_AT_location, 0, Block);
+  }
   addToContextOwner(VariableDie, DI_GV.getContext());
   
   // Expose as global. FIXME - need to check external flag.
@@ -1652,9 +1695,7 @@ void DwarfDebug::constructSubprogramDIE(MDNode *N) {
   ModuleCU->insertDIE(N, SubprogramDie);
 
   // Add to context owner.
-  if (SP.getContext().getNode() == SP.getCompileUnit().getNode())
-    if (TopLevelDIEs.insert(SubprogramDie))
-      TopLevelDIEsVector.push_back(SubprogramDie);
+  addToContextOwner(SubprogramDie, SP.getContext());
 
   // Expose as global.
   ModuleCU->addGlobal(SP.getName(), SubprogramDie);
@@ -2365,7 +2406,6 @@ void DwarfDebug::emitDebugInfo() {
   EmitLabel("info_end", ModuleCU->getID());
 
   Asm->EOL();
-
 }
 
 /// emitAbbreviations - Emit the abbreviation section.
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 12ad322b6853..2b8164e0b199 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -285,6 +285,7 @@ class DwarfDebug : public Dwarf {
   void addSourceLine(DIE *Die, const DIGlobal *G);
   void addSourceLine(DIE *Die, const DISubprogram *SP);
   void addSourceLine(DIE *Die, const DIType *Ty);
+  void addSourceLine(DIE *Die, const DINameSpace *NS);
 
   /// addAddress - Add an address attribute to a die based on the location
   /// provided.
@@ -315,6 +316,10 @@ class DwarfDebug : public Dwarf {
   /// addType - Add a new type attribute to the specified entity.
   void addType(DIE *Entity, DIType Ty);
 
+ 
+  /// getOrCreateNameSpace - Create a DIE for DINameSpace.
+  DIE *getOrCreateNameSpace(DINameSpace NS);
+
   /// getOrCreateTypeDIE - Find existing DIE or create new DIE for the
   /// given DIType.
   DIE *getOrCreateTypeDIE(DIType Ty);
diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp
index 3fd077f68731..d01f300990cb 100644
--- a/lib/CodeGen/AsmPrinter/DwarfException.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp
@@ -35,19 +35,13 @@
 #include "llvm/ADT/StringExtras.h"
 using namespace llvm;
 
-static TimerGroup &getDwarfTimerGroup() {
-  static TimerGroup DwarfTimerGroup("DWARF Exception");
-  return DwarfTimerGroup;
-}
-
 DwarfException::DwarfException(raw_ostream &OS, AsmPrinter *A,
                                const MCAsmInfo *T)
   : Dwarf(OS, A, T, "eh"), shouldEmitTable(false), shouldEmitMoves(false),
     shouldEmitTableModule(false), shouldEmitMovesModule(false),
     ExceptionTimer(0) {
   if (TimePassesIsEnabled)
-    ExceptionTimer = new Timer("DWARF Exception Writer",
-                               getDwarfTimerGroup());
+    ExceptionTimer = new Timer("DWARF Exception Writer");
 }
 
 DwarfException::~DwarfException() {
@@ -292,13 +286,14 @@ void DwarfException::EmitFDE(const FunctionEHFrameInfo &EHFrameInfo) {
       Asm->EmitULEB128Bytes(is4Byte ? 4 : 8);
       Asm->EOL("Augmentation size");
 
-      // We force 32-bits here because we've encoded our LSDA in the CIE with
-      // `dwarf::DW_EH_PE_sdata4'. And the CIE and FDE should agree.
       if (EHFrameInfo.hasLandingPads)
-        EmitReference("exception", EHFrameInfo.Number, true, true);
-      else
-        Asm->EmitInt32((int)0);
-
+        EmitReference("exception", EHFrameInfo.Number, true, false);
+      else {
+        if (is4Byte)
+          Asm->EmitInt32((int)0);
+        else
+          Asm->EmitInt64((int)0);
+      }
       Asm->EOL("Language Specific Data Area");
     } else {
       Asm->EmitULEB128Bytes(0);
diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp
index 3887e6d44e4b..92849d31e549 100644
--- a/lib/CodeGen/BranchFolding.cpp
+++ b/lib/CodeGen/BranchFolding.cpp
@@ -98,7 +98,7 @@ BranchFolder::BranchFolder(bool defaultEnableTailMerge) {
 /// function, updating the CFG.
 void BranchFolder::RemoveDeadBlock(MachineBasicBlock *MBB) {
   assert(MBB->pred_empty() && "MBB must be dead!");
-  DEBUG(errs() << "\nRemoving MBB: " << *MBB);
+  DEBUG(dbgs() << "\nRemoving MBB: " << *MBB);
 
   MachineFunction *MF = MBB->getParent();
   // drop all successors.
@@ -636,7 +636,7 @@ unsigned BranchFolder::CreateCommonTailOnlyBlock(MachineBasicBlock *&PredBB,
     SameTails[commonTailIndex].getTailStartPos();
   MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock();
 
-  DEBUG(errs() << "\nSplitting BB#" << MBB->getNumber() << ", size "
+  DEBUG(dbgs() << "\nSplitting BB#" << MBB->getNumber() << ", size "
                << maxCommonTailLength);
 
   MachineBasicBlock *newMBB = SplitMBBAt(*MBB, BBI);
@@ -666,18 +666,18 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
   // this many instructions in common.
   unsigned minCommonTailLength = TailMergeSize;
 
-  DEBUG(errs() << "\nTryTailMergeBlocks: ";
+  DEBUG(dbgs() << "\nTryTailMergeBlocks: ";
         for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i)
-          errs() << "BB#" << MergePotentials[i].getBlock()->getNumber()
+          dbgs() << "BB#" << MergePotentials[i].getBlock()->getNumber()
                  << (i == e-1 ? "" : ", ");
-        errs() << "\n";
+        dbgs() << "\n";
         if (SuccBB) {
-          errs() << "  with successor BB#" << SuccBB->getNumber() << '\n';
+          dbgs() << "  with successor BB#" << SuccBB->getNumber() << '\n';
           if (PredBB)
-            errs() << "  which has fall-through from BB#"
+            dbgs() << "  which has fall-through from BB#"
                    << PredBB->getNumber() << "\n";
         }
-        errs() << "Looking for common tails of at least "
+        dbgs() << "Looking for common tails of at least "
                << minCommonTailLength << " instruction"
                << (minCommonTailLength == 1 ? "" : "s") << '\n';
        );
@@ -748,19 +748,19 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
     MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock();
     // MBB is common tail.  Adjust all other BB's to jump to this one.
     // Traversal must be forwards so erases work.
-    DEBUG(errs() << "\nUsing common tail in BB#" << MBB->getNumber()
+    DEBUG(dbgs() << "\nUsing common tail in BB#" << MBB->getNumber()
                  << " for ");
     for (unsigned int i=0, e = SameTails.size(); i != e; ++i) {
       if (commonTailIndex == i)
         continue;
-      DEBUG(errs() << "BB#" << SameTails[i].getBlock()->getNumber()
+      DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber()
                    << (i == e-1 ? "" : ", "));
       // Hack the end off BB i, making it jump to BB commonTailIndex instead.
       ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB);
       // BB i is no longer a predecessor of SuccBB; remove it from the worklist.
       MergePotentials.erase(SameTails[i].getMPIter());
     }
-    DEBUG(errs() << "\n");
+    DEBUG(dbgs() << "\n");
     // We leave commonTailIndex in the worklist in case there are other blocks
     // that match it with a smaller number of instructions.
     MadeChange = true;
@@ -999,7 +999,7 @@ ReoptimizeBlock:
     if (PriorCond.empty() && !PriorTBB && MBB->pred_size() == 1 &&
         PrevBB.succ_size() == 1 &&
         !MBB->hasAddressTaken()) {
-      DEBUG(errs() << "\nMerging into block: " << PrevBB
+      DEBUG(dbgs() << "\nMerging into block: " << PrevBB
                    << "From MBB: " << *MBB);
       PrevBB.splice(PrevBB.end(), MBB, MBB->begin(), MBB->end());
       PrevBB.removeSuccessor(PrevBB.succ_begin());;
@@ -1084,7 +1084,7 @@ ReoptimizeBlock:
         // Reverse the branch so we will fall through on the previous true cond.
         SmallVector<MachineOperand, 4> NewPriorCond(PriorCond);
         if (!TII->ReverseBranchCondition(NewPriorCond)) {
-          DEBUG(errs() << "\nMoving MBB: " << *MBB
+          DEBUG(dbgs() << "\nMoving MBB: " << *MBB
                        << "To make fallthrough to: " << *PriorTBB << "\n");
 
           TII->RemoveBranch(PrevBB);
@@ -1222,7 +1222,7 @@ ReoptimizeBlock:
         // Analyze the branch at the end of the pred.
         MachineBasicBlock *PredBB = *PI;
         MachineFunction::iterator PredFallthrough = PredBB; ++PredFallthrough;
-        MachineBasicBlock *PredTBB, *PredFBB;
+        MachineBasicBlock *PredTBB = 0, *PredFBB = 0;
         SmallVector<MachineOperand, 4> PredCond;
         if (PredBB != MBB && !PredBB->canFallThrough() &&
             !TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)
@@ -1274,7 +1274,7 @@ ReoptimizeBlock:
       // Okay, there is no really great place to put this block.  If, however,
       // the block before this one would be a fall-through if this block were
       // removed, move this block to the end of the function.
-      MachineBasicBlock *PrevTBB, *PrevFBB;
+      MachineBasicBlock *PrevTBB = 0, *PrevFBB = 0;
       SmallVector<MachineOperand, 4> PrevCond;
       if (FallThrough != MF.end() &&
           !TII->AnalyzeBranch(PrevBB, PrevTBB, PrevFBB, PrevCond, true) &&
diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp
index dcffb8a247a9..b8ef219bf9c1 100644
--- a/lib/CodeGen/CalcSpillWeights.cpp
+++ b/lib/CodeGen/CalcSpillWeights.cpp
@@ -37,7 +37,7 @@ void CalculateSpillWeights::getAnalysisUsage(AnalysisUsage &au) const {
 
 bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &fn) {
 
-  DEBUG(errs() << "********** Compute Spill Weights **********\n"
+  DEBUG(dbgs() << "********** Compute Spill Weights **********\n"
                << "********** Function: "
                << fn.getFunction()->getName() << '\n');
 
@@ -95,7 +95,7 @@ bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &fn) {
           SlotIndex defIdx = lis->getInstructionIndex(mi).getDefIndex();
           const LiveRange *dlr =
             lis->getInterval(reg).getLiveRangeContaining(defIdx);
-          if (dlr->end > mbbEnd)
+          if (dlr->end >= mbbEnd)
             weight *= 3.0F;
         }
         regInt.weight += weight;
diff --git a/lib/CodeGen/CodePlacementOpt.cpp b/lib/CodeGen/CodePlacementOpt.cpp
index ff71f6b80673..126700b7fe5b 100644
--- a/lib/CodeGen/CodePlacementOpt.cpp
+++ b/lib/CodeGen/CodePlacementOpt.cpp
@@ -233,7 +233,6 @@ bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF,
       !BotHasFallthrough &&
       HasFallthrough(L->getBottomBlock())) {
     ++NumIntraElim;
-    BotHasFallthrough = true;
   }
 
   return Changed;
diff --git a/lib/CodeGen/ELF.h b/lib/CodeGen/ELF.h
index e303ebb0de91..cb5a8c0eae1d 100644
--- a/lib/CodeGen/ELF.h
+++ b/lib/CodeGen/ELF.h
@@ -82,14 +82,14 @@ namespace llvm {
     const GlobalValue *getGlobalValue() const {
       assert(SourceType == isGV && "This is not a global value");
       return Source.GV;
-    };
+    }
 
     // getExternalSym - If this is an external symbol which originated the
     // elf symbol, return a reference to it.
     const char *getExternalSymbol() const {
       assert(SourceType == isExtSym && "This is not an external symbol");
       return Source.Ext;
-    };
+    }
 
     // getGV - From a global value return a elf symbol to represent it
     static ELFSym *getGV(const GlobalValue *GV, unsigned Bind,
diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp
index 297dd3167623..d5fd051d50b0 100644
--- a/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/lib/CodeGen/LLVMTargetMachine.cpp
@@ -83,7 +83,18 @@ LLVMTargetMachine::LLVMTargetMachine(const Target &T,
   AsmInfo = T.createAsmInfo(TargetTriple);
 }
 
+// Set the default code model for the JIT for a generic target.
+// FIXME: Is small right here? or .is64Bit() ? Large : Small?
+void
+LLVMTargetMachine::setCodeModelForJIT() {
+  setCodeModel(CodeModel::Small);
+}
 
+// Set the default code model for static compilation for a generic target.
+void
+LLVMTargetMachine::setCodeModelForStatic() {
+  setCodeModel(CodeModel::Small);
+}
 
 FileModel::Model
 LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
@@ -130,6 +141,9 @@ bool LLVMTargetMachine::addAssemblyEmitter(PassManagerBase &PM,
 bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM,
                                                   MachineCodeEmitter *MCE,
                                                   CodeGenOpt::Level OptLevel) {
+  // Make sure the code model is set.
+  setCodeModelForStatic();
+  
   if (MCE)
     addSimpleCodeEmitter(PM, OptLevel, *MCE);
   if (PrintEmittedAsm)
@@ -146,6 +160,9 @@ bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM,
 bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM,
                                                   JITCodeEmitter *JCE,
                                                   CodeGenOpt::Level OptLevel) {
+  // Make sure the code model is set.
+  setCodeModelForJIT();
+  
   if (JCE)
     addSimpleCodeEmitter(PM, OptLevel, *JCE);
   if (PrintEmittedAsm)
@@ -162,6 +179,9 @@ bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM,
 bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM,
                                                   ObjectCodeEmitter *OCE,
                                                   CodeGenOpt::Level OptLevel) {
+  // Make sure the code model is set.
+  setCodeModelForStatic();
+  
   if (OCE)
     addSimpleCodeEmitter(PM, OptLevel, *OCE);
   if (PrintEmittedAsm)
@@ -181,6 +201,9 @@ bool LLVMTargetMachine::addPassesToEmitFileFinish(PassManagerBase &PM,
 bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
                                                    MachineCodeEmitter &MCE,
                                                    CodeGenOpt::Level OptLevel) {
+  // Make sure the code model is set.
+  setCodeModelForJIT();
+  
   // Add common CodeGen passes.
   if (addCommonCodeGenPasses(PM, OptLevel))
     return true;
@@ -203,6 +226,9 @@ bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
 bool LLVMTargetMachine::addPassesToEmitMachineCode(PassManagerBase &PM,
                                                    JITCodeEmitter &JCE,
                                                    CodeGenOpt::Level OptLevel) {
+  // Make sure the code model is set.
+  setCodeModelForJIT();
+  
   // Add common CodeGen passes.
   if (addCommonCodeGenPasses(PM, OptLevel))
     return true;
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
index 8806439f5439..452f872f2fba 100644
--- a/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -324,8 +324,7 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
     // of the defining block, potentially live across some blocks, then is
     // live into some number of blocks, but gets killed.  Start by adding a
     // range that goes from this definition to the end of the defining block.
-    LiveRange NewLR(defIndex, getMBBEndIdx(mbb).getNextIndex().getLoadIndex(),
-                    ValNo);
+    LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo);
     DEBUG(errs() << " +" << NewLR);
     interval.addRange(NewLR);
 
@@ -334,10 +333,8 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
     // live interval.
     for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(), 
              E = vi.AliveBlocks.end(); I != E; ++I) {
-      LiveRange LR(
-          getMBBStartIdx(mf_->getBlockNumbered(*I)),
-          getMBBEndIdx(mf_->getBlockNumbered(*I)).getNextIndex().getLoadIndex(),
-          ValNo);
+      MachineBasicBlock *aliveBlock = mf_->getBlockNumbered(*I);
+      LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock), ValNo);
       interval.addRange(LR);
       DEBUG(errs() << " +" << LR);
     }
@@ -415,19 +412,32 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
       // first redefinition of the vreg that we have seen, go back and change
       // the live range in the PHI block to be a different value number.
       if (interval.containsOneValue()) {
-        // Remove the old range that we now know has an incorrect number.
+
         VNInfo *VNI = interval.getValNumInfo(0);
-        MachineInstr *Killer = vi.Kills[0];
-        SlotIndex Start = getMBBStartIdx(Killer->getParent());
-        SlotIndex End = getInstructionIndex(Killer).getDefIndex();
-        DEBUG({
-            errs() << " Removing [" << Start << "," << End << "] from: ";
-            interval.print(errs(), tri_);
-            errs() << "\n";
-          });
-        interval.removeRange(Start, End);        
-        assert(interval.ranges.size() == 1 &&
-               "Newly discovered PHI interval has >1 ranges.");
+        // Phi elimination may have reused the register for multiple identical
+        // phi nodes. There will be a kill per phi. Remove the old ranges that
+        // we now know have an incorrect number.
+        for (unsigned ki=0, ke=vi.Kills.size(); ki != ke; ++ki) {
+          MachineInstr *Killer = vi.Kills[ki];
+          SlotIndex Start = getMBBStartIdx(Killer->getParent());
+          SlotIndex End = getInstructionIndex(Killer).getDefIndex();
+          DEBUG({
+              errs() << "\n\t\trenaming [" << Start << "," << End << "] in: ";
+              interval.print(errs(), tri_);
+            });
+          interval.removeRange(Start, End);
+
+          // Replace the interval with one of a NEW value number.  Note that
+          // this value number isn't actually defined by an instruction, weird
+          // huh? :)
+          LiveRange LR(Start, End,
+                       interval.getNextValue(SlotIndex(Start, true),
+                                             0, false, VNInfoAllocator));
+          LR.valno->setIsPHIDef(true);
+          interval.addRange(LR);
+          LR.valno->addKill(End);
+        }
+
         MachineBasicBlock *killMBB = getMBBFromIndex(VNI->def);
         VNI->addKill(indexes_->getTerminatorGap(killMBB));
         VNI->setHasPHIKill(true);
@@ -435,20 +445,6 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
             errs() << " RESULT: ";
             interval.print(errs(), tri_);
           });
-
-        // Replace the interval with one of a NEW value number.  Note that this
-        // value number isn't actually defined by an instruction, weird huh? :)
-        LiveRange LR(Start, End,
-                     interval.getNextValue(SlotIndex(getMBBStartIdx(Killer->getParent()), true),
-                       0, false, VNInfoAllocator));
-        LR.valno->setIsPHIDef(true);
-        DEBUG(errs() << " replace range with " << LR);
-        interval.addRange(LR);
-        LR.valno->addKill(End);
-        DEBUG({
-            errs() << " RESULT: ";
-            interval.print(errs(), tri_);
-          });
       }
 
       // In the case of PHI elimination, each variable definition is only
@@ -468,7 +464,7 @@ void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb,
         CopyMI = mi;
       ValNo = interval.getNextValue(defIndex, CopyMI, true, VNInfoAllocator);
       
-      SlotIndex killIndex = getMBBEndIdx(mbb).getNextIndex().getLoadIndex();
+      SlotIndex killIndex = getMBBEndIdx(mbb);
       LiveRange LR(defIndex, killIndex, ValNo);
       interval.addRange(LR);
       ValNo->addKill(indexes_->getTerminatorGap(mbb));
@@ -1248,7 +1244,7 @@ bool LiveIntervals::anyKillInMBBAfterIdx(const LiveInterval &li,
       continue;
 
     SlotIndex KillIdx = VNI->kills[j];
-    if (KillIdx > Idx && KillIdx < End)
+    if (KillIdx > Idx && KillIdx <= End)
       return true;
   }
   return false;
@@ -2086,7 +2082,7 @@ LiveRange LiveIntervals::addLiveRangeToEndOfBlock(unsigned reg,
   VN->kills.push_back(indexes_->getTerminatorGap(startInst->getParent()));
   LiveRange LR(
      SlotIndex(getInstructionIndex(startInst).getDefIndex()),
-     getMBBEndIdx(startInst->getParent()).getNextIndex().getBaseIndex(), VN);
+     getMBBEndIdx(startInst->getParent()), VN);
   Interval.addRange(LR);
   
   return LR;
diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp
index a58286d61b8e..74a0d574a232 100644
--- a/lib/CodeGen/MachineBasicBlock.cpp
+++ b/lib/CodeGen/MachineBasicBlock.cpp
@@ -450,14 +450,29 @@ void MachineBasicBlock::ReplaceUsesOfBlockWith(MachineBasicBlock *Old,
 
 /// CorrectExtraCFGEdges - Various pieces of code can cause excess edges in the
 /// CFG to be inserted.  If we have proven that MBB can only branch to DestA and
-/// DestB, remove any other MBB successors from the CFG.  DestA and DestB can
-/// be null.
+/// DestB, remove any other MBB successors from the CFG.  DestA and DestB can be
+/// null.
+/// 
 /// Besides DestA and DestB, retain other edges leading to LandingPads
 /// (currently there can be only one; we don't check or require that here).
 /// Note it is possible that DestA and/or DestB are LandingPads.
 bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA,
                                              MachineBasicBlock *DestB,
                                              bool isCond) {
+  // The values of DestA and DestB frequently come from a call to the
+  // 'TargetInstrInfo::AnalyzeBranch' method. We take our meaning of the initial
+  // values from there.
+  //
+  // 1. If both DestA and DestB are null, then the block ends with no branches
+  //    (it falls through to its successor).
+  // 2. If DestA is set, DestB is null, and isCond is false, then the block ends
+  //    with only an unconditional branch.
+  // 3. If DestA is set, DestB is null, and isCond is true, then the block ends
+  //    with a conditional branch that falls through to a successor (DestB).
+  // 4. If DestA and DestB is set and isCond is true, then the block ends with a
+  //    conditional branch followed by an unconditional branch. DestA is the
+  //    'true' destination and DestB is the 'false' destination.
+
   bool MadeChange = false;
   bool AddedFallThrough = false;
 
@@ -483,14 +498,15 @@ bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA,
   MachineBasicBlock::succ_iterator SI = succ_begin();
   MachineBasicBlock *OrigDestA = DestA, *OrigDestB = DestB;
   while (SI != succ_end()) {
-    if (*SI == DestA) {
+    const MachineBasicBlock *MBB = *SI;
+    if (MBB == DestA) {
       DestA = 0;
       ++SI;
-    } else if (*SI == DestB) {
+    } else if (MBB == DestB) {
       DestB = 0;
       ++SI;
-    } else if ((*SI)->isLandingPad() && 
-               *SI!=OrigDestA && *SI!=OrigDestB) {
+    } else if (MBB->isLandingPad() && 
+               MBB != OrigDestA && MBB != OrigDestB) {
       ++SI;
     } else {
       // Otherwise, this is a superfluous edge, remove it.
@@ -498,12 +514,12 @@ bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA,
       MadeChange = true;
     }
   }
-  if (!AddedFallThrough) {
-    assert(DestA == 0 && DestB == 0 &&
-           "MachineCFG is missing edges!");
-  } else if (isCond) {
+
+  if (!AddedFallThrough)
+    assert(DestA == 0 && DestB == 0 && "MachineCFG is missing edges!");
+  else if (isCond)
     assert(DestA == 0 && "MachineCFG is missing edges!");
-  }
+
   return MadeChange;
 }
 
diff --git a/lib/CodeGen/MachineDominators.cpp b/lib/CodeGen/MachineDominators.cpp
index 0f796f3952c3..408873903b0d 100644
--- a/lib/CodeGen/MachineDominators.cpp
+++ b/lib/CodeGen/MachineDominators.cpp
@@ -17,8 +17,10 @@
 
 using namespace llvm;
 
+namespace llvm {
 TEMPLATE_INSTANTIATION(class DomTreeNodeBase<MachineBasicBlock>);
 TEMPLATE_INSTANTIATION(class DominatorTreeBase<MachineBasicBlock>);
+}
 
 char MachineDominatorTree::ID = 0;
 
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index 12b974d30be9..a761c2daa96b 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Constants.h"
 #include "llvm/Function.h"
 #include "llvm/InlineAsm.h"
+#include "llvm/Type.h"
 #include "llvm/Value.h"
 #include "llvm/Assembly/Writer.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -555,8 +556,13 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
       Operands.back().ParentMI = this;
   
       // If the operand is a register, update the operand's use list.
-      if (Op.isReg())
+      if (Op.isReg()) {
         Operands.back().AddRegOperandToRegInfo(RegInfo);
+        // If the register operand is flagged as early, mark the operand as such
+        unsigned OpNo = Operands.size() - 1;
+        if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1)
+          Operands[OpNo].setIsEarlyClobber(true);
+      }
       return;
     }
   }
@@ -573,8 +579,12 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
 
     // Do explicitly set the reginfo for this operand though, to ensure the
     // next/prev fields are properly nulled out.
-    if (Operands[OpNo].isReg())
+    if (Operands[OpNo].isReg()) {
       Operands[OpNo].AddRegOperandToRegInfo(0);
+      // If the register operand is flagged as early, mark the operand as such
+      if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1)
+        Operands[OpNo].setIsEarlyClobber(true);
+    }
 
   } else if (Operands.size()+1 <= Operands.capacity()) {
     // Otherwise, we have to remove register operands from their register use
@@ -594,8 +604,12 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
     Operands.insert(Operands.begin()+OpNo, Op);
     Operands[OpNo].ParentMI = this;
 
-    if (Operands[OpNo].isReg())
+    if (Operands[OpNo].isReg()) {
       Operands[OpNo].AddRegOperandToRegInfo(RegInfo);
+      // If the register operand is flagged as early, mark the operand as such
+      if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1)
+        Operands[OpNo].setIsEarlyClobber(true);
+    }
     
     // Re-add all the implicit ops.
     for (unsigned i = OpNo+1, e = Operands.size(); i != e; ++i) {
@@ -613,6 +627,11 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
   
     // Re-add all the operands.
     AddRegOperandsToUseLists(*RegInfo);
+
+      // If the register operand is flagged as early, mark the operand as such
+    if (Operands[OpNo].isReg()
+        && TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1)
+      Operands[OpNo].setIsEarlyClobber(true);
   }
 }
 
@@ -1141,7 +1160,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const {
 
   // Briefly indicate whether any call clobbers were omitted.
   if (OmittedAnyCallClobbers) {
-    if (FirstOp) FirstOp = false; else OS << ",";
+    if (!FirstOp) OS << ",";
     OS << " ...";
   }
 
@@ -1159,7 +1178,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const {
   }
 
   if (!debugLoc.isUnknown() && MF) {
-    if (!HaveSemi) OS << ";"; HaveSemi = true;
+    if (!HaveSemi) OS << ";";
 
     // TODO: print InlinedAtLoc information
 
diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp
index 66de5359df99..0a57ea151e72 100644
--- a/lib/CodeGen/MachineLICM.cpp
+++ b/lib/CodeGen/MachineLICM.cpp
@@ -322,7 +322,7 @@ bool MachineLICM::IsLoopInvariantInst(MachineInstr &I) {
 
     // If the loop contains the definition of an operand, then the instruction
     // isn't loop invariant.
-    if (CurLoop->contains(RegInfo->getVRegDef(Reg)->getParent()))
+    if (CurLoop->contains(RegInfo->getVRegDef(Reg)))
       return false;
   }
 
diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp
index 63f4f18e4d51..d561a5bb8c99 100644
--- a/lib/CodeGen/MachineLoopInfo.cpp
+++ b/lib/CodeGen/MachineLoopInfo.cpp
@@ -19,12 +19,14 @@
 #include "llvm/CodeGen/Passes.h"
 using namespace llvm;
 
+namespace llvm {
 #define MLB class LoopBase<MachineBasicBlock, MachineLoop>
 TEMPLATE_INSTANTIATION(MLB);
 #undef MLB
 #define MLIB class LoopInfoBase<MachineBasicBlock, MachineLoop>
 TEMPLATE_INSTANTIATION(MLIB);
 #undef MLIB
+}
 
 char MachineLoopInfo::ID = 0;
 static RegisterPass<MachineLoopInfo>
diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp
index 917d0535b2b8..077231904631 100644
--- a/lib/CodeGen/MachineVerifier.cpp
+++ b/lib/CodeGen/MachineVerifier.cpp
@@ -365,24 +365,6 @@ void
 MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
   const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
 
-  // Start with minimal CFG sanity checks.
-  MachineFunction::const_iterator MBBI = MBB;
-  ++MBBI;
-  if (MBBI != MF->end()) {
-    // Block is not last in function.
-    if (!MBB->isSuccessor(MBBI)) {
-      // Block does not fall through.
-      if (MBB->empty()) {
-        report("MBB doesn't fall through but is empty!", MBB);
-      }
-    }
-  } else {
-    // Block is last in function.
-    if (MBB->empty()) {
-      report("MBB is last in function but is empty!", MBB);
-    }
-  }
-
   // Call AnalyzeBranch. If it succeeds, there several more conditions to check.
   MachineBasicBlock *TBB = 0, *FBB = 0;
   SmallVector<MachineOperand, 4> Cond;
@@ -553,7 +535,8 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
         report("Explicit operand marked as implicit", MO, MONum);
     }
   } else {
-    if (MO->isReg() && !MO->isImplicit() && !TI.isVariadic())
+    // ARM adds %reg0 operands to indicate predicates. We'll allow that.
+    if (MO->isReg() && !MO->isImplicit() && !TI.isVariadic() && MO->getReg())
       report("Extra explicit operand on non-variadic instruction", MO, MONum);
   }
 
diff --git a/lib/CodeGen/PBQP/AnnotatedGraph.h b/lib/CodeGen/PBQP/AnnotatedGraph.h
index 904061ca4fbc..a47dce9e6753 100644
--- a/lib/CodeGen/PBQP/AnnotatedGraph.h
+++ b/lib/CodeGen/PBQP/AnnotatedGraph.h
@@ -132,19 +132,19 @@ public:
   }
 
   NodeData& getNodeData(const NodeIterator &nodeItr) {
-    return getNodeEntry(nodeItr).getNodeData();
+    return PGraph::getNodeEntry(nodeItr).getNodeData();
   }
 
   const NodeData& getNodeData(const NodeIterator &nodeItr) const {
-    return getNodeEntry(nodeItr).getNodeData();
+    return PGraph::getNodeEntry(nodeItr).getNodeData();
   }
 
   EdgeData& getEdgeData(const EdgeIterator &edgeItr) {
-    return getEdgeEntry(edgeItr).getEdgeData();
+    return PGraph::getEdgeEntry(edgeItr).getEdgeData();
   }
 
   const EdgeEntry& getEdgeData(const EdgeIterator &edgeItr) const {
-    return getEdgeEntry(edgeItr).getEdgeData();
+    return PGraph::getEdgeEntry(edgeItr).getEdgeData();
   }
 
   SimpleGraph toSimpleGraph() const {
diff --git a/lib/CodeGen/PBQP/GraphBase.h b/lib/CodeGen/PBQP/GraphBase.h
index cc3e017adda1..0c7493b8957f 100644
--- a/lib/CodeGen/PBQP/GraphBase.h
+++ b/lib/CodeGen/PBQP/GraphBase.h
@@ -298,7 +298,7 @@ public:
 
     for (ConstAdjEdgeIterator adjEdgeItr = adjEdgesBegin(node1Itr),
          adjEdgeEnd = adjEdgesEnd(node1Itr);
-         adjEdgeItr != adjEdgesEnd; ++adjEdgeItr) {
+         adjEdgeItr != adjEdgeEnd; ++adjEdgeItr) {
       if ((getEdgeNode1Itr(*adjEdgeItr) == node2Itr) ||
           (getEdgeNode2Itr(*adjEdgeItr) == node2Itr)) {
         return *adjEdgeItr;
diff --git a/lib/CodeGen/PBQP/HeuristicSolver.h b/lib/CodeGen/PBQP/HeuristicSolver.h
index e786246b4e05..16708779e09d 100644
--- a/lib/CodeGen/PBQP/HeuristicSolver.h
+++ b/lib/CodeGen/PBQP/HeuristicSolver.h
@@ -536,7 +536,7 @@ private:
       else reductionFinished = true;
     }
       
-  };
+  }
 
   void processR1() {
 
diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp
index c62d17958d07..58c3dec4c22c 100644
--- a/lib/CodeGen/PHIElimination.cpp
+++ b/lib/CodeGen/PHIElimination.cpp
@@ -35,6 +35,7 @@ using namespace llvm;
 
 STATISTIC(NumAtomic, "Number of atomic phis lowered");
 STATISTIC(NumSplits, "Number of critical edges split on demand");
+STATISTIC(NumReused, "Number of reused lowered phis");
 
 char PHIElimination::ID = 0;
 static RegisterPass<PHIElimination>
@@ -70,7 +71,7 @@ bool llvm::PHIElimination::runOnMachineFunction(MachineFunction &Fn) {
     Changed |= EliminatePHINodes(Fn, *I);
 
   // Remove dead IMPLICIT_DEF instructions.
-  for (SmallPtrSet<MachineInstr*,4>::iterator I = ImpDefs.begin(),
+  for (SmallPtrSet<MachineInstr*, 4>::iterator I = ImpDefs.begin(),
          E = ImpDefs.end(); I != E; ++I) {
     MachineInstr *DefMI = *I;
     unsigned DefReg = DefMI->getOperand(0).getReg();
@@ -78,6 +79,12 @@ bool llvm::PHIElimination::runOnMachineFunction(MachineFunction &Fn) {
       DefMI->eraseFromParent();
   }
 
+  // Clean up the lowered PHI instructions.
+  for (LoweredPHIMap::iterator I = LoweredPHIs.begin(), E = LoweredPHIs.end();
+       I != E; ++I)
+    Fn.DeleteMachineInstr(I->first);
+
+  LoweredPHIs.clear();
   ImpDefs.clear();
   VRegPHIUseCount.clear();
   return Changed;
@@ -168,6 +175,7 @@ llvm::PHIElimination::FindCopyInsertPoint(MachineBasicBlock &MBB,
 void llvm::PHIElimination::LowerAtomicPHINode(
                                       MachineBasicBlock &MBB,
                                       MachineBasicBlock::iterator AfterPHIsIt) {
+  ++NumAtomic;
   // Unlink the PHI node from the basic block, but don't delete the PHI yet.
   MachineInstr *MPhi = MBB.remove(MBB.begin());
 
@@ -179,6 +187,7 @@ void llvm::PHIElimination::LowerAtomicPHINode(
   MachineFunction &MF = *MBB.getParent();
   const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(DestReg);
   unsigned IncomingReg = 0;
+  bool reusedIncoming = false;  // Is IncomingReg reused from an earlier PHI?
 
   // Insert a register to register copy at the top of the current block (but
   // after any remaining phi nodes) which copies the new incoming register
@@ -190,7 +199,18 @@ void llvm::PHIElimination::LowerAtomicPHINode(
     BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(),
             TII->get(TargetInstrInfo::IMPLICIT_DEF), DestReg);
   else {
-    IncomingReg = MF.getRegInfo().createVirtualRegister(RC);
+    // Can we reuse an earlier PHI node? This only happens for critical edges,
+    // typically those created by tail duplication.
+    unsigned &entry = LoweredPHIs[MPhi];
+    if (entry) {
+      // An identical PHI node was already lowered. Reuse the incoming register.
+      IncomingReg = entry;
+      reusedIncoming = true;
+      ++NumReused;
+      DEBUG(errs() << "Reusing %reg" << IncomingReg << " for " << *MPhi);
+    } else {
+      entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC);
+    }
     TII->copyRegToReg(MBB, AfterPHIsIt, DestReg, IncomingReg, RC, RC);
   }
 
@@ -204,8 +224,20 @@ void llvm::PHIElimination::LowerAtomicPHINode(
     MachineInstr *PHICopy = prior(AfterPHIsIt);
 
     if (IncomingReg) {
+      LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg);
+
       // Increment use count of the newly created virtual register.
-      LV->getVarInfo(IncomingReg).NumUses++;
+      VI.NumUses++;
+
+      // When we are reusing the incoming register, it may already have been
+      // killed in this block. The old kill will also have been inserted at
+      // AfterPHIsIt, so it appears before the current PHICopy.
+      if (reusedIncoming)
+        if (MachineInstr *OldKill = VI.findKill(&MBB)) {
+          DEBUG(errs() << "Remove old kill from " << *OldKill);
+          LV->removeVirtualRegisterKilled(IncomingReg, OldKill);
+          DEBUG(MBB.dump());
+        }
 
       // Add information to LiveVariables to know that the incoming value is
       // killed.  Note that because the value is defined in several places (once
@@ -228,7 +260,7 @@ void llvm::PHIElimination::LowerAtomicPHINode(
 
   // Adjust the VRegPHIUseCount map to account for the removal of this PHI node.
   for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2)
-    --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i + 1).getMBB(),
+    --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i+1).getMBB()->getNumber(),
                                  MPhi->getOperand(i).getReg())];
 
   // Now loop over all of the incoming arguments, changing them to copy into the
@@ -266,7 +298,8 @@ void llvm::PHIElimination::LowerAtomicPHINode(
       FindCopyInsertPoint(opBlock, MBB, SrcReg);
 
     // Insert the copy.
-    TII->copyRegToReg(opBlock, InsertPos, IncomingReg, SrcReg, RC, RC);
+    if (!reusedIncoming && IncomingReg)
+      TII->copyRegToReg(opBlock, InsertPos, IncomingReg, SrcReg, RC, RC);
 
     // Now update live variable information if we have it.  Otherwise we're done
     if (!LV) continue;
@@ -283,7 +316,7 @@ void llvm::PHIElimination::LowerAtomicPHINode(
     // point later.
 
     // Is it used by any PHI instructions in this block?
-    bool ValueIsUsed = VRegPHIUseCount[BBVRegPair(&opBlock, SrcReg)] != 0;
+    bool ValueIsUsed = VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)];
 
     // Okay, if we now know that the value is not live out of the block, we can
     // add a kill marker in this block saying that it kills the incoming value!
@@ -293,11 +326,10 @@ void llvm::PHIElimination::LowerAtomicPHINode(
       // terminator instruction at the end of the block may also use the value.
       // In this case, we should mark *it* as being the killing block, not the
       // copy.
-      MachineBasicBlock::iterator KillInst = prior(InsertPos);
+      MachineBasicBlock::iterator KillInst;
       MachineBasicBlock::iterator Term = opBlock.getFirstTerminator();
-      if (Term != opBlock.end()) {
-        if (Term->readsRegister(SrcReg))
-          KillInst = Term;
+      if (Term != opBlock.end() && Term->readsRegister(SrcReg)) {
+        KillInst = Term;
 
         // Check that no other terminators use values.
 #ifndef NDEBUG
@@ -308,7 +340,17 @@ void llvm::PHIElimination::LowerAtomicPHINode(
                  "they are the first terminator in a block!");
         }
 #endif
+      } else if (reusedIncoming || !IncomingReg) {
+        // We may have to rewind a bit if we didn't insert a copy this time.
+        KillInst = Term;
+        while (KillInst != opBlock.begin())
+          if ((--KillInst)->readsRegister(SrcReg))
+            break;
+      } else {
+        // We just inserted this copy.
+        KillInst = prior(InsertPos);
       }
+      assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction");
 
       // Finally, mark it killed.
       LV->addVirtualRegisterKilled(SrcReg, KillInst);
@@ -319,9 +361,9 @@ void llvm::PHIElimination::LowerAtomicPHINode(
     }
   }
 
-  // Really delete the PHI instruction now!
-  MF.DeleteMachineInstr(MPhi);
-  ++NumAtomic;
+  // Really delete the PHI instruction now, if it is not in the LoweredPHIs map.
+  if (reusedIncoming || !IncomingReg)
+    MF.DeleteMachineInstr(MPhi);
 }
 
 /// analyzePHINodes - Gather information about the PHI nodes in here. In
@@ -335,14 +377,15 @@ void llvm::PHIElimination::analyzePHINodes(const MachineFunction& Fn) {
     for (MachineBasicBlock::const_iterator BBI = I->begin(), BBE = I->end();
          BBI != BBE && BBI->getOpcode() == TargetInstrInfo::PHI; ++BBI)
       for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
-        ++VRegPHIUseCount[BBVRegPair(BBI->getOperand(i + 1).getMBB(),
+        ++VRegPHIUseCount[BBVRegPair(BBI->getOperand(i+1).getMBB()->getNumber(),
                                      BBI->getOperand(i).getReg())];
 }
 
 bool llvm::PHIElimination::SplitPHIEdges(MachineFunction &MF,
                                          MachineBasicBlock &MBB,
                                          LiveVariables &LV) {
-  if (MBB.empty() || MBB.front().getOpcode() != TargetInstrInfo::PHI)
+  if (MBB.empty() || MBB.front().getOpcode() != TargetInstrInfo::PHI ||
+      MBB.isLandingPad())
     return false;   // Quick exit for basic blocks without PHIs.
 
   for (MachineBasicBlock::const_iterator BBI = MBB.begin(), BBE = MBB.end();
@@ -408,3 +451,34 @@ MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A,
 
   return NMBB;
 }
+
+unsigned
+PHIElimination::PHINodeTraits::getHashValue(const MachineInstr *MI) {
+  if (!MI || MI==getEmptyKey() || MI==getTombstoneKey())
+    return DenseMapInfo<MachineInstr*>::getHashValue(MI);
+  unsigned hash = 0;
+  for (unsigned ni = 1, ne = MI->getNumOperands(); ni != ne; ni += 2)
+    hash = hash*37 + DenseMapInfo<BBVRegPair>::
+      getHashValue(BBVRegPair(MI->getOperand(ni+1).getMBB()->getNumber(),
+                              MI->getOperand(ni).getReg()));
+  return hash;
+}
+
+bool PHIElimination::PHINodeTraits::isEqual(const MachineInstr *LHS,
+                                            const MachineInstr *RHS) {
+  const MachineInstr *EmptyKey = getEmptyKey();
+  const MachineInstr *TombstoneKey = getTombstoneKey();
+  if (!LHS || !RHS || LHS==EmptyKey || RHS==EmptyKey ||
+      LHS==TombstoneKey || RHS==TombstoneKey)
+    return LHS==RHS;
+
+  unsigned ne = LHS->getNumOperands();
+  if (ne != RHS->getNumOperands())
+      return false;
+  // Ignore operand 0, the defined register.
+  for (unsigned ni = 1; ni != ne; ni += 2)
+    if (LHS->getOperand(ni).getReg() != RHS->getOperand(ni).getReg() ||
+        LHS->getOperand(ni+1).getMBB() != RHS->getOperand(ni+1).getMBB())
+      return false;
+  return true;
+}
diff --git a/lib/CodeGen/PHIElimination.h b/lib/CodeGen/PHIElimination.h
index b0b71ce2bc5e..1bcc9dc7d9ca 100644
--- a/lib/CodeGen/PHIElimination.h
+++ b/lib/CodeGen/PHIElimination.h
@@ -16,8 +16,6 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/Target/TargetInstrInfo.h"
 
-#include <map>
-
 namespace llvm {
 
   /// Lower PHI instructions to copies.  
@@ -120,8 +118,8 @@ namespace llvm {
       return I;
     }
 
-    typedef std::pair<const MachineBasicBlock*, unsigned> BBVRegPair;
-    typedef std::map<BBVRegPair, unsigned> VRegPHIUse;
+    typedef std::pair<unsigned, unsigned> BBVRegPair;
+    typedef DenseMap<BBVRegPair, unsigned> VRegPHIUse;
 
     VRegPHIUse VRegPHIUseCount;
     PHIDefMap PHIDefs;
@@ -129,6 +127,17 @@ namespace llvm {
 
     // Defs of PHI sources which are implicit_def.
     SmallPtrSet<MachineInstr*, 4> ImpDefs;
+
+    // Lowered PHI nodes may be reused. We provide special DenseMap traits to
+    // match PHI nodes with identical arguments.
+    struct PHINodeTraits : public DenseMapInfo<MachineInstr*> {
+      static unsigned getHashValue(const MachineInstr *PtrVal);
+      static bool isEqual(const MachineInstr *LHS, const MachineInstr *RHS);
+    };
+
+    // Map reusable lowered PHI node -> incoming join register.
+    typedef DenseMap<MachineInstr*, unsigned, PHINodeTraits> LoweredPHIMap;
+    LoweredPHIMap LoweredPHIs;
   };
 
 }
diff --git a/lib/CodeGen/PreAllocSplitting.cpp b/lib/CodeGen/PreAllocSplitting.cpp
index b0d7a4757eff..1c5222c38e7c 100644
--- a/lib/CodeGen/PreAllocSplitting.cpp
+++ b/lib/CodeGen/PreAllocSplitting.cpp
@@ -378,7 +378,7 @@ PreAllocSplitting::UpdateSpillSlotInterval(VNInfo *ValNo, SlotIndex SpillIndex,
 
   SmallPtrSet<MachineBasicBlock*, 4> Processed;
   SlotIndex EndIdx = LIs->getMBBEndIdx(MBB);
-  LiveRange SLR(SpillIndex, EndIdx.getNextSlot(), CurrSValNo);
+  LiveRange SLR(SpillIndex, EndIdx, CurrSValNo);
   CurrSLI->addRange(SLR);
   Processed.insert(MBB);
 
@@ -475,7 +475,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI,
     SlotIndex EndIndex = LIs->getMBBEndIdx(MBB);
     
     RetVNI = NewVNs[Walker];
-    LI->addRange(LiveRange(DefIndex, EndIndex.getNextSlot(), RetVNI));
+    LI->addRange(LiveRange(DefIndex, EndIndex, RetVNI));
   } else if (!ContainsDefs && ContainsUses) {
     SmallPtrSet<MachineInstr*, 2>& BlockUses = Uses[MBB];
     
@@ -511,8 +511,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI,
     UseIndex = UseIndex.getUseIndex();
     SlotIndex EndIndex;
     if (IsIntraBlock) {
-      EndIndex = LIs->getInstructionIndex(UseI);
-      EndIndex = EndIndex.getUseIndex();
+      EndIndex = LIs->getInstructionIndex(UseI).getDefIndex();
     } else
       EndIndex = LIs->getMBBEndIdx(MBB);
 
@@ -521,7 +520,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI,
     RetVNI = PerformPHIConstruction(Walker, MBB, LI, Visited, Defs, Uses,
                                     NewVNs, LiveOut, Phis, false, true);
     
-    LI->addRange(LiveRange(UseIndex, EndIndex.getNextSlot(), RetVNI));
+    LI->addRange(LiveRange(UseIndex, EndIndex, RetVNI));
     
     // FIXME: Need to set kills properly for inter-block stuff.
     if (RetVNI->isKill(UseIndex)) RetVNI->removeKill(UseIndex);
@@ -571,8 +570,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI,
     StartIndex = foundDef ? StartIndex.getDefIndex() : StartIndex.getUseIndex();
     SlotIndex EndIndex;
     if (IsIntraBlock) {
-      EndIndex = LIs->getInstructionIndex(UseI);
-      EndIndex = EndIndex.getUseIndex();
+      EndIndex = LIs->getInstructionIndex(UseI).getDefIndex();
     } else
       EndIndex = LIs->getMBBEndIdx(MBB);
 
@@ -582,7 +580,7 @@ PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI,
       RetVNI = PerformPHIConstruction(Walker, MBB, LI, Visited, Defs, Uses,
                                       NewVNs, LiveOut, Phis, false, true);
 
-    LI->addRange(LiveRange(StartIndex, EndIndex.getNextSlot(), RetVNI));
+    LI->addRange(LiveRange(StartIndex, EndIndex, RetVNI));
     
     if (foundUse && RetVNI->isKill(StartIndex))
       RetVNI->removeKill(StartIndex);
@@ -663,7 +661,7 @@ PreAllocSplitting::PerformPHIConstructionFallBack(MachineBasicBlock::iterator Us
     for (DenseMap<MachineBasicBlock*, VNInfo*>::iterator I =
            IncomingVNs.begin(), E = IncomingVNs.end(); I != E; ++I) {
       I->second->setHasPHIKill(true);
-      SlotIndex KillIndex = LIs->getMBBEndIdx(I->first);
+      SlotIndex KillIndex(LIs->getMBBEndIdx(I->first), true);
       if (!I->second->isKill(KillIndex))
         I->second->addKill(KillIndex);
     }
@@ -671,11 +669,10 @@ PreAllocSplitting::PerformPHIConstructionFallBack(MachineBasicBlock::iterator Us
       
   SlotIndex EndIndex;
   if (IsIntraBlock) {
-    EndIndex = LIs->getInstructionIndex(UseI);
-    EndIndex = EndIndex.getUseIndex();
+    EndIndex = LIs->getInstructionIndex(UseI).getDefIndex();
   } else
     EndIndex = LIs->getMBBEndIdx(MBB);
-  LI->addRange(LiveRange(StartIndex, EndIndex.getNextSlot(), RetVNI));
+  LI->addRange(LiveRange(StartIndex, EndIndex, RetVNI));
   if (IsIntraBlock)
     RetVNI->addKill(EndIndex);
 
@@ -902,8 +899,6 @@ MachineInstr* PreAllocSplitting::FoldSpill(unsigned vreg,
                                            MachineBasicBlock* MBB,
                                            int& SS,
                                     SmallPtrSet<MachineInstr*, 4>& RefsInMBB) {
-  MachineBasicBlock::iterator Pt = MBB->begin();
-
   // Go top down if RefsInMBB is empty.
   if (RefsInMBB.empty())
     return 0;
diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp
index e94247f3714c..709d46a310b1 100644
--- a/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/lib/CodeGen/PrologEpilogInserter.cpp
@@ -860,7 +860,7 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
               // Remove all instructions up 'til the last use, since they're
               // just calculating the value we already have.
               BB->erase(I, LastUseMI);
-              MI = I = LastUseMI;
+              I = LastUseMI;
 
               // Extend the live range of the scratch register
               PrevLastUseMI->getOperand(PrevLastUseOp).setIsKill(false);
diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp
index c02d47b9a384..9e97d89c62e1 100644
--- a/lib/CodeGen/RegAllocLinearScan.cpp
+++ b/lib/CodeGen/RegAllocLinearScan.cpp
@@ -891,7 +891,7 @@ namespace {
     const RALinScan &Allocator;
 
   public:
-    WeightCompare(const RALinScan &Alloc) : Allocator(Alloc) {};
+    WeightCompare(const RALinScan &Alloc) : Allocator(Alloc) {}
 
     typedef std::pair<unsigned, float> RegWeightPair;
     bool operator()(const RegWeightPair &LHS, const RegWeightPair &RHS) const {
diff --git a/lib/CodeGen/RegAllocLocal.cpp b/lib/CodeGen/RegAllocLocal.cpp
index 7bb020a65e7e..aea5cff79ba1 100644
--- a/lib/CodeGen/RegAllocLocal.cpp
+++ b/lib/CodeGen/RegAllocLocal.cpp
@@ -233,14 +233,17 @@ namespace {
     /// in one of several ways: if the register is available in a physical
     /// register already, it uses that physical register.  If the value is not
     /// in a physical register, and if there are physical registers available,
-    /// it loads it into a register.  If register pressure is high, and it is
-    /// possible, it tries to fold the load of the virtual register into the
-    /// instruction itself.  It avoids doing this if register pressure is low to
-    /// improve the chance that subsequent instructions can use the reloaded
-    /// value.  This method returns the modified instruction.
+    /// it loads it into a register: PhysReg if that is an available physical
+    /// register, otherwise any physical register of the right class.
+    /// If register pressure is high, and it is possible, it tries to fold the
+    /// load of the virtual register into the instruction itself.  It avoids
+    /// doing this if register pressure is low to improve the chance that
+    /// subsequent instructions can use the reloaded value.  This method
+    /// returns the modified instruction.
     ///
     MachineInstr *reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
-                                unsigned OpNum, SmallSet<unsigned, 4> &RRegs);
+                                unsigned OpNum, SmallSet<unsigned, 4> &RRegs,
+                                unsigned PhysReg);
 
     /// ComputeLocalLiveness - Computes liveness of registers within a basic
     /// block, setting the killed/dead flags as appropriate.
@@ -471,15 +474,17 @@ unsigned RALocal::getReg(MachineBasicBlock &MBB, MachineInstr *I,
 /// one of several ways: if the register is available in a physical register
 /// already, it uses that physical register.  If the value is not in a physical
 /// register, and if there are physical registers available, it loads it into a
+/// register: PhysReg if that is an available physical register, otherwise any
 /// register.  If register pressure is high, and it is possible, it tries to
 /// fold the load of the virtual register into the instruction itself.  It
 /// avoids doing this if register pressure is low to improve the chance that
-/// subsequent instructions can use the reloaded value.  This method returns the
-/// modified instruction.
+/// subsequent instructions can use the reloaded value.  This method returns
+/// the modified instruction.
 ///
 MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
                                      unsigned OpNum,
-                                     SmallSet<unsigned, 4> &ReloadedRegs) {
+                                     SmallSet<unsigned, 4> &ReloadedRegs,
+                                     unsigned PhysReg) {
   unsigned VirtReg = MI->getOperand(OpNum).getReg();
 
   // If the virtual register is already available, just update the instruction
@@ -494,7 +499,11 @@ MachineInstr *RALocal::reloadVirtReg(MachineBasicBlock &MBB, MachineInstr *MI,
   // Otherwise, we need to fold it into the current instruction, or reload it.
   // If we have registers available to hold the value, use them.
   const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(VirtReg);
-  unsigned PhysReg = getFreeReg(RC);
+  // If we already have a PhysReg (this happens when the instruction is a
+  // reg-to-reg copy with a PhysReg destination) use that.
+  if (!PhysReg || !TargetRegisterInfo::isPhysicalRegister(PhysReg) ||
+      !isPhysRegAvailable(PhysReg))
+    PhysReg = getFreeReg(RC);
   int FrameIndex = getStackSpaceFor(VirtReg, RC);
 
   if (PhysReg) {   // Register is available, allocate it!
@@ -752,6 +761,12 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) {
         errs() << '\n';
       });
 
+    // Determine whether this is a copy instruction.  The cases where the
+    // source or destination are phys regs are handled specially.
+    unsigned SrcCopyReg, DstCopyReg, SrcCopySubReg, DstCopySubReg;
+    bool isCopy = TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg, 
+                                   SrcCopySubReg, DstCopySubReg);
+
     // Loop over the implicit uses, making sure that they are at the head of the
     // use order list, so they don't get reallocated.
     if (TID.ImplicitUses) {
@@ -835,7 +850,8 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) {
       // here we are looking for only used operands (never def&use)
       if (MO.isReg() && !MO.isDef() && MO.getReg() && !MO.isImplicit() &&
           TargetRegisterInfo::isVirtualRegister(MO.getReg()))
-        MI = reloadVirtReg(MBB, MI, i, ReloadedRegs);
+        MI = reloadVirtReg(MBB, MI, i, ReloadedRegs,
+                           isCopy ? DstCopyReg : 0);
     }
 
     // If this instruction is the last user of this register, kill the
@@ -948,8 +964,17 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) {
         unsigned DestPhysReg;
 
         // If DestVirtReg already has a value, use it.
-        if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg)))
-          DestPhysReg = getReg(MBB, MI, DestVirtReg);
+        if (!(DestPhysReg = getVirt2PhysRegMapSlot(DestVirtReg))) {
+          // If this is a copy, the source reg is a phys reg, and
+          // that reg is available, use that phys reg for DestPhysReg.
+          if (isCopy &&
+              TargetRegisterInfo::isPhysicalRegister(SrcCopyReg) &&
+              isPhysRegAvailable(SrcCopyReg)) {
+            DestPhysReg = SrcCopyReg;
+            assignVirtToPhysReg(DestVirtReg, DestPhysReg);
+          } else
+            DestPhysReg = getReg(MBB, MI, DestVirtReg);
+        }
         MF->getRegInfo().setPhysRegUsed(DestPhysReg);
         markVirtRegModified(DestVirtReg);
         getVirtRegLastUse(DestVirtReg) = std::make_pair((MachineInstr*)0, 0);
@@ -995,9 +1020,9 @@ void RALocal::AllocateBasicBlock(MachineBasicBlock &MBB) {
     // Finally, if this is a noop copy instruction, zap it.  (Except that if
     // the copy is dead, it must be kept to avoid messing up liveness info for
     // the register scavenger.  See pr4100.)
-    unsigned SrcReg, DstReg, SrcSubReg, DstSubReg;
-    if (TII->isMoveInstr(*MI, SrcReg, DstReg, SrcSubReg, DstSubReg) &&
-        SrcReg == DstReg && DeadDefs.empty())
+    if (TII->isMoveInstr(*MI, SrcCopyReg, DstCopyReg,
+                         SrcCopySubReg, DstCopySubReg) &&
+        SrcCopyReg == DstCopyReg && DeadDefs.empty())
       MBB.erase(MI);
   }
 
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 2b5218786003..e6aa14cd73e1 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -2755,7 +2755,34 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
     return SDValue(N, 0);
 
-  return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue();
+  if (N1C) {
+    SDValue NewSRL = visitShiftByConstant(N, N1C->getZExtValue());
+    if (NewSRL.getNode())
+      return NewSRL;
+  }
+
+  // Here is a common situation. We want to optimize:
+  //
+  //   %a = ...
+  //   %b = and i32 %a, 2
+  //   %c = srl i32 %b, 1
+  //   brcond i32 %c ...
+  //
+  // into
+  // 
+  //   %a = ...
+  //   %b = and %a, 2
+  //   %c = setcc eq %b, 0
+  //   brcond %c ...
+  //
+  // However when after the source operand of SRL is optimized into AND, the SRL
+  // itself may not be optimized further. Look for it and add the BRCOND into
+  // the worklist.
+  if (N->hasOneUse() &&
+      N->use_begin()->getOpcode() == ISD::BRCOND)
+    AddToWorkList(*N->use_begin());
+
+  return SDValue();
 }
 
 SDValue DAGCombiner::visitCTLZ(SDNode *N) {
@@ -3202,19 +3229,6 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
                        X, DAG.getConstant(Mask, VT));
   }
 
-  // Fold (zext (and x, cst)) -> (and (zext x), cst)
-  if (N0.getOpcode() == ISD::AND &&
-      N0.getOperand(1).getOpcode() == ISD::Constant &&
-      N0.getOperand(0).getOpcode() != ISD::TRUNCATE &&
-      N0.getOperand(0).hasOneUse()) {
-    APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
-    Mask.zext(VT.getSizeInBits());
-    return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
-                       DAG.getNode(ISD::ZERO_EXTEND, N->getDebugLoc(), VT,
-                                   N0.getOperand(0)),
-                       DAG.getConstant(Mask, VT));
-  }
-
   // fold (zext (load x)) -> (zext (truncate (zextload x)))
   if (ISD::isNON_EXTLoad(N0.getNode()) &&
       ((!LegalOperations && !cast<LoadSDNode>(N0)->isVolatile()) ||
diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp
index 4ead9c9d8c42..33694f283e75 100644
--- a/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -43,7 +43,6 @@
 #include "llvm/GlobalVariable.h"
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
-#include "llvm/LLVMContext.h"
 #include "llvm/CodeGen/FastISel.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
@@ -334,7 +333,7 @@ bool FastISel::SelectCall(User *I) {
     return true;
   case Intrinsic::dbg_declare: {
     DbgDeclareInst *DI = cast<DbgDeclareInst>(I);
-    if (!isValidDebugInfoIntrinsic(*DI, CodeGenOpt::None) || !DW
+    if (!DIDescriptor::ValidDebugInfo(DI->getVariable(), CodeGenOpt::None)||!DW
         || !DW->ShouldEmitDwarfDebug())
       return true;
 
@@ -349,11 +348,8 @@ bool FastISel::SelectCall(User *I) {
     if (SI == StaticAllocaMap.end()) break; // VLAs.
     int FI = SI->second;
     if (MMI) {
-      MetadataContext &TheMetadata = 
-        DI->getParent()->getContext().getMetadata();
-      unsigned MDDbgKind = TheMetadata.getMDKind("dbg");
-      MDNode *Dbg = TheMetadata.getMD(MDDbgKind, DI);
-      MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg);
+      if (MDNode *Dbg = DI->getMetadata("dbg"))
+        MMI->setVariableDbgInfo(DI->getVariable(), FI, Dbg);
     }
     return true;
   }
@@ -548,9 +544,6 @@ FastISel::SelectInstruction(Instruction *I) {
 /// the CFG.
 void
 FastISel::FastEmitBranch(MachineBasicBlock *MSucc) {
-  MachineFunction::iterator NextMBB =
-     llvm::next(MachineFunction::iterator(MBB));
-
   if (MBB->isLayoutSuccessor(MSucc)) {
     // The unconditional fall-through case, which needs no instructions.
   } else {
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index f9c05d081da0..474d83396660 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -187,7 +187,6 @@ SDValue
 SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT,  DebugLoc dl, 
                                                  SDValue N1, SDValue N2,
                                              SmallVectorImpl<int> &Mask) const {
-  EVT EltVT = NVT.getVectorElementType();
   unsigned NumMaskElts = VT.getVectorNumElements();
   unsigned NumDestElts = NVT.getVectorNumElements();
   unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
@@ -461,8 +460,7 @@ SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
          !ST->getMemoryVT().isVector() &&
          "Unaligned store of unknown type.");
   // Get the half-size VT
-  EVT NewStoredVT =
-    (MVT::SimpleValueType)(ST->getMemoryVT().getSimpleVT().SimpleTy - 1);
+  EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext());
   int NumBits = NewStoredVT.getSizeInBits();
   int IncrementSize = NumBits / 8;
 
@@ -1170,8 +1168,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
         Tmp2 = LegalizeOp(Ch);
       } else if (SrcWidth & (SrcWidth - 1)) {
         // If not loading a power-of-2 number of bits, expand as two loads.
-        assert(SrcVT.isExtended() && !SrcVT.isVector() &&
-               "Unsupported extload!");
+        assert(!SrcVT.isVector() && "Unsupported extload!");
         unsigned RoundWidth = 1 << Log2_32(SrcWidth);
         assert(RoundWidth < SrcWidth);
         unsigned ExtraWidth = SrcWidth - RoundWidth;
@@ -1384,8 +1381,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) {
                                    SVOffset, NVT, isVolatile, Alignment);
       } else if (StWidth & (StWidth - 1)) {
         // If not storing a power-of-2 number of bits, expand as two stores.
-        assert(StVT.isExtended() && !StVT.isVector() &&
-               "Unsupported truncstore!");
+        assert(!StVT.isVector() && "Unsupported truncstore!");
         unsigned RoundWidth = 1 << Log2_32(StWidth);
         assert(RoundWidth < StWidth);
         unsigned ExtraWidth = StWidth - RoundWidth;
@@ -1869,7 +1865,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
                     0, TLI.getLibcallCallingConv(LC), false,
                     /*isReturnValueUsed=*/true,
                     Callee, Args, DAG,
-                    Node->getDebugLoc());
+                    Node->getDebugLoc(), DAG.GetOrdering(Node));
 
   // Legalize the call sequence, starting with the chain.  This will advance
   // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
@@ -2274,7 +2270,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node,
                       false, false, false, false, 0, CallingConv::C, false,
                       /*isReturnValueUsed=*/true,
                       DAG.getExternalSymbol("abort", TLI.getPointerTy()),
-                      Args, DAG, dl);
+                      Args, DAG, dl, DAG.GetOrdering(Node));
     Results.push_back(CallResult.second);
     break;
   }
@@ -2750,7 +2746,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node,
     SDValue RHS = Node->getOperand(1);
     SDValue BottomHalf;
     SDValue TopHalf;
-    static unsigned Ops[2][3] =
+    static const unsigned Ops[2][3] =
         { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
           { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
     bool isSigned = Node->getOpcode() == ISD::SMULO;
@@ -2967,7 +2963,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node,
     break;
   case ISD::BSWAP: {
     unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
-    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1);
+    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
     Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1);
     Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1,
                           DAG.getConstant(DiffBits, TLI.getShiftAmountTy()));
diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 84e39b480396..2831617ebd46 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -637,7 +637,8 @@ void DAGTypeLegalizer::SoftenSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
     }
   }
 
-  EVT RetVT = MVT::i32; // FIXME: is this the correct return type?
+  // Use the target specific return value for comparions lib calls.
+  EVT RetVT = TLI.getCmpLibcallReturnType();
   SDValue Ops[2] = { LHSInt, RHSInt };
   NewLHS = MakeLibCall(LC1, RetVT, Ops, 2, false/*sign irrelevant*/, dl);
   NewRHS = DAG.getConstant(0, RetVT);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 2f4457e41fc3..bd3b97a9ff13 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -2026,8 +2026,6 @@ void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
   GetExpandedInteger(NewLHS, LHSLo, LHSHi);
   GetExpandedInteger(NewRHS, RHSLo, RHSHi);
 
-  EVT VT = NewLHS.getValueType();
-
   if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
     if (RHSLo == RHSHi) {
       if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 003cea7a6f43..d9efd4f78ad4 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -1033,7 +1033,8 @@ SDValue DAGTypeLegalizer::MakeLibCall(RTLIB::Libcall LC, EVT RetVT,
     TLI.LowerCallTo(DAG.getEntryNode(), RetTy, isSigned, !isSigned, false,
                     false, 0, TLI.getLibcallCallingConv(LC), false,
                     /*isReturnValueUsed=*/true,
-                    Callee, Args, DAG, dl);
+                    Callee, Args, DAG, dl,
+                    DAG.GetOrdering(DAG.getEntryNode().getNode()));
   return CallInfo.first;
 }
 
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index dbd3e39b5426..a1b6ced0c424 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -464,7 +464,6 @@ void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
 
 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
   EVT LoVT, HiVT;
-  DebugLoc dl = N->getDebugLoc();
   GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
   Lo = DAG.getUNDEF(LoVT);
   Hi = DAG.getUNDEF(HiVT);
diff --git a/lib/CodeGen/SelectionDAG/SDNodeOrdering.h b/lib/CodeGen/SelectionDAG/SDNodeOrdering.h
new file mode 100644
index 000000000000..f88b26d5c422
--- /dev/null
+++ b/lib/CodeGen/SelectionDAG/SDNodeOrdering.h
@@ -0,0 +1,54 @@
+//===-- llvm/CodeGen/SDNodeOrdering.h - SDNode Ordering ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the SDNodeOrdering class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_SDNODEORDERING_H
+#define LLVM_CODEGEN_SDNODEORDERING_H
+
+#include "llvm/ADT/DenseMap.h"
+
+namespace llvm {
+
+class SDNode;
+
+/// SDNodeOrdering - Maps a unique (monotonically increasing) value to each
+/// SDNode that roughly corresponds to the ordering of the original LLVM
+/// instruction. This is used for turning off scheduling, because we'll forgo
+/// the normal scheduling algorithms and output the instructions according to
+/// this ordering.
+class SDNodeOrdering {
+  DenseMap<const SDNode*, unsigned> OrderMap;
+
+  void operator=(const SDNodeOrdering&);   // Do not implement.
+  SDNodeOrdering(const SDNodeOrdering&);   // Do not implement.
+public:
+  SDNodeOrdering() {}
+
+  void add(const SDNode *Node, unsigned O) {
+    OrderMap[Node] = O;
+  }
+  void remove(const SDNode *Node) {
+    DenseMap<const SDNode*, unsigned>::iterator Itr = OrderMap.find(Node);
+    if (Itr != OrderMap.end())
+      OrderMap.erase(Itr);
+  }
+  void clear() {
+    OrderMap.clear();
+  }
+  unsigned getOrder(const SDNode *Node) {
+    return OrderMap[Node];
+  }
+};
+
+} // end llvm namespace
+
+#endif
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index b2ee8bb179ec..d53de347a556 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -20,16 +20,10 @@
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtarget.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-cl::opt<bool>
-DisableInstScheduling("disable-inst-scheduling",
-                      cl::init(false),
-                      cl::desc("Disable instruction scheduling"));
-
 ScheduleDAGSDNodes::ScheduleDAGSDNodes(MachineFunction &mf)
   : ScheduleDAG(mf) {
 }
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index da55e6bf4a01..77301b042022 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -10,7 +10,9 @@
 // This implements the SelectionDAG class.
 //
 //===----------------------------------------------------------------------===//
+
 #include "llvm/CodeGen/SelectionDAG.h"
+#include "SDNodeOrdering.h"
 #include "llvm/Constants.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Function.h"
@@ -48,8 +50,6 @@
 #include <cmath>
 using namespace llvm;
 
-extern cl::opt<bool> DisableInstScheduling;
-
 /// makeVTList - Return an instance of the SDVTList struct initialized with the
 /// specified members.
 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
@@ -554,9 +554,6 @@ void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes,
     }
 
     DeallocateNode(N);
-
-    // Remove the ordering of this node.
-    if (Ordering) Ordering->remove(N);
   }
 }
 
@@ -582,9 +579,6 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
   N->DropOperands();
 
   DeallocateNode(N);
-
-  // Remove the ordering of this node.
-  if (Ordering) Ordering->remove(N);
 }
 
 void SelectionDAG::DeallocateNode(SDNode *N) {
@@ -703,7 +697,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1);
   AddNodeIDCustom(ID, N);
   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
-  if (Ordering) Ordering->remove(Node);
   return Node;
 }
 
@@ -722,7 +715,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2);
   AddNodeIDCustom(ID, N);
   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
-  if (Ordering) Ordering->remove(Node);
   return Node;
 }
 
@@ -741,7 +733,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps);
   AddNodeIDCustom(ID, N);
   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
-  if (Ordering) Ordering->remove(Node);
   return Node;
 }
 
@@ -798,10 +789,8 @@ SelectionDAG::SelectionDAG(TargetLowering &tli, FunctionLoweringInfo &fli)
               getVTList(MVT::Other)),
     Root(getEntryNode()), Ordering(0) {
   AllNodes.push_back(&EntryNode);
-  if (DisableInstScheduling) {
-    Ordering = new NodeOrdering();
-    Ordering->add(&EntryNode);
-  }
+  if (DisableScheduling)
+    Ordering = new SDNodeOrdering();
 }
 
 void SelectionDAG::init(MachineFunction &mf, MachineModuleInfo *mmi,
@@ -840,10 +829,8 @@ void SelectionDAG::clear() {
   EntryNode.UseList = 0;
   AllNodes.push_back(&EntryNode);
   Root = getEntryNode();
-  if (DisableInstScheduling) {
-    Ordering = new NodeOrdering();
-    Ordering->add(&EntryNode);
-  }
+  if (DisableScheduling)
+    Ordering = new SDNodeOrdering();
 }
 
 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) {
@@ -904,17 +891,15 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) {
   ID.AddPointer(&Val);
   void *IP = 0;
   SDNode *N = NULL;
-  if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) {
-    if (Ordering) Ordering->add(N);
+  if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
     if (!VT.isVector())
       return SDValue(N, 0);
-  }
+
   if (!N) {
     N = NodeAllocator.Allocate<ConstantSDNode>();
     new (N) ConstantSDNode(isT, &Val, EltVT);
     CSEMap.InsertNode(N, IP);
     AllNodes.push_back(N);
-    if (Ordering) Ordering->add(N);
   }
 
   SDValue Result(N, 0);
@@ -951,17 +936,15 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
   ID.AddPointer(&V);
   void *IP = 0;
   SDNode *N = NULL;
-  if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) {
-    if (Ordering) Ordering->add(N);
+  if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
     if (!VT.isVector())
       return SDValue(N, 0);
-  }
+
   if (!N) {
     N = NodeAllocator.Allocate<ConstantFPSDNode>();
     new (N) ConstantFPSDNode(isTarget, &V, EltVT);
     CSEMap.InsertNode(N, IP);
     AllNodes.push_back(N);
-    if (Ordering) Ordering->add(N);
   }
 
   SDValue Result(N, 0);
@@ -1016,15 +999,13 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV,
   ID.AddInteger(Offset);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<GlobalAddressSDNode>();
   new (N) GlobalAddressSDNode(Opc, GV, VT, Offset, TargetFlags);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1034,15 +1015,13 @@ SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
   ID.AddInteger(FI);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<FrameIndexSDNode>();
   new (N) FrameIndexSDNode(FI, VT, isTarget);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1056,15 +1035,13 @@ SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
   ID.AddInteger(JTI);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<JumpTableSDNode>();
   new (N) JumpTableSDNode(JTI, VT, isTarget, TargetFlags);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1084,15 +1061,13 @@ SDValue SelectionDAG::getConstantPool(Constant *C, EVT VT,
   ID.AddPointer(C);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<ConstantPoolSDNode>();
   new (N) ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment, TargetFlags);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1113,15 +1088,13 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
   C->AddSelectionDAGCSEId(ID);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<ConstantPoolSDNode>();
   new (N) ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment, TargetFlags);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1130,15 +1103,13 @@ SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0);
   ID.AddPointer(MBB);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<BasicBlockSDNode>();
   new (N) BasicBlockSDNode(MBB);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1154,7 +1125,6 @@ SDValue SelectionDAG::getValueType(EVT VT) {
   N = NodeAllocator.Allocate<VTSDNode>();
   new (N) VTSDNode(VT);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1164,7 +1134,6 @@ SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
   N = NodeAllocator.Allocate<ExternalSymbolSDNode>();
   new (N) ExternalSymbolSDNode(false, Sym, 0, VT);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1177,7 +1146,6 @@ SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
   N = NodeAllocator.Allocate<ExternalSymbolSDNode>();
   new (N) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1190,8 +1158,8 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
     new (N) CondCodeSDNode(Cond);
     CondCodeNodes[Cond] = N;
     AllNodes.push_back(N);
-    if (Ordering) Ordering->add(N);
   }
+
   return SDValue(CondCodeNodes[Cond], 0);
 }
 
@@ -1283,10 +1251,8 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1,
     ID.AddInteger(MaskVec[i]);
 
   void* IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
 
   // Allocate the mask array for the node out of the BumpPtrAllocator, since
   // SDNode doesn't have access to it.  This memory will be "leaked" when
@@ -1298,7 +1264,6 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1,
   new (N) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1316,15 +1281,13 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl,
   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5);
   void* IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   CvtRndSatSDNode *N = NodeAllocator.Allocate<CvtRndSatSDNode>();
   new (N) CvtRndSatSDNode(VT, dl, Ops, 5, Code);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1333,15 +1296,13 @@ SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
   AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0);
   ID.AddInteger(RegNo);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<RegisterSDNode>();
   new (N) RegisterSDNode(RegNo, VT);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1353,15 +1314,13 @@ SDValue SelectionDAG::getLabel(unsigned Opcode, DebugLoc dl,
   AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), &Ops[0], 1);
   ID.AddInteger(LabelID);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<LabelSDNode>();
   new (N) LabelSDNode(Opcode, dl, Root, LabelID);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1375,15 +1334,13 @@ SDValue SelectionDAG::getBlockAddress(BlockAddress *BA, EVT VT,
   ID.AddPointer(BA);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<BlockAddressSDNode>();
   new (N) BlockAddressSDNode(Opc, VT, BA, TargetFlags);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -1396,16 +1353,13 @@ SDValue SelectionDAG::getSrcValue(const Value *V) {
   ID.AddPointer(V);
 
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
 
   SDNode *N = NodeAllocator.Allocate<SrcValueSDNode>();
   new (N) SrcValueSDNode(V);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -2316,16 +2270,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) {
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0);
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<SDNode>();
   new (N) SDNode(Opcode, DL, getVTList(VT));
   CSEMap.InsertNode(N, IP);
 
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -2549,10 +2501,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL,
     SDValue Ops[1] = { Operand };
     AddNodeIDNode(ID, Opcode, VTs, Ops, 1);
     void *IP = 0;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return SDValue(E, 0);
-    }
+
     N = NodeAllocator.Allocate<UnarySDNode>();
     new (N) UnarySDNode(Opcode, DL, VTs, Operand);
     CSEMap.InsertNode(N, IP);
@@ -2562,7 +2513,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL,
   }
 
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -2970,10 +2920,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
     void *IP = 0;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return SDValue(E, 0);
-    }
+
     N = NodeAllocator.Allocate<BinarySDNode>();
     new (N) BinarySDNode(Opcode, DL, VTs, N1, N2);
     CSEMap.InsertNode(N, IP);
@@ -2983,7 +2932,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
   }
 
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -3050,10 +2998,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
     void *IP = 0;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return SDValue(E, 0);
-    }
+
     N = NodeAllocator.Allocate<TernarySDNode>();
     new (N) TernarySDNode(Opcode, DL, VTs, N1, N2, N3);
     CSEMap.InsertNode(N, IP);
@@ -3063,7 +3010,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
   }
 
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -3503,7 +3449,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst,
                     /*isReturnValueUsed=*/false,
                     getExternalSymbol(TLI.getLibcallName(RTLIB::MEMCPY),
                                       TLI.getPointerTy()),
-                    Args, *this, dl);
+                    Args, *this, dl, GetOrdering(Chain.getNode()));
   return CallResult.second;
 }
 
@@ -3552,7 +3498,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst,
                     /*isReturnValueUsed=*/false,
                     getExternalSymbol(TLI.getLibcallName(RTLIB::MEMMOVE),
                                       TLI.getPointerTy()),
-                    Args, *this, dl);
+                    Args, *this, dl, GetOrdering(Chain.getNode()));
   return CallResult.second;
 }
 
@@ -3611,7 +3557,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
                     /*isReturnValueUsed=*/false,
                     getExternalSymbol(TLI.getLibcallName(RTLIB::MEMSET),
                                       TLI.getPointerTy()),
-                    Args, *this, dl);
+                    Args, *this, dl, GetOrdering(Chain.getNode()));
   return CallResult.second;
 }
 
@@ -3659,14 +3605,12 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
   void* IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
     cast<AtomicSDNode>(E)->refineAlignment(MMO);
-    if (Ordering) Ordering->add(E);
     return SDValue(E, 0);
   }
   SDNode* N = NodeAllocator.Allocate<AtomicSDNode>();
   new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -3724,14 +3668,12 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
   void* IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
     cast<AtomicSDNode>(E)->refineAlignment(MMO);
-    if (Ordering) Ordering->add(E);
     return SDValue(E, 0);
   }
   SDNode* N = NodeAllocator.Allocate<AtomicSDNode>();
   new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Val, MMO);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -3804,7 +3746,6 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
     void *IP = 0;
     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
-      if (Ordering) Ordering->add(E);
       return SDValue(E, 0);
     }
 
@@ -3816,7 +3757,6 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
     new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
   }
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -3881,14 +3821,12 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl,
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
     cast<LoadSDNode>(E)->refineAlignment(MMO);
-    if (Ordering) Ordering->add(E);
     return SDValue(E, 0);
   }
   SDNode *N = NodeAllocator.Allocate<LoadSDNode>();
   new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, MMO);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -3959,14 +3897,12 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
     cast<StoreSDNode>(E)->refineAlignment(MMO);
-    if (Ordering) Ordering->add(E);
     return SDValue(E, 0);
   }
   SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
   new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, VT, MMO);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -4021,14 +3957,12 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val,
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
     cast<StoreSDNode>(E)->refineAlignment(MMO);
-    if (Ordering) Ordering->add(E);
     return SDValue(E, 0);
   }
   SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
   new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true, SVT, MMO);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -4045,17 +3979,15 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base,
   ID.AddInteger(ST->getMemoryVT().getRawBits());
   ID.AddInteger(ST->getRawSubclassData());
   void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-    if (Ordering) Ordering->add(E);
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
-  }
+
   SDNode *N = NodeAllocator.Allocate<StoreSDNode>();
   new (N) StoreSDNode(Ops, dl, VTs, AM,
                       ST->isTruncatingStore(), ST->getMemoryVT(),
                       ST->getMemOperand());
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
   return SDValue(N, 0);
 }
 
@@ -4121,10 +4053,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
     AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps);
     void *IP = 0;
 
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return SDValue(E, 0);
-    }
 
     N = NodeAllocator.Allocate<SDNode>();
     new (N) SDNode(Opcode, DL, VTs, Ops, NumOps);
@@ -4135,7 +4065,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
   }
 
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -4191,10 +4120,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
     void *IP = 0;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return SDValue(E, 0);
-    }
+
     if (NumOps == 1) {
       N = NodeAllocator.Allocate<UnarySDNode>();
       new (N) UnarySDNode(Opcode, DL, VTList, Ops[0]);
@@ -4225,7 +4153,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList,
     }
   }
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -4325,6 +4252,7 @@ SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) {
     case 1: return getVTList(VTs[0]);
     case 2: return getVTList(VTs[0], VTs[1]);
     case 3: return getVTList(VTs[0], VTs[1], VTs[2]);
+    case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]);
     default: break;
   }
 
@@ -4688,10 +4616,8 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
   if (VTs.VTs[VTs.NumVTs-1] != MVT::Flag) {
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opc, VTs, Ops, NumOps);
-    if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(ON);
+    if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
       return ON;
-    }
   }
 
   if (!RemoveNodeFromCSEMaps(N))
@@ -4755,7 +4681,6 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
 
   if (IP)
     CSEMap.InsertNode(N, IP);   // Memoize the new node.
-  if (Ordering) Ordering->add(N);
   return N;
 }
 
@@ -4894,10 +4819,8 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps);
     IP = 0;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return cast<MachineSDNode>(E);
-    }
   }
 
   // Allocate a new MachineSDNode.
@@ -4919,7 +4842,6 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs,
     CSEMap.InsertNode(N, IP);
 
   AllNodes.push_back(N);
-  if (Ordering) Ordering->add(N);
 #ifndef NDEBUG
   VerifyNode(N);
 #endif
@@ -4956,10 +4878,8 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
     void *IP = 0;
-    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
-      if (Ordering) Ordering->add(E);
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
       return E;
-    }
   }
   return NULL;
 }
@@ -5291,6 +5211,18 @@ unsigned SelectionDAG::AssignTopologicalOrder() {
   return DAGSize;
 }
 
+/// AssignOrdering - Assign an order to the SDNode.
+void SelectionDAG::AssignOrdering(SDNode *SD, unsigned Order) {
+  assert(SD && "Trying to assign an order to a null node!");
+  if (Ordering)
+    Ordering->add(SD, Order);
+}
+
+/// GetOrdering - Get the order for the SDNode.
+unsigned SelectionDAG::GetOrdering(const SDNode *SD) const {
+  assert(SD && "Trying to get the order of a null node!");
+  return Ordering ? Ordering->getOrder(SD) : 0;
+}
 
 
 //===----------------------------------------------------------------------===//
@@ -5931,6 +5863,10 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
     if (unsigned int TF = BA->getTargetFlags())
       OS << " [TF=" << TF << ']';
   }
+
+  if (G)
+    if (unsigned Order = G->GetOrdering(this))
+      OS << " [ORD=" << Order << ']';
 }
 
 void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const {
@@ -6126,9 +6062,6 @@ void SelectionDAG::dump() const {
   errs() << "\n\n";
 }
 
-void SelectionDAG::NodeOrdering::dump() const {
-}
-
 void SDNode::printr(raw_ostream &OS, const SelectionDAG *G) const {
   print_types(OS, G);
   print_details(OS, G);
@@ -6139,25 +6072,31 @@ static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent,
                        const SelectionDAG *G, VisitedSDNodeSet &once) {
   if (!once.insert(N))          // If we've been here before, return now.
     return;
+
   // Dump the current SDNode, but don't end the line yet.
   OS << std::string(indent, ' ');
   N->printr(OS, G);
+
   // Having printed this SDNode, walk the children:
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
     const SDNode *child = N->getOperand(i).getNode();
+
     if (i) OS << ",";
     OS << " ";
+
     if (child->getNumOperands() == 0) {
       // This child has no grandchildren; print it inline right here.
       child->printr(OS, G);
       once.insert(child);
-    } else {          // Just the address.  FIXME: also print the child's opcode
+    } else {         // Just the address. FIXME: also print the child's opcode.
       OS << (void*)child;
       if (unsigned RN = N->getOperand(i).getResNo())
         OS << ":" << RN;
     }
   }
+
   OS << "\n";
+
   // Dump children that have grandchildren on their own line(s).
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
     const SDNode *child = N->getOperand(i).getNode();
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7568384173d3..74d624f3647d 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Constants.h"
 #include "llvm/CallingConv.h"
 #include "llvm/DerivedTypes.h"
@@ -26,7 +27,6 @@
 #include "llvm/Instructions.h"
 #include "llvm/Intrinsics.h"
 #include "llvm/IntrinsicInst.h"
-#include "llvm/LLVMContext.h"
 #include "llvm/Module.h"
 #include "llvm/CodeGen/FastISel.h"
 #include "llvm/CodeGen/GCStrategy.h"
@@ -70,13 +70,13 @@ LimitFPPrecision("limit-float-precision",
 
 namespace {
   /// RegsForValue - This struct represents the registers (physical or virtual)
-  /// that a particular set of values is assigned, and the type information about
-  /// the value. The most common situation is to represent one value at a time,
-  /// but struct or array values are handled element-wise as multiple values.
-  /// The splitting of aggregates is performed recursively, so that we never
-  /// have aggregate-typed registers. The values at this point do not necessarily
-  /// have legal types, so each value may require one or more registers of some
-  /// legal type.
+  /// that a particular set of values is assigned, and the type information
+  /// about the value. The most common situation is to represent one value at a
+  /// time, but struct or array values are handled element-wise as multiple
+  /// values.  The splitting of aggregates is performed recursively, so that we
+  /// never have aggregate-typed registers. The values at this point do not
+  /// necessarily have legal types, so each value may require one or more
+  /// registers of some legal type.
   ///
   struct RegsForValue {
     /// TLI - The TargetLowering object.
@@ -144,22 +144,23 @@ namespace {
     /// this value and returns the result as a ValueVTs value.  This uses
     /// Chain/Flag as the input and updates them for the output Chain/Flag.
     /// If the Flag pointer is NULL, no flag is used.
-    SDValue getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl,
-                              SDValue &Chain, SDValue *Flag) const;
+    SDValue getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl, unsigned Order,
+                            SDValue &Chain, SDValue *Flag) const;
 
     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
     /// specified value into the registers specified by this object.  This uses
     /// Chain/Flag as the input and updates them for the output Chain/Flag.
     /// If the Flag pointer is NULL, no flag is used.
     void getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
-                       SDValue &Chain, SDValue *Flag) const;
+                       unsigned Order, SDValue &Chain, SDValue *Flag) const;
 
     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
     /// operand list.  This adds the code marker, matching input operand index
     /// (if applicable), and includes the number of values added into it.
     void AddInlineAsmOperands(unsigned Code,
                               bool HasMatching, unsigned MatchingIdx,
-                              SelectionDAG &DAG, std::vector<SDValue> &Ops) const;
+                              SelectionDAG &DAG, unsigned Order,
+                              std::vector<SDValue> &Ops) const;
   };
 }
 
@@ -168,13 +169,14 @@ namespace {
 /// larger then ValueVT then AssertOp can be used to specify whether the extra
 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
 /// (ISD::AssertSext).
-static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
+static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl, unsigned Order,
                                 const SDValue *Parts,
                                 unsigned NumParts, EVT PartVT, EVT ValueVT,
                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
   assert(NumParts > 0 && "No parts to assemble!");
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   SDValue Val = Parts[0];
+  if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
 
   if (NumParts > 1) {
     // Assemble the value from multiple parts.
@@ -193,23 +195,32 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
       EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
 
       if (RoundParts > 2) {
-        Lo = getCopyFromParts(DAG, dl, Parts, RoundParts/2, PartVT, HalfVT);
-        Hi = getCopyFromParts(DAG, dl, Parts+RoundParts/2, RoundParts/2,
+        Lo = getCopyFromParts(DAG, dl, Order, Parts, RoundParts / 2,
                               PartVT, HalfVT);
+        Hi = getCopyFromParts(DAG, dl, Order, Parts + RoundParts / 2,
+                              RoundParts / 2, PartVT, HalfVT);
       } else {
         Lo = DAG.getNode(ISD::BIT_CONVERT, dl, HalfVT, Parts[0]);
         Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HalfVT, Parts[1]);
       }
+
       if (TLI.isBigEndian())
         std::swap(Lo, Hi);
+
       Val = DAG.getNode(ISD::BUILD_PAIR, dl, RoundVT, Lo, Hi);
 
+      if (DisableScheduling) {
+        DAG.AssignOrdering(Lo.getNode(), Order);
+        DAG.AssignOrdering(Hi.getNode(), Order);
+        DAG.AssignOrdering(Val.getNode(), Order);
+      }
+
       if (RoundParts < NumParts) {
         // Assemble the trailing non-power-of-2 part.
         unsigned OddParts = NumParts - RoundParts;
         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
-        Hi = getCopyFromParts(DAG, dl,
-                              Parts+RoundParts, OddParts, PartVT, OddVT);
+        Hi = getCopyFromParts(DAG, dl, Order,
+                              Parts + RoundParts, OddParts, PartVT, OddVT);
 
         // Combine the round and odd parts.
         Lo = Val;
@@ -217,22 +228,28 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
           std::swap(Lo, Hi);
         EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
         Hi = DAG.getNode(ISD::ANY_EXTEND, dl, TotalVT, Hi);
+        if (DisableScheduling) DAG.AssignOrdering(Hi.getNode(), Order);
         Hi = DAG.getNode(ISD::SHL, dl, TotalVT, Hi,
                          DAG.getConstant(Lo.getValueType().getSizeInBits(),
                                          TLI.getPointerTy()));
+        if (DisableScheduling) DAG.AssignOrdering(Hi.getNode(), Order);
         Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, TotalVT, Lo);
+        if (DisableScheduling) DAG.AssignOrdering(Lo.getNode(), Order);
         Val = DAG.getNode(ISD::OR, dl, TotalVT, Lo, Hi);
+        if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
       }
     } else if (ValueVT.isVector()) {
       // Handle a multi-element vector.
       EVT IntermediateVT, RegisterVT;
       unsigned NumIntermediates;
       unsigned NumRegs =
-        TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, 
+        TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
                                    NumIntermediates, RegisterVT);
-      assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
+      assert(NumRegs == NumParts
+             && "Part count doesn't match vector breakdown!");
       NumParts = NumRegs; // Silence a compiler warning.
-      assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
+      assert(RegisterVT == PartVT
+             && "Part type doesn't match vector breakdown!");
       assert(RegisterVT == Parts[0].getValueType() &&
              "Part type doesn't match part!");
 
@@ -242,24 +259,25 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
         // If the register was not expanded, truncate or copy the value,
         // as appropriate.
         for (unsigned i = 0; i != NumParts; ++i)
-          Ops[i] = getCopyFromParts(DAG, dl, &Parts[i], 1,
+          Ops[i] = getCopyFromParts(DAG, dl, Order, &Parts[i], 1,
                                     PartVT, IntermediateVT);
       } else if (NumParts > 0) {
-        // If the intermediate type was expanded, build the intermediate operands
-        // from the parts.
+        // If the intermediate type was expanded, build the intermediate
+        // operands from the parts.
         assert(NumParts % NumIntermediates == 0 &&
                "Must expand into a divisible number of parts!");
         unsigned Factor = NumParts / NumIntermediates;
         for (unsigned i = 0; i != NumIntermediates; ++i)
-          Ops[i] = getCopyFromParts(DAG, dl, &Parts[i * Factor], Factor,
+          Ops[i] = getCopyFromParts(DAG, dl, Order, &Parts[i * Factor], Factor,
                                     PartVT, IntermediateVT);
       }
 
-      // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate
-      // operands.
+      // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
+      // intermediate operands.
       Val = DAG.getNode(IntermediateVT.isVector() ?
                         ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, dl,
                         ValueVT, &Ops[0], NumIntermediates);
+      if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
     } else if (PartVT.isFloatingPoint()) {
       // FP split into multiple FP parts (for ppcf128)
       assert(ValueVT == EVT(MVT::ppcf128) && PartVT == EVT(MVT::f64) &&
@@ -270,12 +288,18 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
       if (TLI.isBigEndian())
         std::swap(Lo, Hi);
       Val = DAG.getNode(ISD::BUILD_PAIR, dl, ValueVT, Lo, Hi);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(Hi.getNode(), Order);
+        DAG.AssignOrdering(Lo.getNode(), Order);
+        DAG.AssignOrdering(Val.getNode(), Order);
+      }
     } else {
       // FP split into integer parts (soft fp)
       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
              !PartVT.isVector() && "Unexpected split");
       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
-      Val = getCopyFromParts(DAG, dl, Parts, NumParts, PartVT, IntVT);
+      Val = getCopyFromParts(DAG, dl, Order, Parts, NumParts, PartVT, IntVT);
     }
   }
 
@@ -287,14 +311,20 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
 
   if (PartVT.isVector()) {
     assert(ValueVT.isVector() && "Unknown vector conversion!");
-    return DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val);
+    SDValue Res = DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), Order);
+    return Res;
   }
 
   if (ValueVT.isVector()) {
     assert(ValueVT.getVectorElementType() == PartVT &&
            ValueVT.getVectorNumElements() == 1 &&
            "Only trivial scalar-to-vector conversions should get here!");
-    return DAG.getNode(ISD::BUILD_VECTOR, dl, ValueVT, Val);
+    SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, dl, ValueVT, Val);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), Order);
+    return Res;
   }
 
   if (PartVT.isInteger() &&
@@ -306,22 +336,36 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
       if (AssertOp != ISD::DELETED_NODE)
         Val = DAG.getNode(AssertOp, dl, PartVT, Val,
                           DAG.getValueType(ValueVT));
-      return DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val);
+      if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+      Val = DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val);
+      if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+      return Val;
     } else {
-      return DAG.getNode(ISD::ANY_EXTEND, dl, ValueVT, Val);
+      Val = DAG.getNode(ISD::ANY_EXTEND, dl, ValueVT, Val);
+      if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+      return Val;
     }
   }
 
   if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
-    if (ValueVT.bitsLT(Val.getValueType()))
+    if (ValueVT.bitsLT(Val.getValueType())) {
       // FP_ROUND's are always exact here.
-      return DAG.getNode(ISD::FP_ROUND, dl, ValueVT, Val,
-                         DAG.getIntPtrConstant(1));
-    return DAG.getNode(ISD::FP_EXTEND, dl, ValueVT, Val);
+      Val = DAG.getNode(ISD::FP_ROUND, dl, ValueVT, Val,
+                        DAG.getIntPtrConstant(1));
+      if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+      return Val;
+    }
+
+    Val = DAG.getNode(ISD::FP_EXTEND, dl, ValueVT, Val);
+    if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+    return Val;
   }
 
-  if (PartVT.getSizeInBits() == ValueVT.getSizeInBits())
-    return DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val);
+  if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
+    Val = DAG.getNode(ISD::BIT_CONVERT, dl, ValueVT, Val);
+    if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+    return Val;
+  }
 
   llvm_unreachable("Unknown mismatch!");
   return SDValue();
@@ -330,8 +374,9 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc dl,
 /// getCopyToParts - Create a series of nodes that contain the specified value
 /// split into legal parts.  If the parts contain more bits than Val, then, for
 /// integers, ExtendKind can be used to specify how to generate the extra bits.
-static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
-                           SDValue *Parts, unsigned NumParts, EVT PartVT,
+static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, unsigned Order,
+                           SDValue Val, SDValue *Parts, unsigned NumParts,
+                           EVT PartVT,
                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT PtrVT = TLI.getPointerTy();
@@ -375,6 +420,8 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
       }
     }
 
+    if (DisableScheduling) DAG.AssignOrdering(Val.getNode(), Order);
+
     // The value may have changed - recompute ValueVT.
     ValueVT = Val.getValueType();
     assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
@@ -397,20 +444,33 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
       SDValue OddVal = DAG.getNode(ISD::SRL, dl, ValueVT, Val,
                                    DAG.getConstant(RoundBits,
                                                    TLI.getPointerTy()));
-      getCopyToParts(DAG, dl, OddVal, Parts + RoundParts, OddParts, PartVT);
+      getCopyToParts(DAG, dl, Order, OddVal, Parts + RoundParts,
+                     OddParts, PartVT);
+
       if (TLI.isBigEndian())
         // The odd parts were reversed by getCopyToParts - unreverse them.
         std::reverse(Parts + RoundParts, Parts + NumParts);
+
       NumParts = RoundParts;
       ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
       Val = DAG.getNode(ISD::TRUNCATE, dl, ValueVT, Val);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(OddVal.getNode(), Order);
+        DAG.AssignOrdering(Val.getNode(), Order);
+      }
     }
 
     // The number of parts is a power of 2.  Repeatedly bisect the value using
     // EXTRACT_ELEMENT.
     Parts[0] = DAG.getNode(ISD::BIT_CONVERT, dl,
-                           EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()),
+                           EVT::getIntegerVT(*DAG.getContext(),
+                                             ValueVT.getSizeInBits()),
                            Val);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Parts[0].getNode(), Order);
+
     for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
       for (unsigned i = 0; i < NumParts; i += StepSize) {
         unsigned ThisBits = StepSize * PartBits / 2;
@@ -425,11 +485,20 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
                             ThisVT, Part0,
                             DAG.getConstant(0, PtrVT));
 
+        if (DisableScheduling) {
+          DAG.AssignOrdering(Part0.getNode(), Order);
+          DAG.AssignOrdering(Part1.getNode(), Order);
+        }
+
         if (ThisBits == PartBits && ThisVT != PartVT) {
           Part0 = DAG.getNode(ISD::BIT_CONVERT, dl,
                                                 PartVT, Part0);
           Part1 = DAG.getNode(ISD::BIT_CONVERT, dl,
                                                 PartVT, Part1);
+          if (DisableScheduling) {
+            DAG.AssignOrdering(Part0.getNode(), Order);
+            DAG.AssignOrdering(Part1.getNode(), Order);
+          }
         }
       }
     }
@@ -443,7 +512,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
   // Vector ValueVT.
   if (NumParts == 1) {
     if (PartVT != ValueVT) {
-      if (PartVT.isVector()) {
+      if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
         Val = DAG.getNode(ISD::BIT_CONVERT, dl, PartVT, Val);
       } else {
         assert(ValueVT.getVectorElementType() == PartVT &&
@@ -455,6 +524,9 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
       }
     }
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(Val.getNode(), Order);
+
     Parts[0] = Val;
     return;
   }
@@ -472,7 +544,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
 
   // Split the vector into intermediate operands.
   SmallVector<SDValue, 8> Ops(NumIntermediates);
-  for (unsigned i = 0; i != NumIntermediates; ++i)
+  for (unsigned i = 0; i != NumIntermediates; ++i) {
     if (IntermediateVT.isVector())
       Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl,
                            IntermediateVT, Val,
@@ -483,12 +555,16 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
                            IntermediateVT, Val,
                            DAG.getConstant(i, PtrVT));
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(Ops[i].getNode(), Order);
+  }
+
   // Split the intermediate operands into legal parts.
   if (NumParts == NumIntermediates) {
     // If the register was not expanded, promote or copy the value,
     // as appropriate.
     for (unsigned i = 0; i != NumParts; ++i)
-      getCopyToParts(DAG, dl, Ops[i], &Parts[i], 1, PartVT);
+      getCopyToParts(DAG, dl, Order, Ops[i], &Parts[i], 1, PartVT);
   } else if (NumParts > 0) {
     // If the intermediate type was expanded, split each the value into
     // legal parts.
@@ -496,7 +572,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc dl, SDValue Val,
            "Must expand into a divisible number of parts!");
     unsigned Factor = NumParts / NumIntermediates;
     for (unsigned i = 0; i != NumIntermediates; ++i)
-      getCopyToParts(DAG, dl, Ops[i], &Parts[i * Factor], Factor, PartVT);
+      getCopyToParts(DAG, dl, Order, Ops[i], &Parts[i*Factor], Factor, PartVT);
   }
 }
 
@@ -583,8 +659,8 @@ void SelectionDAGBuilder::visit(Instruction &I) {
 }
 
 void SelectionDAGBuilder::visit(unsigned Opcode, User &I) {
-  // Tell the DAG that we're processing a new instruction.
-  DAG.NewInst();
+  // We're processing a new instruction.
+  ++SDNodeOrder;
 
   // Note: this doesn't use InstVisitor, because it has to work with
   // ConstantExpr's in addition to instructions.
@@ -592,7 +668,7 @@ void SelectionDAGBuilder::visit(unsigned Opcode, User &I) {
   default: llvm_unreachable("Unknown instruction type encountered!");
     // Build the switch statement using the Instruction.def file.
 #define HANDLE_INST(NUM, OPCODE, CLASS) \
-  case Instruction::OPCODE:return visit##OPCODE((CLASS&)I);
+  case Instruction::OPCODE: return visit##OPCODE((CLASS&)I);
 #include "llvm/Instruction.def"
   }
 }
@@ -638,8 +714,12 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
         for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
           Constants.push_back(SDValue(Val, i));
       }
-      return DAG.getMergeValues(&Constants[0], Constants.size(),
-                                getCurDebugLoc());
+
+      SDValue Res = DAG.getMergeValues(&Constants[0], Constants.size(),
+                                       getCurDebugLoc());
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+      return Res;
     }
 
     if (isa<StructType>(C->getType()) || isa<ArrayType>(C->getType())) {
@@ -661,7 +741,12 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
         else
           Constants[i] = DAG.getConstant(0, EltVT);
       }
-      return DAG.getMergeValues(&Constants[0], NumElts, getCurDebugLoc());
+
+      SDValue Res = DAG.getMergeValues(&Constants[0], NumElts,
+                                       getCurDebugLoc());
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+      return Res;
     }
 
     if (BlockAddress *BA = dyn_cast<BlockAddress>(C))
@@ -689,8 +774,12 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
     }
 
     // Create a BUILD_VECTOR node.
-    return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(),
-                                    VT, &Ops[0], Ops.size());
+    SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(),
+                              VT, &Ops[0], Ops.size());
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
+    return NodeMap[V] = Res;
   }
 
   // If this is a static alloca, generate it as the frameindex instead of
@@ -707,7 +796,8 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
 
   RegsForValue RFV(*DAG.getContext(), TLI, InReg, V->getType());
   SDValue Chain = DAG.getEntryNode();
-  return RFV.getCopyFromRegs(DAG, getCurDebugLoc(), Chain, NULL);
+  return RFV.getCopyFromRegs(DAG, getCurDebugLoc(),
+                             SDNodeOrder, Chain, NULL);
 }
 
 /// Get the EVTs and ArgFlags collections that represent the return type
@@ -766,7 +856,7 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) {
   SDValue Chain = getControlRoot();
   SmallVector<ISD::OutputArg, 8> Outs;
   FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo();
-  
+
   if (!FLI.CanLowerReturn) {
     unsigned DemoteReg = FLI.DemoteRegister;
     const Function *F = I.getParent()->getParent();
@@ -775,12 +865,12 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) {
     // Leave Outs empty so that LowerReturn won't try to load return
     // registers the usual way.
     SmallVector<EVT, 1> PtrValueVTs;
-    ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()), 
+    ComputeValueVTs(TLI, PointerType::getUnqual(F->getReturnType()),
                     PtrValueVTs);
 
     SDValue RetPtr = DAG.getRegister(DemoteReg, PtrValueVTs[0]);
     SDValue RetOp = getValue(I.getOperand(0));
-  
+
     SmallVector<EVT, 4> ValueVTs;
     SmallVector<uint64_t, 4> Offsets;
     ComputeValueVTs(TLI, I.getOperand(0)->getType(), ValueVTs, &Offsets);
@@ -788,22 +878,32 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) {
 
     SmallVector<SDValue, 4> Chains(NumValues);
     EVT PtrVT = PtrValueVTs[0];
-    for (unsigned i = 0; i != NumValues; ++i)
-      Chains[i] = DAG.getStore(Chain, getCurDebugLoc(),
-                  SDValue(RetOp.getNode(), RetOp.getResNo() + i),
-                  DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, RetPtr,
-                  DAG.getConstant(Offsets[i], PtrVT)),
-                  NULL, Offsets[i], false, 0);
+    for (unsigned i = 0; i != NumValues; ++i) {
+      SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, RetPtr,
+                                DAG.getConstant(Offsets[i], PtrVT));
+      Chains[i] =
+        DAG.getStore(Chain, getCurDebugLoc(),
+                     SDValue(RetOp.getNode(), RetOp.getResNo() + i),
+                     Add, NULL, Offsets[i], false, 0);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(Add.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Chains[i].getNode(), SDNodeOrder);
+      }
+    }
+
     Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),
                         MVT::Other, &Chains[0], NumValues);
-  }
-  else {
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Chain.getNode(), SDNodeOrder);
+  } else {
     for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) {
       SmallVector<EVT, 4> ValueVTs;
       ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs);
       unsigned NumValues = ValueVTs.size();
       if (NumValues == 0) continue;
-  
+
       SDValue RetOp = getValue(I.getOperand(i));
       for (unsigned j = 0, f = NumValues; j != f; ++j) {
         EVT VT = ValueVTs[j];
@@ -816,8 +916,8 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) {
         else if (F->paramHasAttr(0, Attribute::ZExt))
           ExtendKind = ISD::ZERO_EXTEND;
 
-        // FIXME: C calling convention requires the return type to be promoted to
-        // at least 32-bit. But this is not necessary for non-C calling
+        // FIXME: C calling convention requires the return type to be promoted
+        // to at least 32-bit. But this is not necessary for non-C calling
         // conventions. The frontend should mark functions whose return values
         // require promoting with signext or zeroext attributes.
         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) {
@@ -829,7 +929,7 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) {
         unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), VT);
         EVT PartVT = TLI.getRegisterType(*DAG.getContext(), VT);
         SmallVector<SDValue, 4> Parts(NumParts);
-        getCopyToParts(DAG, getCurDebugLoc(),
+        getCopyToParts(DAG, getCurDebugLoc(), SDNodeOrder,
                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
                        &Parts[0], NumParts, PartVT, ExtendKind);
 
@@ -862,6 +962,9 @@ void SelectionDAGBuilder::visitRet(ReturnInst &I) {
 
   // Update the DAG with the new chain value resulting from return lowering.
   DAG.setRoot(Chain);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Chain.getNode(), SDNodeOrder);
 }
 
 /// CopyToExportRegsIfNeeded - If the given value has virtual registers
@@ -1110,10 +1213,16 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) {
     CurMBB->addSuccessor(Succ0MBB);
 
     // If this is not a fall-through branch, emit the branch.
-    if (Succ0MBB != NextBlock)
-      DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(),
+    if (Succ0MBB != NextBlock) {
+      SDValue V = DAG.getNode(ISD::BR, getCurDebugLoc(),
                               MVT::Other, getControlRoot(),
-                              DAG.getBasicBlock(Succ0MBB)));
+                              DAG.getBasicBlock(Succ0MBB));
+      DAG.setRoot(V);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(V.getNode(), SDNodeOrder);
+    }
+
     return;
   }
 
@@ -1172,6 +1281,7 @@ void SelectionDAGBuilder::visitBr(BranchInst &I) {
   // Create a CaseBlock record representing this branch.
   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
                NULL, Succ0MBB, Succ1MBB, CurMBB);
+
   // Use visitSwitchCase to actually insert the fast branch sequence for this
   // cond branch.
   visitSwitchCase(CB);
@@ -1217,6 +1327,9 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) {
     }
   }
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(Cond.getNode(), SDNodeOrder);
+
   // Update successor info
   CurMBB->addSuccessor(CB.TrueBB);
   CurMBB->addSuccessor(CB.FalseBB);
@@ -1234,26 +1347,36 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB) {
     std::swap(CB.TrueBB, CB.FalseBB);
     SDValue True = DAG.getConstant(1, Cond.getValueType());
     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Cond.getNode(), SDNodeOrder);
   }
+
   SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
                                MVT::Other, getControlRoot(), Cond,
                                DAG.getBasicBlock(CB.TrueBB));
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder);
+
   // If the branch was constant folded, fix up the CFG.
   if (BrCond.getOpcode() == ISD::BR) {
     CurMBB->removeSuccessor(CB.FalseBB);
-    DAG.setRoot(BrCond);
   } else {
     // Otherwise, go ahead and insert the false branch.
     if (BrCond == getControlRoot())
       CurMBB->removeSuccessor(CB.TrueBB);
 
-    if (CB.FalseBB == NextBlock)
-      DAG.setRoot(BrCond);
-    else
-      DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
-                              DAG.getBasicBlock(CB.FalseBB)));
+    if (CB.FalseBB != NextBlock) {
+      BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
+                           DAG.getBasicBlock(CB.FalseBB));
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder);
+    }
   }
+
+  DAG.setRoot(BrCond);
 }
 
 /// visitJumpTable - Emit JumpTable node in the current MBB
@@ -1264,9 +1387,16 @@ void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
   SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurDebugLoc(),
                                      JT.Reg, PTy);
   SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
-  DAG.setRoot(DAG.getNode(ISD::BR_JT, getCurDebugLoc(),
-                          MVT::Other, Index.getValue(1),
-                          Table, Index));
+  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurDebugLoc(),
+                                    MVT::Other, Index.getValue(1),
+                                    Table, Index);
+  DAG.setRoot(BrJumpTable);
+
+  if (DisableScheduling) {
+    DAG.AssignOrdering(Index.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(Table.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(BrJumpTable.getNode(), SDNodeOrder);
+  }
 }
 
 /// visitJumpTableHeader - This function emits necessary code to produce index
@@ -1278,7 +1408,7 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
   // difference between smallest and largest cases.
   SDValue SwitchOp = getValue(JTH.SValue);
   EVT VT = SwitchOp.getValueType();
-  SDValue SUB = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp,
+  SDValue Sub = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp,
                             DAG.getConstant(JTH.First, VT));
 
   // The SDNode we just created, which holds the value being switched on minus
@@ -1286,7 +1416,7 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
   // can be used as an index into the jump table in a subsequent basic block.
   // This value may be smaller or larger than the target's pointer type, and
   // therefore require extension or truncating.
-  SwitchOp = DAG.getZExtOrTrunc(SUB, getCurDebugLoc(), TLI.getPointerTy());
+  SwitchOp = DAG.getZExtOrTrunc(Sub, getCurDebugLoc(), TLI.getPointerTy());
 
   unsigned JumpTableReg = FuncInfo.MakeReg(TLI.getPointerTy());
   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurDebugLoc(),
@@ -1297,14 +1427,22 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
   // for the switch statement if the value being switched on exceeds the largest
   // case in the switch.
   SDValue CMP = DAG.getSetCC(getCurDebugLoc(),
-                             TLI.getSetCCResultType(SUB.getValueType()), SUB,
+                             TLI.getSetCCResultType(Sub.getValueType()), Sub,
                              DAG.getConstant(JTH.Last-JTH.First,VT),
                              ISD::SETUGT);
 
+  if (DisableScheduling) {
+    DAG.AssignOrdering(Sub.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(SwitchOp.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(CopyTo.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(CMP.getNode(), SDNodeOrder);
+  }
+
   // Set NextBlock to be the MBB immediately after the current one, if any.
   // This is used to avoid emitting unnecessary branches to the next block.
   MachineBasicBlock *NextBlock = 0;
   MachineFunction::iterator BBI = CurMBB;
+
   if (++BBI != FuncInfo.MF->end())
     NextBlock = BBI;
 
@@ -1312,11 +1450,18 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
                                MVT::Other, CopyTo, CMP,
                                DAG.getBasicBlock(JT.Default));
 
-  if (JT.MBB == NextBlock)
-    DAG.setRoot(BrCond);
-  else
-    DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrCond,
-                            DAG.getBasicBlock(JT.MBB)));
+  if (DisableScheduling)
+    DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder);
+
+  if (JT.MBB != NextBlock) {
+    BrCond = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrCond,
+                         DAG.getBasicBlock(JT.MBB));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(BrCond.getNode(), SDNodeOrder);
+  }
+
+  DAG.setRoot(BrCond);
 }
 
 /// visitBitTestHeader - This function emits necessary code to produce value
@@ -1325,21 +1470,29 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) {
   // Subtract the minimum value
   SDValue SwitchOp = getValue(B.SValue);
   EVT VT = SwitchOp.getValueType();
-  SDValue SUB = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp,
+  SDValue Sub = DAG.getNode(ISD::SUB, getCurDebugLoc(), VT, SwitchOp,
                             DAG.getConstant(B.First, VT));
 
   // Check range
   SDValue RangeCmp = DAG.getSetCC(getCurDebugLoc(),
-                                  TLI.getSetCCResultType(SUB.getValueType()),
-                                  SUB, DAG.getConstant(B.Range, VT),
+                                  TLI.getSetCCResultType(Sub.getValueType()),
+                                  Sub, DAG.getConstant(B.Range, VT),
                                   ISD::SETUGT);
 
-  SDValue ShiftOp = DAG.getZExtOrTrunc(SUB, getCurDebugLoc(), TLI.getPointerTy());
+  SDValue ShiftOp = DAG.getZExtOrTrunc(Sub, getCurDebugLoc(),
+                                       TLI.getPointerTy());
 
   B.Reg = FuncInfo.MakeReg(TLI.getPointerTy());
   SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), getCurDebugLoc(),
                                     B.Reg, ShiftOp);
 
+  if (DisableScheduling) {
+    DAG.AssignOrdering(Sub.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(RangeCmp.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(ShiftOp.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(CopyTo.getNode(), SDNodeOrder);
+  }
+
   // Set NextBlock to be the MBB immediately after the current one, if any.
   // This is used to avoid emitting unnecessary branches to the next block.
   MachineBasicBlock *NextBlock = 0;
@@ -1356,11 +1509,18 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B) {
                                 MVT::Other, CopyTo, RangeCmp,
                                 DAG.getBasicBlock(B.Default));
 
-  if (MBB == NextBlock)
-    DAG.setRoot(BrRange);
-  else
-    DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, CopyTo,
-                            DAG.getBasicBlock(MBB)));
+  if (DisableScheduling)
+    DAG.AssignOrdering(BrRange.getNode(), SDNodeOrder);
+
+  if (MBB != NextBlock) {
+    BrRange = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, CopyTo,
+                          DAG.getBasicBlock(MBB));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(BrRange.getNode(), SDNodeOrder);
+  }
+
+  DAG.setRoot(BrRange);
 }
 
 /// visitBitTestCase - this function produces one "bit test"
@@ -1384,6 +1544,13 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB,
                                 AndOp, DAG.getConstant(0, TLI.getPointerTy()),
                                 ISD::SETNE);
 
+  if (DisableScheduling) {
+    DAG.AssignOrdering(ShiftOp.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(SwitchVal.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(AndOp.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(AndCmp.getNode(), SDNodeOrder);
+  }
+
   CurMBB->addSuccessor(B.TargetBB);
   CurMBB->addSuccessor(NextMBB);
 
@@ -1391,6 +1558,9 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB,
                               MVT::Other, getControlRoot(),
                               AndCmp, DAG.getBasicBlock(B.TargetBB));
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(BrAnd.getNode(), SDNodeOrder);
+
   // Set NextBlock to be the MBB immediately after the current one, if any.
   // This is used to avoid emitting unnecessary branches to the next block.
   MachineBasicBlock *NextBlock = 0;
@@ -1398,11 +1568,15 @@ void SelectionDAGBuilder::visitBitTestCase(MachineBasicBlock* NextMBB,
   if (++BBI != FuncInfo.MF->end())
     NextBlock = BBI;
 
-  if (NextMBB == NextBlock)
-    DAG.setRoot(BrAnd);
-  else
-    DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrAnd,
-                            DAG.getBasicBlock(NextMBB)));
+  if (NextMBB != NextBlock) {
+    BrAnd = DAG.getNode(ISD::BR, getCurDebugLoc(), MVT::Other, BrAnd,
+                        DAG.getBasicBlock(NextMBB));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(BrAnd.getNode(), SDNodeOrder);
+  }
+
+  DAG.setRoot(BrAnd);
 }
 
 void SelectionDAGBuilder::visitInvoke(InvokeInst &I) {
@@ -1425,9 +1599,13 @@ void SelectionDAGBuilder::visitInvoke(InvokeInst &I) {
   CurMBB->addSuccessor(LandingPad);
 
   // Drop into normal successor.
-  DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(),
-                          MVT::Other, getControlRoot(),
-                          DAG.getBasicBlock(Return)));
+  SDValue Branch = DAG.getNode(ISD::BR, getCurDebugLoc(),
+                               MVT::Other, getControlRoot(),
+                               DAG.getBasicBlock(Return));
+  DAG.setRoot(Branch);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Branch.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitUnwind(UnwindInst &I) {
@@ -1669,7 +1847,7 @@ bool SelectionDAGBuilder::handleBTSplitSwitchCase(CaseRec& CR,
     APInt Range = ComputeRange(LEnd, RBegin);
     assert((Range - 2ULL).isNonNegative() &&
            "Invalid case distance");
-    double LDensity = (double)LSize.roundToDouble() / 
+    double LDensity = (double)LSize.roundToDouble() /
                            (LEnd - First + 1ULL).roundToDouble();
     double RDensity = (double)RSize.roundToDouble() /
                            (Last - RBegin + 1ULL).roundToDouble();
@@ -1787,8 +1965,9 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
       // Don't bother the code below, if there are too much unique destinations
       return false;
   }
-  DEBUG(errs() << "Total number of unique destinations: " << Dests.size() << '\n'
-               << "Total number of comparisons: " << numCmps << '\n');
+  DEBUG(errs() << "Total number of unique destinations: "
+        << Dests.size() << '\n'
+        << "Total number of comparisons: " << numCmps << '\n');
 
   // Compute span of values.
   const APInt& minValue = cast<ConstantInt>(FrontCase.Low)->getValue();
@@ -1883,7 +2062,6 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
   return true;
 }
 
-
 /// Clusterify - Transform simple list of Cases into list of CaseRange's
 size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
                                        const SwitchInst& SI) {
@@ -1930,7 +2108,6 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
 void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) {
   // Figure out which block is immediately after the current one.
   MachineBasicBlock *NextBlock = 0;
-
   MachineBasicBlock *Default = FuncInfo.MBBMap[SI.getDefaultDest()];
 
   // If there is only the default destination, branch to it if it is not the
@@ -1940,10 +2117,16 @@ void SelectionDAGBuilder::visitSwitch(SwitchInst &SI) {
 
     // If this is not a fall-through branch, emit the branch.
     CurMBB->addSuccessor(Default);
-    if (Default != NextBlock)
-      DAG.setRoot(DAG.getNode(ISD::BR, getCurDebugLoc(),
-                              MVT::Other, getControlRoot(),
-                              DAG.getBasicBlock(Default)));
+    if (Default != NextBlock) {
+      SDValue Res = DAG.getNode(ISD::BR, getCurDebugLoc(),
+                                MVT::Other, getControlRoot(),
+                                DAG.getBasicBlock(Default));
+      DAG.setRoot(Res);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+    }
+
     return;
   }
 
@@ -1995,11 +2178,14 @@ void SelectionDAGBuilder::visitIndirectBr(IndirectBrInst &I) {
   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i)
     CurMBB->addSuccessor(FuncInfo.MBBMap[I.getSuccessor(i)]);
 
-  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurDebugLoc(),
-                          MVT::Other, getControlRoot(),
-                          getValue(I.getAddress())));
-}
+  SDValue Res = DAG.getNode(ISD::BRIND, getCurDebugLoc(),
+                            MVT::Other, getControlRoot(),
+                            getValue(I.getAddress()));
+  DAG.setRoot(Res);
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+}
 
 void SelectionDAGBuilder::visitFSub(User &I) {
   // -0.0 - X --> fneg
@@ -2013,17 +2199,28 @@ void SelectionDAGBuilder::visitFSub(User &I) {
       Constant *CNZ = ConstantVector::get(&NZ[0], NZ.size());
       if (CV == CNZ) {
         SDValue Op2 = getValue(I.getOperand(1));
-        setValue(&I, DAG.getNode(ISD::FNEG, getCurDebugLoc(),
-                                 Op2.getValueType(), Op2));
+        SDValue Res = DAG.getNode(ISD::FNEG, getCurDebugLoc(),
+                                  Op2.getValueType(), Op2);
+        setValue(&I, Res);
+
+        if (DisableScheduling)
+          DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
         return;
       }
     }
   }
+
   if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0)))
     if (CFP->isExactlyValue(ConstantFP::getNegativeZero(Ty)->getValueAPF())) {
       SDValue Op2 = getValue(I.getOperand(1));
-      setValue(&I, DAG.getNode(ISD::FNEG, getCurDebugLoc(),
-                               Op2.getValueType(), Op2));
+      SDValue Res = DAG.getNode(ISD::FNEG, getCurDebugLoc(),
+                                Op2.getValueType(), Op2);
+      setValue(&I, Res);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
       return;
     }
 
@@ -2033,9 +2230,12 @@ void SelectionDAGBuilder::visitFSub(User &I) {
 void SelectionDAGBuilder::visitBinary(User &I, unsigned OpCode) {
   SDValue Op1 = getValue(I.getOperand(0));
   SDValue Op2 = getValue(I.getOperand(1));
+  SDValue Res = DAG.getNode(OpCode, getCurDebugLoc(),
+                            Op1.getValueType(), Op1, Op2);
+  setValue(&I, Res);
 
-  setValue(&I, DAG.getNode(OpCode, getCurDebugLoc(),
-                           Op1.getValueType(), Op1, Op2));
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitShift(User &I, unsigned Opcode) {
@@ -2068,8 +2268,15 @@ void SelectionDAGBuilder::visitShift(User &I, unsigned Opcode) {
                         TLI.getPointerTy(), Op2);
   }
 
-  setValue(&I, DAG.getNode(Opcode, getCurDebugLoc(),
-                           Op1.getValueType(), Op1, Op2));
+  SDValue Res = DAG.getNode(Opcode, getCurDebugLoc(),
+                            Op1.getValueType(), Op1, Op2);
+  setValue(&I, Res);
+
+  if (DisableScheduling) {
+    DAG.AssignOrdering(Op1.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(Op2.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+  }
 }
 
 void SelectionDAGBuilder::visitICmp(User &I) {
@@ -2081,9 +2288,13 @@ void SelectionDAGBuilder::visitICmp(User &I) {
   SDValue Op1 = getValue(I.getOperand(0));
   SDValue Op2 = getValue(I.getOperand(1));
   ISD::CondCode Opcode = getICmpCondCode(predicate);
-  
+
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Opcode));
+  SDValue Res = DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Opcode);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitFCmp(User &I) {
@@ -2096,37 +2307,54 @@ void SelectionDAGBuilder::visitFCmp(User &I) {
   SDValue Op2 = getValue(I.getOperand(1));
   ISD::CondCode Condition = getFCmpCondCode(predicate);
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Condition));
+  SDValue Res = DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Condition);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitSelect(User &I) {
   SmallVector<EVT, 4> ValueVTs;
   ComputeValueVTs(TLI, I.getType(), ValueVTs);
   unsigned NumValues = ValueVTs.size();
-  if (NumValues != 0) {
-    SmallVector<SDValue, 4> Values(NumValues);
-    SDValue Cond     = getValue(I.getOperand(0));
-    SDValue TrueVal  = getValue(I.getOperand(1));
-    SDValue FalseVal = getValue(I.getOperand(2));
+  if (NumValues == 0) return;
+
+  SmallVector<SDValue, 4> Values(NumValues);
+  SDValue Cond     = getValue(I.getOperand(0));
+  SDValue TrueVal  = getValue(I.getOperand(1));
+  SDValue FalseVal = getValue(I.getOperand(2));
 
-    for (unsigned i = 0; i != NumValues; ++i)
-      Values[i] = DAG.getNode(ISD::SELECT, getCurDebugLoc(),
-                              TrueVal.getNode()->getValueType(i), Cond,
-                              SDValue(TrueVal.getNode(), TrueVal.getResNo() + i),
-                              SDValue(FalseVal.getNode(), FalseVal.getResNo() + i));
+  for (unsigned i = 0; i != NumValues; ++i) {
+    Values[i] = DAG.getNode(ISD::SELECT, getCurDebugLoc(),
+                            TrueVal.getNode()->getValueType(i), Cond,
+                            SDValue(TrueVal.getNode(),
+                                    TrueVal.getResNo() + i),
+                            SDValue(FalseVal.getNode(),
+                                    FalseVal.getResNo() + i));
 
-    setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
-                             DAG.getVTList(&ValueVTs[0], NumValues),
-                             &Values[0], NumValues));
+    if (DisableScheduling)
+      DAG.AssignOrdering(Values[i].getNode(), SDNodeOrder);
   }
-}
 
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
+                            DAG.getVTList(&ValueVTs[0], NumValues),
+                            &Values[0], NumValues);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+}
 
 void SelectionDAGBuilder::visitTrunc(User &I) {
   // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitZExt(User &I) {
@@ -2134,7 +2362,11 @@ void SelectionDAGBuilder::visitZExt(User &I) {
   // ZExt also can't be a cast to bool for same reason. So, nothing much to do
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitSExt(User &I) {
@@ -2142,50 +2374,78 @@ void SelectionDAGBuilder::visitSExt(User &I) {
   // SExt also can't be a cast to bool for same reason. So, nothing much to do
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::SIGN_EXTEND, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitFPTrunc(User &I) {
   // FPTrunc is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::FP_ROUND, getCurDebugLoc(),
-                           DestVT, N, DAG.getIntPtrConstant(0)));
+  SDValue Res = DAG.getNode(ISD::FP_ROUND, getCurDebugLoc(),
+                            DestVT, N, DAG.getIntPtrConstant(0));
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitFPExt(User &I){
   // FPTrunc is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::FP_EXTEND, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitFPToUI(User &I) {
   // FPToUI is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::FP_TO_UINT, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitFPToSI(User &I) {
   // FPToSI is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::FP_TO_SINT, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitUIToFP(User &I) {
   // UIToFP is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::UINT_TO_FP, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitSIToFP(User &I){
   // SIToFP is never a no-op cast, no need to check
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurDebugLoc(), DestVT, N));
+  SDValue Res = DAG.getNode(ISD::SINT_TO_FP, getCurDebugLoc(), DestVT, N);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitPtrToInt(User &I) {
@@ -2194,8 +2454,11 @@ void SelectionDAGBuilder::visitPtrToInt(User &I) {
   SDValue N = getValue(I.getOperand(0));
   EVT SrcVT = N.getValueType();
   EVT DestVT = TLI.getValueType(I.getType());
-  SDValue Result = DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT);
-  setValue(&I, Result);
+  SDValue Res = DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitIntToPtr(User &I) {
@@ -2204,41 +2467,61 @@ void SelectionDAGBuilder::visitIntToPtr(User &I) {
   SDValue N = getValue(I.getOperand(0));
   EVT SrcVT = N.getValueType();
   EVT DestVT = TLI.getValueType(I.getType());
-  setValue(&I, DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT));
+  SDValue Res = DAG.getZExtOrTrunc(N, getCurDebugLoc(), DestVT);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitBitCast(User &I) {
   SDValue N = getValue(I.getOperand(0));
   EVT DestVT = TLI.getValueType(I.getType());
 
-  // BitCast assures us that source and destination are the same size so this
-  // is either a BIT_CONVERT or a no-op.
-  if (DestVT != N.getValueType())
-    setValue(&I, DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(),
-                             DestVT, N)); // convert types
-  else
-    setValue(&I, N); // noop cast.
+  // BitCast assures us that source and destination are the same size so this is
+  // either a BIT_CONVERT or a no-op.
+  if (DestVT != N.getValueType()) {
+    SDValue Res = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(),
+                              DestVT, N); // convert types.
+    setValue(&I, Res);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+  } else {
+    setValue(&I, N);            // noop cast.
+  }
 }
 
 void SelectionDAGBuilder::visitInsertElement(User &I) {
   SDValue InVec = getValue(I.getOperand(0));
   SDValue InVal = getValue(I.getOperand(1));
   SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(),
-                                TLI.getPointerTy(),
-                                getValue(I.getOperand(2)));
+                              TLI.getPointerTy(),
+                              getValue(I.getOperand(2)));
+  SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurDebugLoc(),
+                            TLI.getValueType(I.getType()),
+                            InVec, InVal, InIdx);
+  setValue(&I, Res);
 
-  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurDebugLoc(),
-                           TLI.getValueType(I.getType()),
-                           InVec, InVal, InIdx));
+  if (DisableScheduling) {
+    DAG.AssignOrdering(InIdx.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+  }
 }
 
 void SelectionDAGBuilder::visitExtractElement(User &I) {
   SDValue InVec = getValue(I.getOperand(0));
   SDValue InIdx = DAG.getNode(ISD::ZERO_EXTEND, getCurDebugLoc(),
-                                TLI.getPointerTy(),
-                                getValue(I.getOperand(1)));
-  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
-                           TLI.getValueType(I.getType()), InVec, InIdx));
+                              TLI.getPointerTy(),
+                              getValue(I.getOperand(1)));
+  SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
+                            TLI.getValueType(I.getType()), InVec, InIdx);
+  setValue(&I, Res);
+
+  if (DisableScheduling) {
+    DAG.AssignOrdering(InIdx.getNode(), SDNodeOrder);
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+  }
 }
 
 
@@ -2260,7 +2543,7 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
   // Convert the ConstantVector mask operand into an array of ints, with -1
   // representing undef values.
   SmallVector<Constant*, 8> MaskElts;
-  cast<Constant>(I.getOperand(2))->getVectorElements(*DAG.getContext(), 
+  cast<Constant>(I.getOperand(2))->getVectorElements(*DAG.getContext(),
                                                      MaskElts);
   unsigned MaskNumElts = MaskElts.size();
   for (unsigned i = 0; i != MaskNumElts; ++i) {
@@ -2269,14 +2552,19 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
     else
       Mask.push_back(cast<ConstantInt>(MaskElts[i])->getSExtValue());
   }
-  
+
   EVT VT = TLI.getValueType(I.getType());
   EVT SrcVT = Src1.getValueType();
   unsigned SrcNumElts = SrcVT.getVectorNumElements();
 
   if (SrcNumElts == MaskNumElts) {
-    setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
-                                      &Mask[0]));
+    SDValue Res = DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
+                                       &Mask[0]);
+    setValue(&I, Res);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
     return;
   }
 
@@ -2287,8 +2575,13 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
     // lengths match.
     if (SrcNumElts*2 == MaskNumElts && SequentialMask(Mask, 0)) {
       // The shuffle is concatenating two vectors together.
-      setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, getCurDebugLoc(),
-                               VT, Src1, Src2));
+      SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, getCurDebugLoc(),
+                                VT, Src1, Src2);
+      setValue(&I, Res);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
       return;
     }
 
@@ -2302,12 +2595,12 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
     SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
     MOps1[0] = Src1;
     MOps2[0] = Src2;
-    
-    Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS, 
-                                                  getCurDebugLoc(), VT, 
+
+    Src1 = Src1U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
+                                                  getCurDebugLoc(), VT,
                                                   &MOps1[0], NumConcat);
     Src2 = Src2U ? DAG.getUNDEF(VT) : DAG.getNode(ISD::CONCAT_VECTORS,
-                                                  getCurDebugLoc(), VT, 
+                                                  getCurDebugLoc(), VT,
                                                   &MOps2[0], NumConcat);
 
     // Readjust mask for new input vector length.
@@ -2319,8 +2612,17 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
       else
         MappedOps.push_back(Idx + MaskNumElts - SrcNumElts);
     }
-    setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2, 
-                                      &MappedOps[0]));
+
+    SDValue Res = DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
+                                       &MappedOps[0]);
+    setValue(&I, Res);
+
+    if (DisableScheduling) {
+      DAG.AssignOrdering(Src1.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(Src2.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+    }
+
     return;
   }
 
@@ -2336,7 +2638,7 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
       int Input = 0;
       if (Idx < 0)
         continue;
-      
+
       if (Idx >= (int)SrcNumElts) {
         Input = 1;
         Idx -= SrcNumElts;
@@ -2349,7 +2651,8 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
 
     // Check if the access is smaller than the vector size and can we find
     // a reasonable extract index.
-    int RangeUse[2] = { 2, 2 };  // 0 = Unused, 1 = Extract, 2 = Can not Extract.
+    int RangeUse[2] = { 2, 2 };  // 0 = Unused, 1 = Extract, 2 = Can not
+                                 // Extract.
     int StartIdx[2];  // StartIdx to extract from
     for (int Input=0; Input < 2; ++Input) {
       if (MinRange[Input] == (int)(SrcNumElts+1) && MaxRange[Input] == -1) {
@@ -2371,20 +2674,28 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
     }
 
     if (RangeUse[0] == 0 && RangeUse[1] == 0) {
-      setValue(&I, DAG.getUNDEF(VT));  // Vectors are not used.
+      SDValue Res = DAG.getUNDEF(VT);
+      setValue(&I, Res);  // Vectors are not used.
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
       return;
     }
     else if (RangeUse[0] < 2 && RangeUse[1] < 2) {
       // Extract appropriate subvector and generate a vector shuffle
       for (int Input=0; Input < 2; ++Input) {
-        SDValue& Src = Input == 0 ? Src1 : Src2;
-        if (RangeUse[Input] == 0) {
+        SDValue &Src = Input == 0 ? Src1 : Src2;
+        if (RangeUse[Input] == 0)
           Src = DAG.getUNDEF(VT);
-        } else {
+        else
           Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, getCurDebugLoc(), VT,
                             Src, DAG.getIntPtrConstant(StartIdx[Input]));
-        }
+
+        if (DisableScheduling)
+          DAG.AssignOrdering(Src.getNode(), SDNodeOrder);
       }
+
       // Calculate new mask.
       SmallVector<int, 8> MappedOps;
       for (unsigned i = 0; i != MaskNumElts; ++i) {
@@ -2396,8 +2707,14 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
         else
           MappedOps.push_back(Idx - SrcNumElts - StartIdx[1] + MaskNumElts);
       }
-      setValue(&I, DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
-                                        &MappedOps[0]));
+
+      SDValue Res = DAG.getVectorShuffle(VT, getCurDebugLoc(), Src1, Src2,
+                                         &MappedOps[0]);
+      setValue(&I, Res);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+
       return;
     }
   }
@@ -2413,17 +2730,29 @@ void SelectionDAGBuilder::visitShuffleVector(User &I) {
       Ops.push_back(DAG.getUNDEF(EltVT));
     } else {
       int Idx = Mask[i];
+      SDValue Res;
+
       if (Idx < (int)SrcNumElts)
-        Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
-                                  EltVT, Src1, DAG.getConstant(Idx, PtrVT)));
+        Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
+                          EltVT, Src1, DAG.getConstant(Idx, PtrVT));
       else
-        Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
-                                  EltVT, Src2,
-                                  DAG.getConstant(Idx - SrcNumElts, PtrVT)));
+        Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurDebugLoc(),
+                          EltVT, Src2,
+                          DAG.getConstant(Idx - SrcNumElts, PtrVT));
+
+      Ops.push_back(Res);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     }
   }
-  setValue(&I, DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(),
-                           VT, &Ops[0], Ops.size()));
+
+  SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, getCurDebugLoc(),
+                            VT, &Ops[0], Ops.size());
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitInsertValue(InsertValueInst &I) {
@@ -2462,9 +2791,13 @@ void SelectionDAGBuilder::visitInsertValue(InsertValueInst &I) {
     Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
                 SDValue(Agg.getNode(), Agg.getResNo() + i);
 
-  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
-                           DAG.getVTList(&AggValueVTs[0], NumAggValues),
-                           &Values[0], NumAggValues));
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
+                            DAG.getVTList(&AggValueVTs[0], NumAggValues),
+                            &Values[0], NumAggValues);
+  setValue(&I, Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 void SelectionDAGBuilder::visitExtractValue(ExtractValueInst &I) {
@@ -2490,11 +2823,14 @@ void SelectionDAGBuilder::visitExtractValue(ExtractValueInst &I) {
         DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
         SDValue(Agg.getNode(), Agg.getResNo() + i);
 
-  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
-                           DAG.getVTList(&ValValueVTs[0], NumValValues),
-                           &Values[0], NumValValues));
-}
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
+                            DAG.getVTList(&ValValueVTs[0], NumValValues),
+                            &Values[0], NumValValues);
+  setValue(&I, Res);
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+}
 
 void SelectionDAGBuilder::visitGetElementPtr(User &I) {
   SDValue N = getValue(I.getOperand(0));
@@ -2510,7 +2846,11 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) {
         uint64_t Offset = TD->getStructLayout(StTy)->getElementOffset(Field);
         N = DAG.getNode(ISD::ADD, getCurDebugLoc(), N.getValueType(), N,
                         DAG.getIntPtrConstant(Offset));
+
+        if (DisableScheduling)
+          DAG.AssignOrdering(N.getNode(), SDNodeOrder);
       }
+
       Ty = StTy->getElementType(Field);
     } else {
       Ty = cast<SequentialType>(Ty)->getElementType();
@@ -2523,14 +2863,21 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) {
         SDValue OffsVal;
         EVT PTy = TLI.getPointerTy();
         unsigned PtrBits = PTy.getSizeInBits();
-        if (PtrBits < 64) {
+        if (PtrBits < 64)
           OffsVal = DAG.getNode(ISD::TRUNCATE, getCurDebugLoc(),
                                 TLI.getPointerTy(),
                                 DAG.getConstant(Offs, MVT::i64));
-        } else
+        else
           OffsVal = DAG.getIntPtrConstant(Offs);
+
         N = DAG.getNode(ISD::ADD, getCurDebugLoc(), N.getValueType(), N,
                         OffsVal);
+
+        if (DisableScheduling) {
+          DAG.AssignOrdering(OffsVal.getNode(), SDNodeOrder);
+          DAG.AssignOrdering(N.getNode(), SDNodeOrder);
+        }
+
         continue;
       }
 
@@ -2556,12 +2903,19 @@ void SelectionDAGBuilder::visitGetElementPtr(User &I) {
           IdxN = DAG.getNode(ISD::MUL, getCurDebugLoc(),
                              N.getValueType(), IdxN, Scale);
         }
+
+        if (DisableScheduling)
+          DAG.AssignOrdering(IdxN.getNode(), SDNodeOrder);
       }
 
       N = DAG.getNode(ISD::ADD, getCurDebugLoc(),
                       N.getValueType(), N, IdxN);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(N.getNode(), SDNodeOrder);
     }
   }
+
   setValue(&I, N);
 }
 
@@ -2578,16 +2932,20 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) {
              I.getAlignment());
 
   SDValue AllocSize = getValue(I.getArraySize());
-  
+
   AllocSize = DAG.getNode(ISD::MUL, getCurDebugLoc(), AllocSize.getValueType(),
                           AllocSize,
                           DAG.getConstant(TySize, AllocSize.getValueType()));
-  
-  
-  
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder);
+
   EVT IntPtr = TLI.getPointerTy();
   AllocSize = DAG.getZExtOrTrunc(AllocSize, getCurDebugLoc(), IntPtr);
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder);
+
   // Handle alignment.  If the requested alignment is less than or equal to
   // the stack alignment, ignore it.  If the size is greater than or equal to
   // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
@@ -2601,10 +2959,15 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) {
   AllocSize = DAG.getNode(ISD::ADD, getCurDebugLoc(),
                           AllocSize.getValueType(), AllocSize,
                           DAG.getIntPtrConstant(StackAlign-1));
+  if (DisableScheduling)
+    DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder);
+
   // Mask out the low bits for alignment purposes.
   AllocSize = DAG.getNode(ISD::AND, getCurDebugLoc(),
                           AllocSize.getValueType(), AllocSize,
                           DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1)));
+  if (DisableScheduling)
+    DAG.AssignOrdering(AllocSize.getNode(), SDNodeOrder);
 
   SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) };
   SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
@@ -2613,6 +2976,9 @@ void SelectionDAGBuilder::visitAlloca(AllocaInst &I) {
   setValue(&I, DSA);
   DAG.setRoot(DSA.getValue(1));
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(DSA.getNode(), SDNodeOrder);
+
   // Inform the Frame Information that we have just allocated a variable-sized
   // object.
   FuncInfo.MF->getFrameInfo()->CreateVariableSizedObject();
@@ -2651,30 +3017,41 @@ void SelectionDAGBuilder::visitLoad(LoadInst &I) {
   SmallVector<SDValue, 4> Chains(NumValues);
   EVT PtrVT = Ptr.getValueType();
   for (unsigned i = 0; i != NumValues; ++i) {
+    SDValue A = DAG.getNode(ISD::ADD, getCurDebugLoc(),
+                            PtrVT, Ptr,
+                            DAG.getConstant(Offsets[i], PtrVT));
     SDValue L = DAG.getLoad(ValueVTs[i], getCurDebugLoc(), Root,
-                            DAG.getNode(ISD::ADD, getCurDebugLoc(),
-                                        PtrVT, Ptr,
-                                        DAG.getConstant(Offsets[i], PtrVT)),
-                            SV, Offsets[i], isVolatile, Alignment);
+                            A, SV, Offsets[i], isVolatile, Alignment);
+
     Values[i] = L;
     Chains[i] = L.getValue(1);
+
+    if (DisableScheduling) {
+      DAG.AssignOrdering(A.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(L.getNode(), SDNodeOrder);
+    }
   }
 
   if (!ConstantMemory) {
     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),
-                                  MVT::Other,
-                                  &Chains[0], NumValues);
+                                MVT::Other, &Chains[0], NumValues);
     if (isVolatile)
       DAG.setRoot(Chain);
     else
       PendingLoads.push_back(Chain);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Chain.getNode(), SDNodeOrder);
   }
 
-  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
-                           DAG.getVTList(&ValueVTs[0], NumValues),
-                           &Values[0], NumValues));
-}
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, getCurDebugLoc(),
+                            DAG.getVTList(&ValueVTs[0], NumValues),
+                            &Values[0], NumValues);
+  setValue(&I, Res);
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+}
 
 void SelectionDAGBuilder::visitStore(StoreInst &I) {
   Value *SrcV = I.getOperand(0);
@@ -2698,16 +3075,26 @@ void SelectionDAGBuilder::visitStore(StoreInst &I) {
   EVT PtrVT = Ptr.getValueType();
   bool isVolatile = I.isVolatile();
   unsigned Alignment = I.getAlignment();
-  for (unsigned i = 0; i != NumValues; ++i)
+
+  for (unsigned i = 0; i != NumValues; ++i) {
+    SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr,
+                              DAG.getConstant(Offsets[i], PtrVT));
     Chains[i] = DAG.getStore(Root, getCurDebugLoc(),
                              SDValue(Src.getNode(), Src.getResNo() + i),
-                             DAG.getNode(ISD::ADD, getCurDebugLoc(),
-                                         PtrVT, Ptr,
-                                         DAG.getConstant(Offsets[i], PtrVT)),
-                             PtrV, Offsets[i], isVolatile, Alignment);
+                             Add, PtrV, Offsets[i], isVolatile, Alignment);
+
+    if (DisableScheduling) {
+      DAG.AssignOrdering(Add.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(Chains[i].getNode(), SDNodeOrder);
+    }
+  }
+
+  SDValue Res = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),
+                            MVT::Other, &Chains[0], NumValues);
+  DAG.setRoot(Res);
 
-  DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),
-                          MVT::Other, &Chains[0], NumValues));
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
 }
 
 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
@@ -2752,6 +3139,7 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I,
            "Intrinsic uses a non-legal type?");
   }
 #endif // NDEBUG
+
   if (HasChain)
     ValueVTs.push_back(MVT::Other);
 
@@ -2766,16 +3154,19 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I,
                                      Info.memVT, Info.ptrVal, Info.offset,
                                      Info.align, Info.vol,
                                      Info.readMem, Info.writeMem);
-  }
-  else if (!HasChain)
+  } else if (!HasChain) {
     Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurDebugLoc(),
                          VTs, &Ops[0], Ops.size());
-  else if (I.getType() != Type::getVoidTy(*DAG.getContext()))
+  } else if (I.getType() != Type::getVoidTy(*DAG.getContext())) {
     Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurDebugLoc(),
                          VTs, &Ops[0], Ops.size());
-  else
+  } else {
     Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurDebugLoc(),
                          VTs, &Ops[0], Ops.size());
+  }
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Result.getNode(), SDNodeOrder);
 
   if (HasChain) {
     SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
@@ -2784,11 +3175,16 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I,
     else
       DAG.setRoot(Chain);
   }
+
   if (I.getType() != Type::getVoidTy(*DAG.getContext())) {
     if (const VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
       EVT VT = TLI.getValueType(PTy);
       Result = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(), VT, Result);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Result.getNode(), SDNodeOrder);
     }
+
     setValue(&I, Result);
   }
 }
@@ -2800,12 +3196,20 @@ void SelectionDAGBuilder::visitTargetIntrinsic(CallInst &I,
 ///
 /// where Op is the hexidecimal representation of floating point value.
 static SDValue
-GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) {
+GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl, unsigned Order) {
   SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
                            DAG.getConstant(0x007fffff, MVT::i32));
   SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
                            DAG.getConstant(0x3f800000, MVT::i32));
-  return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t2);
+  SDValue Res = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t2);
+
+  if (DisableScheduling) {
+    DAG.AssignOrdering(t1.getNode(), Order);
+    DAG.AssignOrdering(t2.getNode(), Order);
+    DAG.AssignOrdering(Res.getNode(), Order);
+  }
+
+  return Res;
 }
 
 /// GetExponent - Get the exponent:
@@ -2815,14 +3219,23 @@ GetSignificand(SelectionDAG &DAG, SDValue Op, DebugLoc dl) {
 /// where Op is the hexidecimal representation of floating point value.
 static SDValue
 GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI,
-            DebugLoc dl) {
+            DebugLoc dl, unsigned Order) {
   SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
                            DAG.getConstant(0x7f800000, MVT::i32));
   SDValue t1 = DAG.getNode(ISD::SRL, dl, MVT::i32, t0,
                            DAG.getConstant(23, TLI.getPointerTy()));
   SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
                            DAG.getConstant(127, MVT::i32));
-  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
+  SDValue Res = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
+
+  if (DisableScheduling) {
+    DAG.AssignOrdering(t0.getNode(), Order);
+    DAG.AssignOrdering(t1.getNode(), Order);
+    DAG.AssignOrdering(t2.getNode(), Order);
+    DAG.AssignOrdering(Res.getNode(), Order);
+  }
+
+  return Res;
 }
 
 /// getF32Constant - Get 32-bit floating point constant.
@@ -2846,6 +3259,10 @@ SelectionDAGBuilder::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) {
                   I.getOperand(1));
   setValue(&I, L);
   DAG.setRoot(L.getValue(1));
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(L.getNode(), SDNodeOrder);
+
   return 0;
 }
 
@@ -2859,6 +3276,10 @@ SelectionDAGBuilder::implVisitAluOverflow(CallInst &I, ISD::NodeType Op) {
   SDValue Result = DAG.getNode(Op, getCurDebugLoc(), VTs, Op1, Op2);
 
   setValue(&I, Result);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Result.getNode(), SDNodeOrder);
+
   return 0;
 }
 
@@ -2886,10 +3307,20 @@ SelectionDAGBuilder::visitExp(CallInst &I) {
     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
 
+    if (DisableScheduling) {
+      DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(X.getNode(), SDNodeOrder);
+    }
+
     //   IntegerPartOfX <<= 23;
     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
                                  DAG.getConstant(23, TLI.getPointerTy()));
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder);
+
     if (LimitFloatPrecision <= 6) {
       // For floating-point precision of 6:
       //
@@ -2912,6 +3343,16 @@ SelectionDAGBuilder::visitExp(CallInst &I) {
                                TwoToFracPartOfX, IntegerPartOfX);
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t6);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -2938,6 +3379,18 @@ SelectionDAGBuilder::visitExp(CallInst &I) {
                                TwoToFracPartOfX, IntegerPartOfX);
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t8);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -2977,12 +3430,32 @@ SelectionDAGBuilder::visitExp(CallInst &I) {
                                 TwoToFracPartOfX, IntegerPartOfX);
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, t14);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t9.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t10.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t11.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t12.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t13.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t14.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFracPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     }
   } else {
     // No special expansion.
     result = DAG.getNode(ISD::FEXP, dl,
                          getValue(I.getOperand(1)).getValueType(),
                          getValue(I.getOperand(1)));
+    if (DisableScheduling)
+      DAG.AssignOrdering(result.getNode(), SDNodeOrder);
   }
 
   setValue(&I, result);
@@ -3000,14 +3473,20 @@ SelectionDAGBuilder::visitLog(CallInst &I) {
     SDValue Op = getValue(I.getOperand(1));
     SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op);
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(Op1.getNode(), SDNodeOrder);
+
     // Scale the exponent by log(2) [0.69314718f].
-    SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
+    SDValue Exp = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder);
     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
                                         getF32Constant(DAG, 0x3f317218));
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder);
+
     // Get the significand and build it into a floating-point number with
     // exponent of 1.
-    SDValue X = GetSignificand(DAG, Op1, dl);
+    SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder);
 
     if (LimitFloatPrecision <= 6) {
       // For floating-point precision of 6:
@@ -3027,6 +3506,14 @@ SelectionDAGBuilder::visitLog(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, LogOfMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -3053,6 +3540,18 @@ SelectionDAGBuilder::visitLog(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, LogOfMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -3087,12 +3586,31 @@ SelectionDAGBuilder::visitLog(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, LogOfMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t9.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t10.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(LogOfMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     }
   } else {
     // No special expansion.
     result = DAG.getNode(ISD::FLOG, dl,
                          getValue(I.getOperand(1)).getValueType(),
                          getValue(I.getOperand(1)));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(result.getNode(), SDNodeOrder);
   }
 
   setValue(&I, result);
@@ -3110,12 +3628,18 @@ SelectionDAGBuilder::visitLog2(CallInst &I) {
     SDValue Op = getValue(I.getOperand(1));
     SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op);
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(Op1.getNode(), SDNodeOrder);
+
     // Get the exponent.
-    SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
+    SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder);
 
     // Get the significand and build it into a floating-point number with
     // exponent of 1.
-    SDValue X = GetSignificand(DAG, Op1, dl);
+    SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder);
 
     // Different possible minimax approximations of significand in
     // floating-point for various degrees of accuracy over [1,2].
@@ -3135,6 +3659,14 @@ SelectionDAGBuilder::visitLog2(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, Log2ofMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -3161,6 +3693,18 @@ SelectionDAGBuilder::visitLog2(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, Log2ofMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -3196,12 +3740,31 @@ SelectionDAGBuilder::visitLog2(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, Log2ofMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t9.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t10.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Log2ofMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     }
   } else {
     // No special expansion.
     result = DAG.getNode(ISD::FLOG2, dl,
                          getValue(I.getOperand(1)).getValueType(),
                          getValue(I.getOperand(1)));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(result.getNode(), SDNodeOrder);
   }
 
   setValue(&I, result);
@@ -3219,14 +3782,20 @@ SelectionDAGBuilder::visitLog10(CallInst &I) {
     SDValue Op = getValue(I.getOperand(1));
     SDValue Op1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op);
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(Op1.getNode(), SDNodeOrder);
+
     // Scale the exponent by log10(2) [0.30102999f].
-    SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
+    SDValue Exp = GetExponent(DAG, Op1, TLI, dl, SDNodeOrder);
     SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
                                         getF32Constant(DAG, 0x3e9a209a));
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(LogOfExponent.getNode(), SDNodeOrder);
+
     // Get the significand and build it into a floating-point number with
     // exponent of 1.
-    SDValue X = GetSignificand(DAG, Op1, dl);
+    SDValue X = GetSignificand(DAG, Op1, dl, SDNodeOrder);
 
     if (LimitFloatPrecision <= 6) {
       // For floating-point precision of 6:
@@ -3246,6 +3815,14 @@ SelectionDAGBuilder::visitLog10(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, Log10ofMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -3268,6 +3845,16 @@ SelectionDAGBuilder::visitLog10(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, Log10ofMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -3298,12 +3885,29 @@ SelectionDAGBuilder::visitLog10(CallInst &I) {
 
       result = DAG.getNode(ISD::FADD, dl,
                            MVT::f32, LogOfExponent, Log10ofMantissa);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(Log10ofMantissa.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     }
   } else {
     // No special expansion.
     result = DAG.getNode(ISD::FLOG10, dl,
                          getValue(I.getOperand(1)).getValueType(),
                          getValue(I.getOperand(1)));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(result.getNode(), SDNodeOrder);
   }
 
   setValue(&I, result);
@@ -3322,6 +3926,9 @@ SelectionDAGBuilder::visitExp2(CallInst &I) {
 
     SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op);
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder);
+
     //   FractionalPartOfX = x - (float)IntegerPartOfX;
     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, Op, t1);
@@ -3330,6 +3937,12 @@ SelectionDAGBuilder::visitExp2(CallInst &I) {
     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
                                  DAG.getConstant(23, TLI.getPointerTy()));
 
+    if (DisableScheduling) {
+      DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(X.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder);
+    }
+
     if (LimitFloatPrecision <= 6) {
       // For floating-point precision of 6:
       //
@@ -3351,6 +3964,16 @@ SelectionDAGBuilder::visitExp2(CallInst &I) {
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl,
                            MVT::f32, TwoToFractionalPartOfX);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -3376,6 +3999,18 @@ SelectionDAGBuilder::visitExp2(CallInst &I) {
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl,
                            MVT::f32, TwoToFractionalPartOfX);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -3412,12 +4047,33 @@ SelectionDAGBuilder::visitExp2(CallInst &I) {
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl,
                            MVT::f32, TwoToFractionalPartOfX);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t9.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t10.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t11.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t12.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t13.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t14.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     }
   } else {
     // No special expansion.
     result = DAG.getNode(ISD::FEXP2, dl,
                          getValue(I.getOperand(1)).getValueType(),
                          getValue(I.getOperand(1)));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(result.getNode(), SDNodeOrder);
   }
 
   setValue(&I, result);
@@ -3459,10 +4115,20 @@ SelectionDAGBuilder::visitPow(CallInst &I) {
     SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
     SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
 
+    if (DisableScheduling) {
+      DAG.AssignOrdering(t0.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(t1.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(X.getNode(), SDNodeOrder);
+    }
+
     //   IntegerPartOfX <<= 23;
     IntegerPartOfX = DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
                                  DAG.getConstant(23, TLI.getPointerTy()));
 
+    if (DisableScheduling)
+      DAG.AssignOrdering(IntegerPartOfX.getNode(), SDNodeOrder);
+
     if (LimitFloatPrecision <= 6) {
       // For floating-point precision of 6:
       //
@@ -3484,6 +4150,16 @@ SelectionDAGBuilder::visitPow(CallInst &I) {
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl,
                            MVT::f32, TwoToFractionalPartOfX);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else if (LimitFloatPrecision > 6 && LimitFloatPrecision <= 12) {
       // For floating-point precision of 12:
       //
@@ -3509,6 +4185,18 @@ SelectionDAGBuilder::visitPow(CallInst &I) {
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl,
                            MVT::f32, TwoToFractionalPartOfX);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     } else { // LimitFloatPrecision > 12 && LimitFloatPrecision <= 18
       // For floating-point precision of 18:
       //
@@ -3545,6 +4233,24 @@ SelectionDAGBuilder::visitPow(CallInst &I) {
 
       result = DAG.getNode(ISD::BIT_CONVERT, dl,
                            MVT::f32, TwoToFractionalPartOfX);
+
+      if (DisableScheduling) {
+        DAG.AssignOrdering(t2.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t3.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t4.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t5.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t6.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t7.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t8.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t9.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t10.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t11.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t12.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t13.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(t14.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(TwoToFractionalPartOfX.getNode(), SDNodeOrder);
+        DAG.AssignOrdering(result.getNode(), SDNodeOrder);
+      }
     }
   } else {
     // No special expansion.
@@ -3552,17 +4258,76 @@ SelectionDAGBuilder::visitPow(CallInst &I) {
                          getValue(I.getOperand(1)).getValueType(),
                          getValue(I.getOperand(1)),
                          getValue(I.getOperand(2)));
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(result.getNode(), SDNodeOrder);
   }
 
   setValue(&I, result);
 }
 
+
+/// ExpandPowI - Expand a llvm.powi intrinsic.
+static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS,
+                          SelectionDAG &DAG) {
+  // If RHS is a constant, we can expand this out to a multiplication tree,
+  // otherwise we end up lowering to a call to __powidf2 (for example).  When
+  // optimizing for size, we only want to do this if the expansion would produce
+  // a small number of multiplies, otherwise we do the full expansion.
+  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
+    // Get the exponent as a positive value.
+    unsigned Val = RHSC->getSExtValue();
+    if ((int)Val < 0) Val = -Val;
+
+    // powi(x, 0) -> 1.0
+    if (Val == 0)
+      return DAG.getConstantFP(1.0, LHS.getValueType());
+
+    Function *F = DAG.getMachineFunction().getFunction();
+    if (!F->hasFnAttr(Attribute::OptimizeForSize) ||
+        // If optimizing for size, don't insert too many multiplies.  This
+        // inserts up to 5 multiplies.
+        CountPopulation_32(Val)+Log2_32(Val) < 7) {
+      // We use the simple binary decomposition method to generate the multiply
+      // sequence.  There are more optimal ways to do this (for example,
+      // powi(x,15) generates one more multiply than it should), but this has
+      // the benefit of being both really simple and much better than a libcall.
+      SDValue Res;  // Logically starts equal to 1.0
+      SDValue CurSquare = LHS;
+      while (Val) {
+        if (Val & 1) {
+          if (Res.getNode())
+            Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
+          else
+            Res = CurSquare;  // 1.0*CurSquare.
+        }
+
+        CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
+                                CurSquare, CurSquare);
+        Val >>= 1;
+      }
+
+      // If the original was negative, invert the result, producing 1/(x*x*x).
+      if (RHSC->getSExtValue() < 0)
+        Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
+                          DAG.getConstantFP(1.0, LHS.getValueType()), Res);
+      return Res;
+    }
+  }
+
+  // Otherwise, expand to a libcall.
+  return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
+}
+
+
 /// visitIntrinsicCall - Lower the call to the specified intrinsic function.  If
 /// we want to emit this as a call to a named external function, return the name
 /// otherwise lower it and return null.
 const char *
 SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
   DebugLoc dl = getCurDebugLoc();
+  SDValue Res;
+
   switch (Intrinsic) {
   default:
     // By default, turn this into a target intrinsic node.
@@ -3572,26 +4337,33 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
   case Intrinsic::vaend:    visitVAEnd(I); return 0;
   case Intrinsic::vacopy:   visitVACopy(I); return 0;
   case Intrinsic::returnaddress:
-    setValue(&I, DAG.getNode(ISD::RETURNADDR, dl, TLI.getPointerTy(),
-                             getValue(I.getOperand(1))));
+    Res = DAG.getNode(ISD::RETURNADDR, dl, TLI.getPointerTy(),
+                      getValue(I.getOperand(1)));
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::frameaddress:
-    setValue(&I, DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(),
-                             getValue(I.getOperand(1))));
+    Res = DAG.getNode(ISD::FRAMEADDR, dl, TLI.getPointerTy(),
+                      getValue(I.getOperand(1)));
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::setjmp:
     return "_setjmp"+!TLI.usesUnderscoreSetJmp();
-    break;
   case Intrinsic::longjmp:
     return "_longjmp"+!TLI.usesUnderscoreLongJmp();
-    break;
   case Intrinsic::memcpy: {
     SDValue Op1 = getValue(I.getOperand(1));
     SDValue Op2 = getValue(I.getOperand(2));
     SDValue Op3 = getValue(I.getOperand(3));
     unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue();
-    DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false,
-                              I.getOperand(1), 0, I.getOperand(2), 0));
+    Res = DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false,
+                        I.getOperand(1), 0, I.getOperand(2), 0);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::memset: {
@@ -3599,8 +4371,11 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDValue Op2 = getValue(I.getOperand(2));
     SDValue Op3 = getValue(I.getOperand(3));
     unsigned Align = cast<ConstantInt>(I.getOperand(4))->getZExtValue();
-    DAG.setRoot(DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align,
-                              I.getOperand(1), 0));
+    Res = DAG.getMemset(getRoot(), dl, Op1, Op2, Op3, Align,
+                        I.getOperand(1), 0);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::memmove: {
@@ -3616,30 +4391,36 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
       Size = C->getZExtValue();
     if (AA->alias(I.getOperand(1), Size, I.getOperand(2), Size) ==
         AliasAnalysis::NoAlias) {
-      DAG.setRoot(DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false,
-                                I.getOperand(1), 0, I.getOperand(2), 0));
+      Res = DAG.getMemcpy(getRoot(), dl, Op1, Op2, Op3, Align, false,
+                          I.getOperand(1), 0, I.getOperand(2), 0);
+      DAG.setRoot(Res);
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
       return 0;
     }
 
-    DAG.setRoot(DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align,
-                               I.getOperand(1), 0, I.getOperand(2), 0));
+    Res = DAG.getMemmove(getRoot(), dl, Op1, Op2, Op3, Align,
+                         I.getOperand(1), 0, I.getOperand(2), 0);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
-  case Intrinsic::dbg_stoppoint: 
+  case Intrinsic::dbg_stoppoint:
   case Intrinsic::dbg_region_start:
   case Intrinsic::dbg_region_end:
   case Intrinsic::dbg_func_start:
     // FIXME - Remove this instructions once the dust settles.
     return 0;
   case Intrinsic::dbg_declare: {
-    if (OptLevel != CodeGenOpt::None) 
+    if (OptLevel != CodeGenOpt::None)
       // FIXME: Variable debug info is not supported here.
       return 0;
     DwarfWriter *DW = DAG.getDwarfWriter();
     if (!DW)
       return 0;
     DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
-    if (!isValidDebugInfoIntrinsic(DI, CodeGenOpt::None))
+    if (!DIDescriptor::ValidDebugInfo(DI.getVariable(), CodeGenOpt::None))
       return 0;
 
     MDNode *Variable = DI.getVariable();
@@ -3652,18 +4433,13 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
       return 0;
     DenseMap<const AllocaInst*, int>::iterator SI =
       FuncInfo.StaticAllocaMap.find(AI);
-    if (SI == FuncInfo.StaticAllocaMap.end()) 
+    if (SI == FuncInfo.StaticAllocaMap.end())
       return 0; // VLAs.
     int FI = SI->second;
 
-    MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
-    if (MMI) {
-      MetadataContext &TheMetadata = 
-        DI.getParent()->getContext().getMetadata();
-      unsigned MDDbgKind = TheMetadata.getMDKind("dbg");
-      MDNode *Dbg = TheMetadata.getMD(MDDbgKind, &DI);
-      MMI->setVariableDbgInfo(Variable, FI, Dbg);
-    }
+    if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo())
+      if (MDNode *Dbg = DI.getMetadata("dbg"))
+        MMI->setVariableDbgInfo(Variable, FI, Dbg);
     return 0;
   }
   case Intrinsic::eh_exception: {
@@ -3675,6 +4451,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDValue Op = DAG.getNode(ISD::EXCEPTIONADDR, dl, VTs, Ops, 1);
     setValue(&I, Op);
     DAG.setRoot(Op.getValue(1));
+    if (DisableScheduling)
+      DAG.AssignOrdering(Op.getNode(), SDNodeOrder);
     return 0;
   }
 
@@ -3701,7 +4479,12 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
 
     DAG.setRoot(Op.getValue(1));
 
-    setValue(&I, DAG.getSExtOrTrunc(Op, dl, MVT::i32));
+    Res = DAG.getSExtOrTrunc(Op, dl, MVT::i32);
+    setValue(&I, Res);
+    if (DisableScheduling) {
+      DAG.AssignOrdering(Op.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+    }
     return 0;
   }
 
@@ -3711,14 +4494,16 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     if (MMI) {
       // Find the type id for the given typeinfo.
       GlobalVariable *GV = ExtractTypeInfo(I.getOperand(1));
-
       unsigned TypeID = MMI->getTypeIDFor(GV);
-      setValue(&I, DAG.getConstant(TypeID, MVT::i32));
+      Res = DAG.getConstant(TypeID, MVT::i32);
     } else {
       // Return something different to eh_selector.
-      setValue(&I, DAG.getConstant(1, MVT::i32));
+      Res = DAG.getConstant(1, MVT::i32);
     }
 
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
 
@@ -3726,11 +4511,14 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
   case Intrinsic::eh_return_i64:
     if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) {
       MMI->setCallsEHReturn(true);
-      DAG.setRoot(DAG.getNode(ISD::EH_RETURN, dl,
-                              MVT::Other,
-                              getControlRoot(),
-                              getValue(I.getOperand(1)),
-                              getValue(I.getOperand(2))));
+      Res = DAG.getNode(ISD::EH_RETURN, dl,
+                        MVT::Other,
+                        getControlRoot(),
+                        getValue(I.getOperand(1)),
+                        getValue(I.getOperand(2)));
+      DAG.setRoot(Res);
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     } else {
       setValue(&I, DAG.getConstant(0, TLI.getPointerTy()));
     }
@@ -3740,26 +4528,28 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     if (MachineModuleInfo *MMI = DAG.getMachineModuleInfo()) {
       MMI->setCallsUnwindInit(true);
     }
-
     return 0;
-
   case Intrinsic::eh_dwarf_cfa: {
     EVT VT = getValue(I.getOperand(1)).getValueType();
     SDValue CfaArg = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), dl,
                                         TLI.getPointerTy());
-
     SDValue Offset = DAG.getNode(ISD::ADD, dl,
                                  TLI.getPointerTy(),
                                  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
                                              TLI.getPointerTy()),
                                  CfaArg);
-    setValue(&I, DAG.getNode(ISD::ADD, dl,
+    SDValue FA = DAG.getNode(ISD::FRAMEADDR, dl,
                              TLI.getPointerTy(),
-                             DAG.getNode(ISD::FRAMEADDR, dl,
-                                         TLI.getPointerTy(),
-                                         DAG.getConstant(0,
-                                                         TLI.getPointerTy())),
-                             Offset));
+                             DAG.getConstant(0, TLI.getPointerTy()));
+    Res = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(),
+                      FA, Offset);
+    setValue(&I, Res);
+    if (DisableScheduling) {
+      DAG.AssignOrdering(CfaArg.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(Offset.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(FA.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
+    }
     return 0;
   }
   case Intrinsic::convertff:
@@ -3784,36 +4574,48 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     case Intrinsic::convertuu:  Code = ISD::CVT_UU; break;
     }
     EVT DestVT = TLI.getValueType(I.getType());
-    Value* Op1 = I.getOperand(1);
-    setValue(&I, DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1),
-                                DAG.getValueType(DestVT),
-                                DAG.getValueType(getValue(Op1).getValueType()),
-                                getValue(I.getOperand(2)),
-                                getValue(I.getOperand(3)),
-                                Code));
+    Value *Op1 = I.getOperand(1);
+    Res = DAG.getConvertRndSat(DestVT, getCurDebugLoc(), getValue(Op1),
+                               DAG.getValueType(DestVT),
+                               DAG.getValueType(getValue(Op1).getValueType()),
+                               getValue(I.getOperand(2)),
+                               getValue(I.getOperand(3)),
+                               Code);
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
-
   case Intrinsic::sqrt:
-    setValue(&I, DAG.getNode(ISD::FSQRT, dl,
-                             getValue(I.getOperand(1)).getValueType(),
-                             getValue(I.getOperand(1))));
+    Res = DAG.getNode(ISD::FSQRT, dl,
+                      getValue(I.getOperand(1)).getValueType(),
+                      getValue(I.getOperand(1)));
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::powi:
-    setValue(&I, DAG.getNode(ISD::FPOWI, dl,
-                             getValue(I.getOperand(1)).getValueType(),
-                             getValue(I.getOperand(1)),
-                             getValue(I.getOperand(2))));
+    Res = ExpandPowI(dl, getValue(I.getOperand(1)), getValue(I.getOperand(2)),
+                     DAG);
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::sin:
-    setValue(&I, DAG.getNode(ISD::FSIN, dl,
-                             getValue(I.getOperand(1)).getValueType(),
-                             getValue(I.getOperand(1))));
+    Res = DAG.getNode(ISD::FSIN, dl,
+                      getValue(I.getOperand(1)).getValueType(),
+                      getValue(I.getOperand(1)));
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::cos:
-    setValue(&I, DAG.getNode(ISD::FCOS, dl,
-                             getValue(I.getOperand(1)).getValueType(),
-                             getValue(I.getOperand(1))));
+    Res = DAG.getNode(ISD::FCOS, dl,
+                      getValue(I.getOperand(1)).getValueType(),
+                      getValue(I.getOperand(1)));
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::log:
     visitLog(I);
@@ -3835,55 +4637,74 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     return 0;
   case Intrinsic::pcmarker: {
     SDValue Tmp = getValue(I.getOperand(1));
-    DAG.setRoot(DAG.getNode(ISD::PCMARKER, dl, MVT::Other, getRoot(), Tmp));
+    Res = DAG.getNode(ISD::PCMARKER, dl, MVT::Other, getRoot(), Tmp);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::readcyclecounter: {
     SDValue Op = getRoot();
-    SDValue Tmp = DAG.getNode(ISD::READCYCLECOUNTER, dl,
-                              DAG.getVTList(MVT::i64, MVT::Other),
-                              &Op, 1);
-    setValue(&I, Tmp);
-    DAG.setRoot(Tmp.getValue(1));
+    Res = DAG.getNode(ISD::READCYCLECOUNTER, dl,
+                      DAG.getVTList(MVT::i64, MVT::Other),
+                      &Op, 1);
+    setValue(&I, Res);
+    DAG.setRoot(Res.getValue(1));
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::bswap:
-    setValue(&I, DAG.getNode(ISD::BSWAP, dl,
-                             getValue(I.getOperand(1)).getValueType(),
-                             getValue(I.getOperand(1))));
+    Res = DAG.getNode(ISD::BSWAP, dl,
+                      getValue(I.getOperand(1)).getValueType(),
+                      getValue(I.getOperand(1)));
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::cttz: {
     SDValue Arg = getValue(I.getOperand(1));
     EVT Ty = Arg.getValueType();
-    SDValue result = DAG.getNode(ISD::CTTZ, dl, Ty, Arg);
-    setValue(&I, result);
+    Res = DAG.getNode(ISD::CTTZ, dl, Ty, Arg);
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::ctlz: {
     SDValue Arg = getValue(I.getOperand(1));
     EVT Ty = Arg.getValueType();
-    SDValue result = DAG.getNode(ISD::CTLZ, dl, Ty, Arg);
-    setValue(&I, result);
+    Res = DAG.getNode(ISD::CTLZ, dl, Ty, Arg);
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::ctpop: {
     SDValue Arg = getValue(I.getOperand(1));
     EVT Ty = Arg.getValueType();
-    SDValue result = DAG.getNode(ISD::CTPOP, dl, Ty, Arg);
-    setValue(&I, result);
+    Res = DAG.getNode(ISD::CTPOP, dl, Ty, Arg);
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::stacksave: {
     SDValue Op = getRoot();
-    SDValue Tmp = DAG.getNode(ISD::STACKSAVE, dl,
-              DAG.getVTList(TLI.getPointerTy(), MVT::Other), &Op, 1);
-    setValue(&I, Tmp);
-    DAG.setRoot(Tmp.getValue(1));
+    Res = DAG.getNode(ISD::STACKSAVE, dl,
+                      DAG.getVTList(TLI.getPointerTy(), MVT::Other), &Op, 1);
+    setValue(&I, Res);
+    DAG.setRoot(Res.getValue(1));
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::stackrestore: {
-    SDValue Tmp = getValue(I.getOperand(1));
-    DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, dl, MVT::Other, getRoot(), Tmp));
+    Res = getValue(I.getOperand(1));
+    Res = DAG.getNode(ISD::STACKRESTORE, dl, MVT::Other, getRoot(), Res);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::stackprotector: {
@@ -3901,11 +4722,13 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
 
     // Store the stack protector onto the stack.
-    SDValue Result = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN,
-                                  PseudoSourceValue::getFixedStack(FI),
-                                  0, true);
-    setValue(&I, Result);
-    DAG.setRoot(Result);
+    Res = DAG.getStore(getRoot(), getCurDebugLoc(), Src, FIN,
+                       PseudoSourceValue::getFixedStack(FI),
+                       0, true);
+    setValue(&I, Res);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::objectsize: {
@@ -3917,10 +4740,14 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     SDValue Arg = getValue(I.getOperand(0));
     EVT Ty = Arg.getValueType();
 
-    if (CI->getZExtValue() < 2)
-      setValue(&I, DAG.getConstant(-1ULL, Ty));
+    if (CI->getZExtValue() == 0)
+      Res = DAG.getConstant(-1ULL, Ty);
     else
-      setValue(&I, DAG.getConstant(0, Ty));
+      Res = DAG.getConstant(0, Ty);
+
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::var_annotation:
@@ -3938,15 +4765,16 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     Ops[4] = DAG.getSrcValue(I.getOperand(1));
     Ops[5] = DAG.getSrcValue(F);
 
-    SDValue Tmp = DAG.getNode(ISD::TRAMPOLINE, dl,
-                              DAG.getVTList(TLI.getPointerTy(), MVT::Other),
-                              Ops, 6);
+    Res = DAG.getNode(ISD::TRAMPOLINE, dl,
+                      DAG.getVTList(TLI.getPointerTy(), MVT::Other),
+                      Ops, 6);
 
-    setValue(&I, Tmp);
-    DAG.setRoot(Tmp.getValue(1));
+    setValue(&I, Res);
+    DAG.setRoot(Res.getValue(1));
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
-
   case Intrinsic::gcroot:
     if (GFI) {
       Value *Alloca = I.getOperand(1);
@@ -3956,22 +4784,22 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
       GFI->addStackRoot(FI->getIndex(), TypeMap);
     }
     return 0;
-
   case Intrinsic::gcread:
   case Intrinsic::gcwrite:
     llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
     return 0;
-
-  case Intrinsic::flt_rounds: {
-    setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32));
+  case Intrinsic::flt_rounds:
+    Res = DAG.getNode(ISD::FLT_ROUNDS_, dl, MVT::i32);
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
-  }
-
-  case Intrinsic::trap: {
-    DAG.setRoot(DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot()));
+  case Intrinsic::trap:
+    Res = DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot());
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
-  }
-
   case Intrinsic::uadd_with_overflow:
     return implVisitAluOverflow(I, ISD::UADDO);
   case Intrinsic::sadd_with_overflow:
@@ -3991,7 +4819,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     Ops[1] = getValue(I.getOperand(1));
     Ops[2] = getValue(I.getOperand(2));
     Ops[3] = getValue(I.getOperand(3));
-    DAG.setRoot(DAG.getNode(ISD::PREFETCH, dl, MVT::Other, &Ops[0], 4));
+    Res = DAG.getNode(ISD::PREFETCH, dl, MVT::Other, &Ops[0], 4);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
 
@@ -4001,7 +4832,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
     for (int x = 1; x < 6; ++x)
       Ops[x] = getValue(I.getOperand(x));
 
-    DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6));
+    Res = DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, &Ops[0], 6);
+    DAG.setRoot(Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::atomic_cmp_swap: {
@@ -4016,6 +4850,8 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
                     I.getOperand(1));
     setValue(&I, L);
     DAG.setRoot(L.getValue(1));
+    if (DisableScheduling)
+      DAG.AssignOrdering(L.getNode(), SDNodeOrder);
     return 0;
   }
   case Intrinsic::atomic_load_add:
@@ -4044,7 +4880,10 @@ SelectionDAGBuilder::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) {
   case Intrinsic::invariant_start:
   case Intrinsic::lifetime_start:
     // Discard region information.
-    setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
+    Res = DAG.getUNDEF(TLI.getPointerTy());
+    setValue(&I, Res);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Res.getNode(), SDNodeOrder);
     return 0;
   case Intrinsic::invariant_end:
   case Intrinsic::lifetime_end:
@@ -4143,11 +4982,10 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee,
   SmallVector<EVT, 4> OutVTs;
   SmallVector<ISD::ArgFlagsTy, 4> OutsFlags;
   SmallVector<uint64_t, 4> Offsets;
-  getReturnInfo(RetTy, CS.getAttributes().getRetAttributes(), 
-    OutVTs, OutsFlags, TLI, &Offsets);
-  
+  getReturnInfo(RetTy, CS.getAttributes().getRetAttributes(),
+                OutVTs, OutsFlags, TLI, &Offsets);
 
-  bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(), 
+  bool CanLowerReturn = TLI.CanLowerReturn(CS.getCallingConv(),
                         FTy->isVarArg(), OutVTs, OutsFlags, DAG);
 
   SDValue DemoteStackSlot;
@@ -4219,14 +5057,16 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee,
                     CS.getCallingConv(),
                     isTailCall,
                     !CS.getInstruction()->use_empty(),
-                    Callee, Args, DAG, getCurDebugLoc());
+                    Callee, Args, DAG, getCurDebugLoc(), SDNodeOrder);
   assert((isTailCall || Result.second.getNode()) &&
          "Non-null chain expected with non-tail call!");
   assert((Result.second.getNode() || !Result.first.getNode()) &&
          "Null value expected with tail call!");
-  if (Result.first.getNode())
+  if (Result.first.getNode()) {
     setValue(CS.getInstruction(), Result.first);
-  else if (!CanLowerReturn && Result.second.getNode()) {
+    if (DisableScheduling)
+      DAG.AssignOrdering(Result.first.getNode(), SDNodeOrder);
+  } else if (!CanLowerReturn && Result.second.getNode()) {
     // The instruction result is the result of loading from the
     // hidden sret parameter.
     SmallVector<EVT, 1> PVTs;
@@ -4240,27 +5080,40 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee,
     SmallVector<SDValue, 4> Chains(NumValues);
 
     for (unsigned i = 0; i < NumValues; ++i) {
+      SDValue Add = DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT,
+                                DemoteStackSlot,
+                                DAG.getConstant(Offsets[i], PtrVT));
       SDValue L = DAG.getLoad(OutVTs[i], getCurDebugLoc(), Result.second,
-        DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, DemoteStackSlot,
-        DAG.getConstant(Offsets[i], PtrVT)),
-        NULL, Offsets[i], false, 1);
+                              Add, NULL, Offsets[i], false, 1);
       Values[i] = L;
       Chains[i] = L.getValue(1);
     }
+
     SDValue Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(),
                                 MVT::Other, &Chains[0], NumValues);
     PendingLoads.push_back(Chain);
 
-    setValue(CS.getInstruction(), DAG.getNode(ISD::MERGE_VALUES,
-             getCurDebugLoc(), DAG.getVTList(&OutVTs[0], NumValues),
-             &Values[0], NumValues));
+    SDValue MV = DAG.getNode(ISD::MERGE_VALUES,
+                             getCurDebugLoc(),
+                             DAG.getVTList(&OutVTs[0], NumValues),
+                             &Values[0], NumValues);
+    setValue(CS.getInstruction(), MV);
+
+    if (DisableScheduling) {
+      DAG.AssignOrdering(Chain.getNode(), SDNodeOrder);
+      DAG.AssignOrdering(MV.getNode(), SDNodeOrder);
+    }
   }
-  // As a special case, a null chain means that a tail call has
-  // been emitted and the DAG root is already updated.
-  if (Result.second.getNode())
+
+  // As a special case, a null chain means that a tail call has been emitted and
+  // the DAG root is already updated.
+  if (Result.second.getNode()) {
     DAG.setRoot(Result.second);
-  else
+    if (DisableScheduling)
+      DAG.AssignOrdering(Result.second.getNode(), SDNodeOrder);
+  } else {
     HasTailCall = true;
+  }
 
   if (LandingPad && MMI) {
     // Insert a label at the end of the invoke call to mark the try range.  This
@@ -4274,6 +5127,140 @@ void SelectionDAGBuilder::LowerCallTo(CallSite CS, SDValue Callee,
   }
 }
 
+/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
+/// value is equal or not-equal to zero.
+static bool IsOnlyUsedInZeroEqualityComparison(Value *V) {
+  for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
+       UI != E; ++UI) {
+    if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI))
+      if (IC->isEquality())
+        if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
+          if (C->isNullValue())
+            continue;
+    // Unknown instruction.
+    return false;
+  }
+  return true;
+}
+
+static SDValue getMemCmpLoad(Value *PtrVal, MVT LoadVT, const Type *LoadTy,
+                             SelectionDAGBuilder &Builder) {
+
+  // Check to see if this load can be trivially constant folded, e.g. if the
+  // input is from a string literal.
+  if (Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
+    // Cast pointer to the type we really want to load.
+    LoadInput = ConstantExpr::getBitCast(LoadInput,
+                                         PointerType::getUnqual(LoadTy));
+
+    if (Constant *LoadCst = ConstantFoldLoadFromConstPtr(LoadInput, Builder.TD))
+      return Builder.getValue(LoadCst);
+  }
+
+  // Otherwise, we have to emit the load.  If the pointer is to unfoldable but
+  // still constant memory, the input chain can be the entry node.
+  SDValue Root;
+  bool ConstantMemory = false;
+
+  // Do not serialize (non-volatile) loads of constant memory with anything.
+  if (Builder.AA->pointsToConstantMemory(PtrVal)) {
+    Root = Builder.DAG.getEntryNode();
+    ConstantMemory = true;
+  } else {
+    // Do not serialize non-volatile loads against each other.
+    Root = Builder.DAG.getRoot();
+  }
+
+  SDValue Ptr = Builder.getValue(PtrVal);
+  SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurDebugLoc(), Root,
+                                        Ptr, PtrVal /*SrcValue*/, 0/*SVOffset*/,
+                                        false /*volatile*/, 1 /* align=1 */);
+
+  if (!ConstantMemory)
+    Builder.PendingLoads.push_back(LoadVal.getValue(1));
+  return LoadVal;
+}
+
+
+/// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
+/// If so, return true and lower it, otherwise return false and it will be
+/// lowered like a normal call.
+bool SelectionDAGBuilder::visitMemCmpCall(CallInst &I) {
+  // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
+  if (I.getNumOperands() != 4)
+    return false;
+
+  Value *LHS = I.getOperand(1), *RHS = I.getOperand(2);
+  if (!isa<PointerType>(LHS->getType()) || !isa<PointerType>(RHS->getType()) ||
+      !isa<IntegerType>(I.getOperand(3)->getType()) ||
+      !isa<IntegerType>(I.getType()))
+    return false;
+
+  ConstantInt *Size = dyn_cast<ConstantInt>(I.getOperand(3));
+
+  // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
+  // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
+  if (Size && IsOnlyUsedInZeroEqualityComparison(&I)) {
+    bool ActuallyDoIt = true;
+    MVT LoadVT;
+    const Type *LoadTy;
+    switch (Size->getZExtValue()) {
+    default:
+      LoadVT = MVT::Other;
+      LoadTy = 0;
+      ActuallyDoIt = false;
+      break;
+    case 2:
+      LoadVT = MVT::i16;
+      LoadTy = Type::getInt16Ty(Size->getContext());
+      break;
+    case 4:
+      LoadVT = MVT::i32;
+      LoadTy = Type::getInt32Ty(Size->getContext());
+      break;
+    case 8:
+      LoadVT = MVT::i64;
+      LoadTy = Type::getInt64Ty(Size->getContext());
+      break;
+        /*
+    case 16:
+      LoadVT = MVT::v4i32;
+      LoadTy = Type::getInt32Ty(Size->getContext());
+      LoadTy = VectorType::get(LoadTy, 4);
+      break;
+         */
+    }
+
+    // This turns into unaligned loads.  We only do this if the target natively
+    // supports the MVT we'll be loading or if it is small enough (<= 4) that
+    // we'll only produce a small number of byte loads.
+
+    // Require that we can find a legal MVT, and only do this if the target
+    // supports unaligned loads of that type.  Expanding into byte loads would
+    // bloat the code.
+    if (ActuallyDoIt && Size->getZExtValue() > 4) {
+      // TODO: Handle 5 byte compare as 4-byte + 1 byte.
+      // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
+      if (!TLI.isTypeLegal(LoadVT) ||!TLI.allowsUnalignedMemoryAccesses(LoadVT))
+        ActuallyDoIt = false;
+    }
+
+    if (ActuallyDoIt) {
+      SDValue LHSVal = getMemCmpLoad(LHS, LoadVT, LoadTy, *this);
+      SDValue RHSVal = getMemCmpLoad(RHS, LoadVT, LoadTy, *this);
+
+      SDValue Res = DAG.getSetCC(getCurDebugLoc(), MVT::i1, LHSVal, RHSVal,
+                                 ISD::SETNE);
+      EVT CallVT = TLI.getValueType(I.getType(), true);
+      setValue(&I, DAG.getZExtOrTrunc(Res, getCurDebugLoc(), CallVT));
+      return true;
+    }
+  }
+
+
+  return false;
+}
+
 
 void SelectionDAGBuilder::visitCall(CallInst &I) {
   const char *RenameFn = 0;
@@ -4348,6 +5335,9 @@ void SelectionDAGBuilder::visitCall(CallInst &I) {
                                    Tmp.getValueType(), Tmp));
           return;
         }
+      } else if (Name == "memcmp") {
+        if (visitMemCmpCall(I))
+          return;
       }
     }
   } else if (isa<InlineAsm>(I.getOperand(0))) {
@@ -4361,21 +5351,19 @@ void SelectionDAGBuilder::visitCall(CallInst &I) {
   else
     Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy());
 
-  // Check if we can potentially perform a tail call. More detailed
-  // checking is be done within LowerCallTo, after more information
-  // about the call is known.
+  // Check if we can potentially perform a tail call. More detailed checking is
+  // be done within LowerCallTo, after more information about the call is known.
   bool isTailCall = PerformTailCallOpt && I.isTailCall();
 
   LowerCallTo(&I, Callee, isTailCall);
 }
 
-
 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from
 /// this value and returns the result as a ValueVT value.  This uses
 /// Chain/Flag as the input and updates them for the output Chain/Flag.
 /// If the Flag pointer is NULL, no flag is used.
 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl,
-                                      SDValue &Chain,
+                                      unsigned Order, SDValue &Chain,
                                       SDValue *Flag) const {
   // Assemble the legal parts into the final values.
   SmallVector<SDValue, 4> Values(ValueVTs.size());
@@ -4389,14 +5377,18 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl,
     Parts.resize(NumRegs);
     for (unsigned i = 0; i != NumRegs; ++i) {
       SDValue P;
-      if (Flag == 0)
+      if (Flag == 0) {
         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
-      else {
+      } else {
         P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
         *Flag = P.getValue(2);
       }
+
       Chain = P.getValue(1);
 
+      if (DisableScheduling)
+        DAG.AssignOrdering(P.getNode(), Order);
+
       // If the source register was virtual and if we know something about it,
       // add an assert node.
       if (TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) &&
@@ -4435,6 +5427,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl,
             P = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
                             RegisterVT, P, DAG.getValueType(FromVT));
 
+            if (DisableScheduling)
+              DAG.AssignOrdering(P.getNode(), Order);
           }
         }
       }
@@ -4442,15 +5436,20 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl,
       Parts[i] = P;
     }
 
-    Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
+    Values[Value] = getCopyFromParts(DAG, dl, Order, Parts.begin(),
                                      NumRegs, RegisterVT, ValueVT);
+    if (DisableScheduling)
+      DAG.AssignOrdering(Values[Value].getNode(), Order);
     Part += NumRegs;
     Parts.clear();
   }
 
-  return DAG.getNode(ISD::MERGE_VALUES, dl,
-                     DAG.getVTList(&ValueVTs[0], ValueVTs.size()),
-                     &Values[0], ValueVTs.size());
+  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
+                            DAG.getVTList(&ValueVTs[0], ValueVTs.size()),
+                            &Values[0], ValueVTs.size());
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), Order);
+  return Res;
 }
 
 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
@@ -4458,7 +5457,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, DebugLoc dl,
 /// Chain/Flag as the input and updates them for the output Chain/Flag.
 /// If the Flag pointer is NULL, no flag is used.
 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
-                                 SDValue &Chain, SDValue *Flag) const {
+                                 unsigned Order, SDValue &Chain,
+                                 SDValue *Flag) const {
   // Get the list of the values's legal parts.
   unsigned NumRegs = Regs.size();
   SmallVector<SDValue, 8> Parts(NumRegs);
@@ -4467,7 +5467,8 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
     unsigned NumParts = TLI->getNumRegisters(*DAG.getContext(), ValueVT);
     EVT RegisterVT = RegVTs[Value];
 
-    getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
+    getCopyToParts(DAG, dl, Order,
+                   Val.getValue(Val.getResNo() + Value),
                    &Parts[Part], NumParts, RegisterVT);
     Part += NumParts;
   }
@@ -4476,13 +5477,17 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
   SmallVector<SDValue, 8> Chains(NumRegs);
   for (unsigned i = 0; i != NumRegs; ++i) {
     SDValue Part;
-    if (Flag == 0)
+    if (Flag == 0) {
       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
-    else {
+    } else {
       Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
       *Flag = Part.getValue(1);
     }
+
     Chains[i] = Part.getValue(0);
+
+    if (DisableScheduling)
+      DAG.AssignOrdering(Part.getNode(), Order);
   }
 
   if (NumRegs == 1 || Flag)
@@ -4499,6 +5504,9 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
     Chain = Chains[NumRegs-1];
   else
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &Chains[0], NumRegs);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Chain.getNode(), Order);
 }
 
 /// AddInlineAsmOperands - Add this value to the specified inlineasm node
@@ -4506,20 +5514,28 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
 /// values added into it.
 void RegsForValue::AddInlineAsmOperands(unsigned Code,
                                         bool HasMatching,unsigned MatchingIdx,
-                                        SelectionDAG &DAG,
+                                        SelectionDAG &DAG, unsigned Order,
                                         std::vector<SDValue> &Ops) const {
-  EVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy();
   assert(Regs.size() < (1 << 13) && "Too many inline asm outputs!");
   unsigned Flag = Code | (Regs.size() << 3);
   if (HasMatching)
     Flag |= 0x80000000 | (MatchingIdx << 16);
-  Ops.push_back(DAG.getTargetConstant(Flag, IntPtrTy));
+  SDValue Res = DAG.getTargetConstant(Flag, MVT::i32);
+  Ops.push_back(Res);
+
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), Order);
+
   for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
     unsigned NumRegs = TLI->getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
     EVT RegisterVT = RegVTs[Value];
     for (unsigned i = 0; i != NumRegs; ++i) {
       assert(Reg < Regs.size() && "Mismatch in # registers expected");
-      Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT));
+      SDValue Res = DAG.getRegister(Regs[Reg++], RegisterVT);
+      Ops.push_back(Res);
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(Res.getNode(), Order);
     }
   }
 }
@@ -4611,7 +5627,7 @@ public:
   /// getCallOperandValEVT - Return the EVT of the Value* that this operand
   /// corresponds to.  If there is no Value* for this operand, it returns
   /// MVT::Other.
-  EVT getCallOperandValEVT(LLVMContext &Context, 
+  EVT getCallOperandValEVT(LLVMContext &Context,
                            const TargetLowering &TLI,
                            const TargetData *TD) const {
     if (CallOperandVal == 0) return MVT::Other;
@@ -4623,8 +5639,12 @@ public:
 
     // If this is an indirect operand, the operand is a pointer to the
     // accessed type.
-    if (isIndirect)
-      OpTy = cast<PointerType>(OpTy)->getElementType();
+    if (isIndirect) {
+      const llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
+      if (!PtrTy)
+        llvm_report_error("Indirect operand for inline asm not a pointer!");
+      OpTy = PtrTy->getElementType();
+    }
 
     // If OpTy is not a single value, it may be a struct/union that we
     // can tile with integers.
@@ -4663,8 +5683,8 @@ private:
 
 /// GetRegistersForValue - Assign registers (virtual or physical) for the
 /// specified operand.  We prefer to assign virtual registers, to allow the
-/// register allocator handle the assignment process.  However, if the asm uses
-/// features that we can't model on machineinstrs, we have SDISel do the
+/// register allocator to handle the assignment process.  However, if the asm
+/// uses features that we can't model on machineinstrs, we have SDISel do the
 /// allocation.  This produces generally horrible, but correct, code.
 ///
 ///   OpInfo describes the operand.
@@ -4728,12 +5748,15 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo,
         // bitcast to the corresponding integer type.  This turns an f64 value
         // into i64, which can be passed with two i32 values on a 32-bit
         // machine.
-        RegVT = EVT::getIntegerVT(Context, 
+        RegVT = EVT::getIntegerVT(Context,
                                   OpInfo.ConstraintVT.getSizeInBits());
         OpInfo.CallOperand = DAG.getNode(ISD::BIT_CONVERT, getCurDebugLoc(),
                                          RegVT, OpInfo.CallOperand);
         OpInfo.ConstraintVT = RegVT;
       }
+
+      if (DisableScheduling)
+        DAG.AssignOrdering(OpInfo.CallOperand.getNode(), SDNodeOrder);
     }
 
     NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
@@ -4770,6 +5793,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo,
         Regs.push_back(*I);
       }
     }
+
     OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT);
     const TargetRegisterInfo *TRI = DAG.getTarget().getRegisterInfo();
     OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI);
@@ -4791,7 +5815,7 @@ GetRegistersForValue(SDISelAsmOperandInfo &OpInfo,
     OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT);
     return;
   }
-  
+
   // This is a reference to a register class that doesn't directly correspond
   // to an LLVM register class.  Allocate NumRegs consecutive, available,
   // registers from the class.
@@ -4853,7 +5877,7 @@ hasInlineAsmMemConstraint(std::vector<InlineAsm::ConstraintInfo> &CInfos,
       if (CType == TargetLowering::C_Memory)
         return true;
     }
-    
+
     // Indirect operand accesses access memory.
     if (CI.isIndirect)
       return true;
@@ -4878,9 +5902,9 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
     ConstraintInfos = IA->ParseConstraints();
 
   bool hasMemory = hasInlineAsmMemConstraint(ConstraintInfos, TLI);
-  
+
   SDValue Chain, Flag;
-  
+
   // We won't need to flush pending loads if this asm doesn't touch
   // memory and is nonvolatile.
   if (hasMemory || IA->hasSideEffects())
@@ -5004,6 +6028,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
 
       // There is no longer a Value* corresponding to this operand.
       OpInfo.CallOperandVal = 0;
+
       // It is now an indirect operand.
       OpInfo.isIndirect = true;
     }
@@ -5013,8 +6038,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
     if (OpInfo.ConstraintType == TargetLowering::C_Register)
       GetRegistersForValue(OpInfo, OutputRegs, InputRegs);
   }
-  ConstraintInfos.clear();
 
+  ConstraintInfos.clear();
 
   // Second pass - Loop over all of the operands, assigning virtual or physregs
   // to register class operands.
@@ -5088,7 +6113,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
                                                2 /* REGDEF */ ,
                                                false,
                                                0,
-                                               DAG, AsmNodeOperands);
+                                               DAG, SDNodeOrder,
+                                               AsmNodeOperands);
       break;
     }
     case InlineAsm::isInput: {
@@ -5130,15 +6156,15 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
           MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
           for (unsigned i = 0, e = InlineAsm::getNumOperandRegisters(OpFlag);
                i != e; ++i)
-            MatchedRegs.Regs.
-              push_back(RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT)));
+            MatchedRegs.Regs.push_back
+              (RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT)));
 
           // Use the produced MatchedRegs object to
           MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(),
-                                    Chain, &Flag);
+                                    SDNodeOrder, Chain, &Flag);
           MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/,
                                            true, OpInfo.getMatchedOperand(),
-                                           DAG, AsmNodeOperands);
+                                           DAG, SDNodeOrder, AsmNodeOperands);
           break;
         } else {
           assert(((OpFlag & 7) == 4) && "Unknown matching constraint!");
@@ -5198,10 +6224,11 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
       }
 
       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(),
-                                        Chain, &Flag);
+                                        SDNodeOrder, Chain, &Flag);
 
       OpInfo.AssignedRegs.AddInlineAsmOperands(1/*REGUSE*/, false, 0,
-                                               DAG, AsmNodeOperands);
+                                               DAG, SDNodeOrder,
+                                               AsmNodeOperands);
       break;
     }
     case InlineAsm::isClobber: {
@@ -5209,7 +6236,8 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
       // allocator is aware that the physreg got clobbered.
       if (!OpInfo.AssignedRegs.Regs.empty())
         OpInfo.AssignedRegs.AddInlineAsmOperands(6 /* EARLYCLOBBER REGDEF */,
-                                                 false, 0, DAG,AsmNodeOperands);
+                                                 false, 0, DAG, SDNodeOrder,
+                                                 AsmNodeOperands);
       break;
     }
     }
@@ -5228,7 +6256,7 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
   // and set it as the value of the call.
   if (!RetValRegs.Regs.empty()) {
     SDValue Val = RetValRegs.getCopyFromRegs(DAG, getCurDebugLoc(),
-                                             Chain, &Flag);
+                                             SDNodeOrder, Chain, &Flag);
 
     // FIXME: Why don't we do this for inline asms with MRVs?
     if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
@@ -5268,21 +6296,25 @@ void SelectionDAGBuilder::visitInlineAsm(CallSite CS) {
     RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
     Value *Ptr = IndirectStoresToEmit[i].second;
     SDValue OutVal = OutRegs.getCopyFromRegs(DAG, getCurDebugLoc(),
-                                             Chain, &Flag);
+                                             SDNodeOrder, Chain, &Flag);
     StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
 
   }
 
   // Emit the non-flagged stores from the physregs.
   SmallVector<SDValue, 8> OutChains;
-  for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i)
-    OutChains.push_back(DAG.getStore(Chain, getCurDebugLoc(),
-                                    StoresToEmit[i].first,
-                                    getValue(StoresToEmit[i].second),
-                                    StoresToEmit[i].second, 0));
+  for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
+    SDValue Val = DAG.getStore(Chain, getCurDebugLoc(),
+                               StoresToEmit[i].first,
+                               getValue(StoresToEmit[i].second),
+                               StoresToEmit[i].second, 0);
+    OutChains.push_back(Val);
+  }
+
   if (!OutChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other,
                         &OutChains[0], OutChains.size());
+
   DAG.setRoot(Chain);
 }
 
@@ -5328,8 +6360,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
                             CallingConv::ID CallConv, bool isTailCall,
                             bool isReturnValueUsed,
                             SDValue Callee,
-                            ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl) {
-
+                            ArgListTy &Args, SelectionDAG &DAG, DebugLoc dl,
+                            unsigned Order) {
   assert((!isTailCall || PerformTailCallOpt) &&
          "isTailCall set when tail-call optimizations are disabled!");
 
@@ -5383,7 +6415,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
       else if (Args[i].isZExt)
         ExtendKind = ISD::ZERO_EXTEND;
 
-      getCopyToParts(DAG, dl, Op, &Parts[0], NumParts, PartVT, ExtendKind);
+      getCopyToParts(DAG, dl, Order, Op, &Parts[0], NumParts,
+                     PartVT, ExtendKind);
 
       for (unsigned j = 0; j != NumParts; ++j) {
         // if it isn't first piece, alignment must be 1
@@ -5444,6 +6477,9 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
                  "LowerCall emitted a value with the wrong type!");
         });
 
+  if (DisableScheduling)
+    DAG.AssignOrdering(Chain.getNode(), Order);
+
   // For a tail call, the return value is merely live-out and there aren't
   // any nodes in the DAG representing it. Return a special value to
   // indicate that a tail call has been emitted and no more Instructions
@@ -5468,9 +6504,11 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
     unsigned NumRegs = getNumRegisters(RetTy->getContext(), VT);
 
     SDValue ReturnValue =
-      getCopyFromParts(DAG, dl, &InVals[CurReg], NumRegs, RegisterVT, VT,
-                       AssertOp);
+      getCopyFromParts(DAG, dl, Order, &InVals[CurReg], NumRegs,
+                       RegisterVT, VT, AssertOp);
     ReturnValues.push_back(ReturnValue);
+    if (DisableScheduling)
+      DAG.AssignOrdering(ReturnValue.getNode(), Order);
     CurReg += NumRegs;
   }
 
@@ -5483,7 +6521,8 @@ TargetLowering::LowerCallTo(SDValue Chain, const Type *RetTy,
   SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
                             DAG.getVTList(&RetTys[0], RetTys.size()),
                             &ReturnValues[0], ReturnValues.size());
-
+  if (DisableScheduling)
+    DAG.AssignOrdering(Res.getNode(), Order);
   return std::make_pair(Res, Chain);
 }
 
@@ -5500,7 +6539,6 @@ SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   return SDValue();
 }
 
-
 void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) {
   SDValue Op = getValue(V);
   assert((Op.getOpcode() != ISD::CopyFromReg ||
@@ -5510,7 +6548,7 @@ void SelectionDAGBuilder::CopyValueToVirtualRegister(Value *V, unsigned Reg) {
 
   RegsForValue RFV(V->getContext(), TLI, Reg, V->getType());
   SDValue Chain = DAG.getEntryNode();
-  RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), Chain, 0);
+  RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), SDNodeOrder, Chain, 0);
   PendingExports.push_back(Chain);
 }
 
@@ -5528,12 +6566,12 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) {
   // Check whether the function can return without sret-demotion.
   SmallVector<EVT, 4> OutVTs;
   SmallVector<ISD::ArgFlagsTy, 4> OutsFlags;
-  getReturnInfo(F.getReturnType(), F.getAttributes().getRetAttributes(), 
+  getReturnInfo(F.getReturnType(), F.getAttributes().getRetAttributes(),
                 OutVTs, OutsFlags, TLI);
   FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo();
 
-  FLI.CanLowerReturn = TLI.CanLowerReturn(F.getCallingConv(), F.isVarArg(), 
-    OutVTs, OutsFlags, DAG);
+  FLI.CanLowerReturn = TLI.CanLowerReturn(F.getCallingConv(), F.isVarArg(),
+                                          OutVTs, OutsFlags, DAG);
   if (!FLI.CanLowerReturn) {
     // Put in an sret pointer parameter before all the other parameters.
     SmallVector<EVT, 1> ValueVTs;
@@ -5613,12 +6651,14 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) {
          "LowerFormalArguments didn't return a valid chain!");
   assert(InVals.size() == Ins.size() &&
          "LowerFormalArguments didn't emit the correct number of values!");
-  DEBUG(for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
-          assert(InVals[i].getNode() &&
-                 "LowerFormalArguments emitted a null value!");
-          assert(Ins[i].VT == InVals[i].getValueType() &&
-                 "LowerFormalArguments emitted a value with the wrong type!");
-        });
+  DEBUG({
+      for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
+        assert(InVals[i].getNode() &&
+               "LowerFormalArguments emitted a null value!");
+        assert(Ins[i].VT == InVals[i].getValueType() &&
+               "LowerFormalArguments emitted a value with the wrong type!");
+      }
+    });
 
   // Update the DAG with the new chain value resulting from argument lowering.
   DAG.setRoot(NewRoot);
@@ -5634,20 +6674,22 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) {
     EVT VT = ValueVTs[0];
     EVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT);
     ISD::NodeType AssertOp = ISD::DELETED_NODE;
-    SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT,
-                                        VT, AssertOp);
+    SDValue ArgValue = getCopyFromParts(DAG, dl, 0, &InVals[0], 1,
+                                        RegVT, VT, AssertOp);
 
     MachineFunction& MF = SDB->DAG.getMachineFunction();
     MachineRegisterInfo& RegInfo = MF.getRegInfo();
     unsigned SRetReg = RegInfo.createVirtualRegister(TLI.getRegClassFor(RegVT));
     FLI.DemoteRegister = SRetReg;
-    NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurDebugLoc(), SRetReg, ArgValue);
+    NewRoot = SDB->DAG.getCopyToReg(NewRoot, SDB->getCurDebugLoc(),
+                                    SRetReg, ArgValue);
     DAG.setRoot(NewRoot);
-    
+
     // i indexes lowered arguments.  Bump it past the hidden sret argument.
     // Idx indexes LLVM arguments.  Don't touch it.
     ++i;
   }
+
   for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E;
       ++I, ++Idx) {
     SmallVector<SDValue, 4> ArgValues;
@@ -5666,19 +6708,25 @@ void SelectionDAGISel::LowerArguments(BasicBlock *LLVMBB) {
         else if (F.paramHasAttr(Idx, Attribute::ZExt))
           AssertOp = ISD::AssertZext;
 
-        ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
-                                             PartVT, VT, AssertOp));
+        ArgValues.push_back(getCopyFromParts(DAG, dl, 0, &InVals[i],
+                                             NumParts, PartVT, VT,
+                                             AssertOp));
       }
+
       i += NumParts;
     }
+
     if (!I->use_empty()) {
-      SDB->setValue(I, DAG.getMergeValues(&ArgValues[0], NumValues,
-                                          SDB->getCurDebugLoc()));
+      SDValue Res = DAG.getMergeValues(&ArgValues[0], NumValues,
+                                       SDB->getCurDebugLoc());
+      SDB->setValue(I, Res);
+
       // If this argument is live outside of the entry block, insert a copy from
       // whereever we got it to the vreg that other BB's will reference it as.
       SDB->CopyToExportRegsIfNeeded(I);
     }
   }
+
   assert(i == InVals.size() && "Argument register count mismatch!");
 
   // Finally, if the target has anything special to do, allow it to do so.
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 244f9b5019e1..88a2017b4746 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -91,11 +91,13 @@ class SelectionDAGBuilder {
 
   DenseMap<const Value*, SDValue> NodeMap;
 
+public:
   /// PendingLoads - Loads are not emitted to the program immediately.  We bunch
   /// them up and then emit token factor nodes when possible.  This allows us to
   /// get simple disambiguation between loads without worrying about alias
   /// analysis.
   SmallVector<SDValue, 8> PendingLoads;
+private:
 
   /// PendingExports - CopyToReg nodes that copy values to virtual registers
   /// for export to other blocks need to be emitted before any terminator
@@ -104,6 +106,10 @@ class SelectionDAGBuilder {
   /// instructions.
   SmallVector<SDValue, 8> PendingExports;
 
+  /// SDNodeOrder - A unique monotonically increasing number used to order the
+  /// SDNodes we create.
+  unsigned SDNodeOrder;
+
   /// Case - A struct to record the Value for a switch case, and the
   /// case's target basic block.
   struct Case {
@@ -300,7 +306,7 @@ public:
   SelectionDAGBuilder(SelectionDAG &dag, TargetLowering &tli,
                       FunctionLoweringInfo &funcinfo,
                       CodeGenOpt::Level ol)
-    : CurDebugLoc(DebugLoc::getUnknownLoc()), 
+    : CurDebugLoc(DebugLoc::getUnknownLoc()), SDNodeOrder(0),
       TLI(tli), DAG(dag), FuncInfo(funcinfo), OptLevel(ol),
       HasTailCall(false),
       Context(dag.getContext()) {
@@ -332,6 +338,8 @@ public:
   DebugLoc getCurDebugLoc() const { return CurDebugLoc; }
   void setCurDebugLoc(DebugLoc dl) { CurDebugLoc = dl; }
 
+  unsigned getSDNodeOrder() const { return SDNodeOrder; }
+
   void CopyValueToVirtualRegister(Value *V, unsigned Reg);
 
   void visit(Instruction &I);
@@ -455,6 +463,8 @@ private:
   void visitStore(StoreInst &I);
   void visitPHI(PHINode &I) { } // PHI nodes are handled specially.
   void visitCall(CallInst &I);
+  bool visitMemCmpCall(CallInst &I);
+  
   void visitInlineAsm(CallSite CS);
   const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic);
   void visitTargetIntrinsic(CallInst &I, unsigned Intrinsic);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index a640c7dc2089..05669c0ec9a9 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -362,32 +362,29 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
 
 /// SetDebugLoc - Update MF's and SDB's DebugLocs if debug information is
 /// attached with this instruction.
-static void SetDebugLoc(unsigned MDDbgKind,
-                        MetadataContext &TheMetadata,
-                        Instruction *I,
+static void SetDebugLoc(unsigned MDDbgKind, Instruction *I,
                         SelectionDAGBuilder *SDB,
-                        FastISel *FastIS,
-                        MachineFunction *MF) {
-  if (!isa<DbgInfoIntrinsic>(I)) 
-    if (MDNode *Dbg = TheMetadata.getMD(MDDbgKind, I)) {
-      DILocation DILoc(Dbg);
-      DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo());
-
-      SDB->setCurDebugLoc(Loc);
-
-      if (FastIS)
-        FastIS->setCurDebugLoc(Loc);
-
-      // If the function doesn't have a default debug location yet, set
-      // it. This is kind of a hack.
-      if (MF->getDefaultDebugLoc().isUnknown())
-        MF->setDefaultDebugLoc(Loc);
-    }
+                        FastISel *FastIS, MachineFunction *MF) {
+  if (isa<DbgInfoIntrinsic>(I)) return;
+  
+  if (MDNode *Dbg = I->getMetadata(MDDbgKind)) {
+    DILocation DILoc(Dbg);
+    DebugLoc Loc = ExtractDebugLocation(DILoc, MF->getDebugLocInfo());
+
+    SDB->setCurDebugLoc(Loc);
+
+    if (FastIS)
+      FastIS->setCurDebugLoc(Loc);
+
+    // If the function doesn't have a default debug location yet, set
+    // it. This is kind of a hack.
+    if (MF->getDefaultDebugLoc().isUnknown())
+      MF->setDefaultDebugLoc(Loc);
+  }
 }
 
 /// ResetDebugLoc - Set MF's and SDB's DebugLocs to Unknown.
-static void ResetDebugLoc(SelectionDAGBuilder *SDB,
-                          FastISel *FastIS) {
+static void ResetDebugLoc(SelectionDAGBuilder *SDB, FastISel *FastIS) {
   SDB->setCurDebugLoc(DebugLoc::getUnknownLoc());
   if (FastIS)
     FastIS->setCurDebugLoc(DebugLoc::getUnknownLoc());
@@ -398,14 +395,12 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
                                         BasicBlock::iterator End,
                                         bool &HadTailCall) {
   SDB->setCurrentBasicBlock(BB);
-  MetadataContext &TheMetadata = LLVMBB->getParent()->getContext().getMetadata();
-  unsigned MDDbgKind = TheMetadata.getMDKind("dbg");
+  unsigned MDDbgKind = LLVMBB->getContext().getMDKindID("dbg");
 
   // Lower all of the non-terminator instructions. If a call is emitted
   // as a tail call, cease emitting nodes for this block.
   for (BasicBlock::iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {
-    if (MDDbgKind)
-      SetDebugLoc(MDDbgKind, TheMetadata, I, SDB, 0, MF);
+    SetDebugLoc(MDDbgKind, I, SDB, 0, MF);
 
     if (!isa<TerminatorInst>(I)) {
       SDB->visit(*I);
@@ -428,7 +423,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
       HandlePHINodesInSuccessorBlocks(LLVMBB);
 
       // Lower the terminator after the copies are emitted.
-      SetDebugLoc(MDDbgKind, TheMetadata, LLVMBB->getTerminator(), SDB, 0, MF);
+      SetDebugLoc(MDDbgKind, LLVMBB->getTerminator(), SDB, 0, MF);
       SDB->visit(*LLVMBB->getTerminator());
       ResetDebugLoc(SDB, 0);
     }
@@ -567,9 +562,9 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
   if (Changed) {
     if (TimePassesIsEnabled) {
       NamedRegionTimer T("Type Legalization 2", GroupName);
-      Changed = CurDAG->LegalizeTypes();
+      CurDAG->LegalizeTypes();
     } else {
-      Changed = CurDAG->LegalizeTypes();
+      CurDAG->LegalizeTypes();
     }
 
     if (ViewDAGCombineLT)
@@ -680,8 +675,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn,
 #endif
                                 );
 
-  MetadataContext &TheMetadata = Fn.getContext().getMetadata();
-  unsigned MDDbgKind = TheMetadata.getMDKind("dbg");
+  unsigned MDDbgKind = Fn.getContext().getMDKindID("dbg");
 
   // Iterate over all basic blocks in the function.
   for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
@@ -779,8 +773,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn,
             break;
           }
 
-        if (MDDbgKind)
-          SetDebugLoc(MDDbgKind, TheMetadata, BI, SDB, FastIS, &MF);
+        SetDebugLoc(MDDbgKind, BI, SDB, FastIS, &MF);
 
         // First try normal tablegen-generated "fast" selection.
         if (FastIS->SelectInstruction(BI)) {
@@ -1182,9 +1175,8 @@ SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops) {
       }
 
       // Add this to the output node.
-      EVT IntPtrTy = TLI.getPointerTy();
       Ops.push_back(CurDAG->getTargetConstant(4/*MEM*/ | (SelOps.size()<< 3),
-                                              IntPtrTy));
+                                              MVT::i32));
       Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
       i += 2;
     }
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 10261691517a..d9a5a13666b3 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -713,6 +713,10 @@ MVT::SimpleValueType TargetLowering::getSetCCResultType(EVT VT) const {
   return PointerTy.SimpleTy;
 }
 
+MVT::SimpleValueType TargetLowering::getCmpLibcallReturnType() const {
+  return MVT::i32; // return the default value
+}
+
 /// getVectorTypeBreakdown - Vector types are broken down into some number of
 /// legal first class types.  For example, MVT::v8f32 maps to 2 MVT::v4f32
 /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
diff --git a/lib/CodeGen/SimpleRegisterCoalescing.cpp b/lib/CodeGen/SimpleRegisterCoalescing.cpp
index ed407eb40cb2..6314331482d1 100644
--- a/lib/CodeGen/SimpleRegisterCoalescing.cpp
+++ b/lib/CodeGen/SimpleRegisterCoalescing.cpp
@@ -1065,7 +1065,7 @@ SimpleRegisterCoalescing::isWinToJoinVRWithSrcPhysReg(MachineInstr *CopyMI,
       if (SuccMBB == CopyMBB)
         continue;
       if (DstInt.overlaps(li_->getMBBStartIdx(SuccMBB),
-                      li_->getMBBEndIdx(SuccMBB).getNextIndex().getBaseIndex()))
+                          li_->getMBBEndIdx(SuccMBB)))
         return false;
     }
   }
@@ -1121,7 +1121,7 @@ SimpleRegisterCoalescing::isWinToJoinVRWithDstPhysReg(MachineInstr *CopyMI,
       if (PredMBB == SMBB)
         continue;
       if (SrcInt.overlaps(li_->getMBBStartIdx(PredMBB),
-                      li_->getMBBEndIdx(PredMBB).getNextIndex().getBaseIndex()))
+                          li_->getMBBEndIdx(PredMBB)))
         return false;
     }
   }
@@ -2246,8 +2246,9 @@ SimpleRegisterCoalescing::JoinIntervals(LiveInterval &LHS, LiveInterval &RHS,
         continue;
 
       // Figure out the value # from the RHS.
-      LHSValsDefinedFromRHS[VNI]=
-        RHS.getLiveRangeContaining(VNI->def.getPrevSlot())->valno;
+      LiveRange *lr = RHS.getLiveRangeContaining(VNI->def.getPrevSlot());
+      assert(lr && "Cannot find live range");
+      LHSValsDefinedFromRHS[VNI] = lr->valno;
     }
 
     // Loop over the value numbers of the RHS, seeing if any are defined from
@@ -2264,8 +2265,9 @@ SimpleRegisterCoalescing::JoinIntervals(LiveInterval &LHS, LiveInterval &RHS,
         continue;
 
       // Figure out the value # from the LHS.
-      RHSValsDefinedFromLHS[VNI]=
-        LHS.getLiveRangeContaining(VNI->def.getPrevSlot())->valno;
+      LiveRange *lr = LHS.getLiveRangeContaining(VNI->def.getPrevSlot());
+      assert(lr && "Cannot find live range");
+      RHSValsDefinedFromLHS[VNI] = lr->valno;
     }
 
     LHSValNoAssignments.resize(LHS.getNumValNums(), -1);
diff --git a/lib/CodeGen/SimpleRegisterCoalescing.h b/lib/CodeGen/SimpleRegisterCoalescing.h
index 605a7404c200..f668064ab08e 100644
--- a/lib/CodeGen/SimpleRegisterCoalescing.h
+++ b/lib/CodeGen/SimpleRegisterCoalescing.h
@@ -33,7 +33,7 @@ namespace llvm {
     MachineInstr *MI;
     unsigned LoopDepth;
     CopyRec(MachineInstr *mi, unsigned depth)
-      : MI(mi), LoopDepth(depth) {};
+      : MI(mi), LoopDepth(depth) {}
   };
 
   class SimpleRegisterCoalescing : public MachineFunctionPass,
@@ -85,7 +85,7 @@ namespace llvm {
     bool coalesceFunction(MachineFunction &mf, RegallocQuery &) {
       // This runs as an independent pass, so don't do anything.
       return false;
-    };
+    }
 
     /// print - Implement the dump method.
     virtual void print(raw_ostream &O, const Module* = 0) const;
diff --git a/lib/CodeGen/SlotIndexes.cpp b/lib/CodeGen/SlotIndexes.cpp
index f85384bc5c77..782af124a664 100644
--- a/lib/CodeGen/SlotIndexes.cpp
+++ b/lib/CodeGen/SlotIndexes.cpp
@@ -92,13 +92,14 @@ bool SlotIndexes::runOnMachineFunction(MachineFunction &fn) {
   functionSize = 0;
   unsigned index = 0;
 
+  push_back(createEntry(0, index));
+
   // Iterate over the the function.
   for (MachineFunction::iterator mbbItr = mf->begin(), mbbEnd = mf->end();
        mbbItr != mbbEnd; ++mbbItr) {
     MachineBasicBlock *mbb = &*mbbItr;
 
     // Insert an index for the MBB start.
-    push_back(createEntry(0, index));
     SlotIndex blockStartIndex(back(), SlotIndex::LOAD);
 
     index += SlotIndex::NUM;
@@ -137,16 +138,16 @@ bool SlotIndexes::runOnMachineFunction(MachineFunction &fn) {
       index += SlotIndex::NUM;
     }
 
-    SlotIndex blockEndIndex(back(), SlotIndex::STORE);
+    // One blank instruction at the end.
+    push_back(createEntry(0, index));    
+
+    SlotIndex blockEndIndex(back(), SlotIndex::LOAD);
     mbb2IdxMap.insert(
       std::make_pair(mbb, std::make_pair(blockStartIndex, blockEndIndex)));
 
     idx2MBBMap.push_back(IdxMBBPair(blockStartIndex, mbb));
   }
 
-  // One blank instruction at the end.
-  push_back(createEntry(0, index));
-
   // Sort the Idx2MBBMap
   std::sort(idx2MBBMap.begin(), idx2MBBMap.end(), Idx2MBBCompare());
 
diff --git a/lib/CodeGen/Spiller.cpp b/lib/CodeGen/Spiller.cpp
index bc246c14b77c..bec92949ccd6 100644
--- a/lib/CodeGen/Spiller.cpp
+++ b/lib/CodeGen/Spiller.cpp
@@ -486,10 +486,10 @@ private:
         SlotIndex newKillRangeEnd = oldKillRange->end;
         oldKillRange->end = copyIdx.getDefIndex();
 
-        if (newKillRangeEnd != lis->getMBBEndIdx(killMBB).getNextSlot()) {
-          assert(newKillRangeEnd > lis->getMBBEndIdx(killMBB).getNextSlot() &&
+        if (newKillRangeEnd != lis->getMBBEndIdx(killMBB)) {
+          assert(newKillRangeEnd > lis->getMBBEndIdx(killMBB) &&
                  "PHI kill range doesn't reach kill-block end. Not sane.");
-          newLI->addRange(LiveRange(lis->getMBBEndIdx(killMBB).getNextSlot(),
+          newLI->addRange(LiveRange(lis->getMBBEndIdx(killMBB),
                                     newKillRangeEnd, newVNI));
         }
 
@@ -500,7 +500,7 @@ private:
         newKillVNI->addKill(lis->getMBBTerminatorGap(killMBB));
         newKillVNI->setHasPHIKill(true);
         li->addRange(LiveRange(copyIdx.getDefIndex(),
-                               lis->getMBBEndIdx(killMBB).getNextSlot(),
+                               lis->getMBBEndIdx(killMBB),
                                newKillVNI));
       }