Taking LP counts into account for FT count inference

(cherry picked from FBD28110493)

bolt/BinaryBasicBlock.h

@@ -75,6 +75,7 @@ class BinaryBasicBlock {
   /// CFG information.
   std::vector<BinaryBasicBlock *> Predecessors;
   std::vector<BinaryBasicBlock *> Successors;
+  std::set<BinaryBasicBlock *> LandingPads;
 
   struct BinaryBranchInfo {
     uint64_t Count;
@@ -236,6 +237,11 @@ public:
                     uint64_t Count = 0,
                     uint64_t MispredictedCount = 0);
 
+  /// Adds block to landing pad list.
+  void addLandingPad(BinaryBasicBlock *LPBlock) {
+    LandingPads.insert(LPBlock);
+  }
+
   /// Remove /p Succ basic block from the list of successors. Update the
   /// list of predecessors of /p Succ and update branch info.
   void removeSuccessor(BinaryBasicBlock *Succ);
@@ -314,7 +320,7 @@ public:
                      const MCSymbol *&FBB,
                      MCInst *&CondBranch,
                      MCInst *&UncondBranch);
-  
+
 private:
 
   /// Adds predecessor to the BB. Most likely you don't need to call this.

bolt/BinaryFunction.cpp

@@ -450,7 +450,6 @@ bool BinaryFunction::disassemble(ArrayRef<uint8_t> FunctionData,
         // or a recursive call.
         bool IsCall = MIA->isCall(Instruction);
         bool IsCondBranch = MIA->isConditionalBranch(Instruction);
-        bool IsInvoke = MIA->isInvoke(Instruction);
         MCSymbol *TargetSymbol{nullptr};
         uint64_t TargetOffset{0};
 
@@ -519,11 +518,10 @@ bool BinaryFunction::disassemble(ArrayRef<uint8_t> FunctionData,
           // Add local branch info.
           LocalBranches.push_back({Offset, TargetOffset});
         }
-        if (IsCondBranch /*|| IsInvoke*/) {
+        if (IsCondBranch) {
           // Add fallthrough branch info.
           FTBranches.push_back({Offset, Offset + Size});
         }
-
       } else {
         // Should be an indirect call or an indirect branch. Bail out on the
         // latter case.
@@ -715,9 +713,18 @@ bool BinaryFunction::buildCFG() {
       CFIOffset = getSize();
     addCFIPlaceholders(CFIOffset, InsertBB);
 
+    // Store info about associated landing pad.
+    if (MIA->isInvoke(InstrInfo.second)) {
+      const MCSymbol *LP;
+      uint64_t Action;
+      std::tie(LP, Action) = MIA->getEHInfo(InstrInfo.second);
+      if (LP) {
+        LPToBBIndex[LP].push_back(getIndex(InsertBB));
+      }
+    }
+
     // How well do we detect tail calls here?
-    if (MIA->isTerminator(InstrInfo.second) /*||
-        MIA->isInvoke(InstrInfo.second)*/) {
+    if (MIA->isTerminator(InstrInfo.second)) {
       PrevBB = InsertBB;
       InsertBB = nullptr;
     }
@@ -802,8 +809,6 @@ bool BinaryFunction::buildCFG() {
     if (BB.succ_size() == 0) {
       IsPrevFT = MIA->isTerminator(*LastInstIter) ? false : true;
     } else if (BB.succ_size() == 1) {
-      /*assert(!MIA->isInvoke(*LastInstIter) &&
-             "found throw with assocoated local branch");*/
       IsPrevFT =  MIA->isConditionalBranch(*LastInstIter) ? true : false;
     } else {
       // Ends with 2 branches, with an indirect jump or it is a conditional
@@ -819,6 +824,17 @@ bool BinaryFunction::buildCFG() {
     DEBUG(dbgs() << "last block was marked as a fall-through\n");
   }
 
+  // Add associated landing pad blocks to each basic block.
+  for (auto &BB : BasicBlocks) {
+    if (LandingPads.find(BB.getLabel()) != LandingPads.end()) {
+      MCSymbol *LP = BB.getLabel();
+      for (unsigned I : LPToBBIndex.at(LP)) {
+        BinaryBasicBlock *ThrowBB = getBasicBlockAtIndex(I);
+        ThrowBB->addLandingPad(&BB);
+      }
+    }
+  }
+
   // Infer frequency for non-taken branches
   if (ExecutionCount != COUNT_NO_PROFILE && !BranchDataOrErr.getError()) {
     inferFallThroughCounts();
@@ -833,6 +849,7 @@ bool BinaryFunction::buildCFG() {
   clearLabels();
   clearLocalBranches();
   clearFTBranches();
+  clearLPToBBIndex();
 
   // Update the state.
   CurrentState = State::CFG;
@@ -899,14 +916,25 @@ void BinaryFunction::inferFallThroughCounts() {
         ReportedBranches += SuccCount.Count;
     }
 
+    // Calculate frequency of throws from this node according to LBR data
+    // for branching into associated landing pads. Since it is possible
+    // for a landing pad to be associated with more than one basic blocks,
+    // we may overestimate the frequency of throws for such blocks.
+    uint64_t ReportedThrows = 0;
+    for (BinaryBasicBlock *LP: CurBB.LandingPads) {
+      ReportedThrows += LP->ExecutionCount;
+    }
+
+    uint64_t TotalReportedJumps = ReportedBranches + ReportedThrows;
+
     // Infer the frequency of the fall-through edge, representing not taking the
     // branch
     uint64_t Inferred = 0;
-    if (BBExecCount > ReportedBranches)
-      Inferred = BBExecCount - ReportedBranches;
+    if (BBExecCount > TotalReportedJumps)
+      Inferred = BBExecCount - TotalReportedJumps;
 
     DEBUG({
-      if (BBExecCount < ReportedBranches)
+      if (BBExecCount < TotalReportedJumps)
         dbgs()
             << "BOLT-WARNING: Fall-through inference is slightly inconsistent. "
                "exec frequency is less than the outgoing edges frequency ("

bolt/BinaryFunction.h

@@ -209,6 +209,13 @@ private:
     return *this;
   }
 
+  /// Release memory taken by landing pad info.
+  BinaryFunction &clearLPToBBIndex() {
+    LandingPadsMapType TempMap;
+    LPToBBIndex.swap(TempMap);
+    return *this;
+  }
+
   BinaryFunction &updateState(BinaryFunction::State State) {
     CurrentState = State;
     return *this;
@@ -230,6 +237,11 @@ private:
   LocalBranchesListType LocalBranches;
   LocalBranchesListType FTBranches;
 
+  /// Storage for all landing pads and their corresponding invokes.
+  using LandingPadsMapType = std::map<const MCSymbol *,
+                                      std::vector<unsigned> >;
+  LandingPadsMapType LPToBBIndex;
+
   /// Map offset in the function to a local label.
   using LabelsMapType = std::map<uint32_t, MCSymbol *>;
   LabelsMapType Labels;
@@ -390,6 +402,10 @@ public:
     return &BasicBlocks.at(Index);
   }
 
+  BinaryBasicBlock * getBasicBlockAtIndex(unsigned Index) {
+    return &BasicBlocks.at(Index);
+  }
+
   /// Return the name of the function as extracted from the binary file.
   StringRef getName() const {
     return Name;