mirror of
https://github.com/llvm/llvm-project.git
synced 2025-04-28 15:06:08 +00:00
901c5be524
Treat a defined register as fully live "at" the instruction and update maximum pressure accordingly. Fixes #3786.
659 lines
21 KiB
C++
659 lines
21 KiB
C++
//===- GCNRegPressure.cpp -------------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
///
|
|
/// \file
|
|
/// This file implements the GCNRegPressure class.
|
|
///
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "GCNRegPressure.h"
|
|
#include "AMDGPU.h"
|
|
#include "llvm/CodeGen/RegisterPressure.h"
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "machine-scheduler"
|
|
|
|
bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,
|
|
const GCNRPTracker::LiveRegSet &S2) {
|
|
if (S1.size() != S2.size())
|
|
return false;
|
|
|
|
for (const auto &P : S1) {
|
|
auto I = S2.find(P.first);
|
|
if (I == S2.end() || I->second != P.second)
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
// GCNRegPressure
|
|
|
|
unsigned GCNRegPressure::getRegKind(Register Reg,
|
|
const MachineRegisterInfo &MRI) {
|
|
assert(Reg.isVirtual());
|
|
const auto RC = MRI.getRegClass(Reg);
|
|
auto STI = static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
|
|
return STI->isSGPRClass(RC)
|
|
? (STI->getRegSizeInBits(*RC) == 32 ? SGPR32 : SGPR_TUPLE)
|
|
: STI->isAGPRClass(RC)
|
|
? (STI->getRegSizeInBits(*RC) == 32 ? AGPR32 : AGPR_TUPLE)
|
|
: (STI->getRegSizeInBits(*RC) == 32 ? VGPR32 : VGPR_TUPLE);
|
|
}
|
|
|
|
void GCNRegPressure::inc(unsigned Reg,
|
|
LaneBitmask PrevMask,
|
|
LaneBitmask NewMask,
|
|
const MachineRegisterInfo &MRI) {
|
|
if (SIRegisterInfo::getNumCoveredRegs(NewMask) ==
|
|
SIRegisterInfo::getNumCoveredRegs(PrevMask))
|
|
return;
|
|
|
|
int Sign = 1;
|
|
if (NewMask < PrevMask) {
|
|
std::swap(NewMask, PrevMask);
|
|
Sign = -1;
|
|
}
|
|
|
|
switch (auto Kind = getRegKind(Reg, MRI)) {
|
|
case SGPR32:
|
|
case VGPR32:
|
|
case AGPR32:
|
|
Value[Kind] += Sign;
|
|
break;
|
|
|
|
case SGPR_TUPLE:
|
|
case VGPR_TUPLE:
|
|
case AGPR_TUPLE:
|
|
assert(PrevMask < NewMask);
|
|
|
|
Value[Kind == SGPR_TUPLE ? SGPR32 : Kind == AGPR_TUPLE ? AGPR32 : VGPR32] +=
|
|
Sign * SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
|
|
|
|
if (PrevMask.none()) {
|
|
assert(NewMask.any());
|
|
const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
|
|
Value[Kind] +=
|
|
Sign * TRI->getRegClassWeight(MRI.getRegClass(Reg)).RegWeight;
|
|
}
|
|
break;
|
|
|
|
default: llvm_unreachable("Unknown register kind");
|
|
}
|
|
}
|
|
|
|
bool GCNRegPressure::less(const GCNSubtarget &ST,
|
|
const GCNRegPressure& O,
|
|
unsigned MaxOccupancy) const {
|
|
const auto SGPROcc = std::min(MaxOccupancy,
|
|
ST.getOccupancyWithNumSGPRs(getSGPRNum()));
|
|
const auto VGPROcc =
|
|
std::min(MaxOccupancy,
|
|
ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts())));
|
|
const auto OtherSGPROcc = std::min(MaxOccupancy,
|
|
ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));
|
|
const auto OtherVGPROcc =
|
|
std::min(MaxOccupancy,
|
|
ST.getOccupancyWithNumVGPRs(O.getVGPRNum(ST.hasGFX90AInsts())));
|
|
|
|
const auto Occ = std::min(SGPROcc, VGPROcc);
|
|
const auto OtherOcc = std::min(OtherSGPROcc, OtherVGPROcc);
|
|
if (Occ != OtherOcc)
|
|
return Occ > OtherOcc;
|
|
|
|
bool SGPRImportant = SGPROcc < VGPROcc;
|
|
const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;
|
|
|
|
// if both pressures disagree on what is more important compare vgprs
|
|
if (SGPRImportant != OtherSGPRImportant) {
|
|
SGPRImportant = false;
|
|
}
|
|
|
|
// compare large regs pressure
|
|
bool SGPRFirst = SGPRImportant;
|
|
for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {
|
|
if (SGPRFirst) {
|
|
auto SW = getSGPRTuplesWeight();
|
|
auto OtherSW = O.getSGPRTuplesWeight();
|
|
if (SW != OtherSW)
|
|
return SW < OtherSW;
|
|
} else {
|
|
auto VW = getVGPRTuplesWeight();
|
|
auto OtherVW = O.getVGPRTuplesWeight();
|
|
if (VW != OtherVW)
|
|
return VW < OtherVW;
|
|
}
|
|
}
|
|
return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):
|
|
(getVGPRNum(ST.hasGFX90AInsts()) <
|
|
O.getVGPRNum(ST.hasGFX90AInsts()));
|
|
}
|
|
|
|
Printable llvm::print(const GCNRegPressure &RP, const GCNSubtarget *ST) {
|
|
return Printable([&RP, ST](raw_ostream &OS) {
|
|
OS << "VGPRs: " << RP.Value[GCNRegPressure::VGPR32] << ' '
|
|
<< "AGPRs: " << RP.getAGPRNum();
|
|
if (ST)
|
|
OS << "(O"
|
|
<< ST->getOccupancyWithNumVGPRs(RP.getVGPRNum(ST->hasGFX90AInsts()))
|
|
<< ')';
|
|
OS << ", SGPRs: " << RP.getSGPRNum();
|
|
if (ST)
|
|
OS << "(O" << ST->getOccupancyWithNumSGPRs(RP.getSGPRNum()) << ')';
|
|
OS << ", LVGPR WT: " << RP.getVGPRTuplesWeight()
|
|
<< ", LSGPR WT: " << RP.getSGPRTuplesWeight();
|
|
if (ST)
|
|
OS << " -> Occ: " << RP.getOccupancy(*ST);
|
|
OS << '\n';
|
|
});
|
|
}
|
|
|
|
static LaneBitmask getDefRegMask(const MachineOperand &MO,
|
|
const MachineRegisterInfo &MRI) {
|
|
assert(MO.isDef() && MO.isReg() && MO.getReg().isVirtual());
|
|
|
|
// We don't rely on read-undef flag because in case of tentative schedule
|
|
// tracking it isn't set correctly yet. This works correctly however since
|
|
// use mask has been tracked before using LIS.
|
|
return MO.getSubReg() == 0 ?
|
|
MRI.getMaxLaneMaskForVReg(MO.getReg()) :
|
|
MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());
|
|
}
|
|
|
|
static void
|
|
collectVirtualRegUses(SmallVectorImpl<RegisterMaskPair> &RegMaskPairs,
|
|
const MachineInstr &MI, const LiveIntervals &LIS,
|
|
const MachineRegisterInfo &MRI) {
|
|
SlotIndex InstrSI;
|
|
for (const auto &MO : MI.operands()) {
|
|
if (!MO.isReg() || !MO.getReg().isVirtual())
|
|
continue;
|
|
if (!MO.isUse() || !MO.readsReg())
|
|
continue;
|
|
|
|
Register Reg = MO.getReg();
|
|
if (llvm::any_of(RegMaskPairs, [Reg](const RegisterMaskPair &RM) {
|
|
return RM.RegUnit == Reg;
|
|
}))
|
|
continue;
|
|
|
|
LaneBitmask UseMask;
|
|
auto &LI = LIS.getInterval(Reg);
|
|
if (!LI.hasSubRanges())
|
|
UseMask = MRI.getMaxLaneMaskForVReg(Reg);
|
|
else {
|
|
// For a tentative schedule LIS isn't updated yet but livemask should
|
|
// remain the same on any schedule. Subreg defs can be reordered but they
|
|
// all must dominate uses anyway.
|
|
if (!InstrSI)
|
|
InstrSI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex();
|
|
UseMask = getLiveLaneMask(LI, InstrSI, MRI);
|
|
}
|
|
|
|
RegMaskPairs.emplace_back(Reg, UseMask);
|
|
}
|
|
}
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
// GCNRPTracker
|
|
|
|
LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,
|
|
const LiveIntervals &LIS,
|
|
const MachineRegisterInfo &MRI) {
|
|
return getLiveLaneMask(LIS.getInterval(Reg), SI, MRI);
|
|
}
|
|
|
|
LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,
|
|
const MachineRegisterInfo &MRI) {
|
|
LaneBitmask LiveMask;
|
|
if (LI.hasSubRanges()) {
|
|
for (const auto &S : LI.subranges())
|
|
if (S.liveAt(SI)) {
|
|
LiveMask |= S.LaneMask;
|
|
assert(LiveMask == (LiveMask & MRI.getMaxLaneMaskForVReg(LI.reg())));
|
|
}
|
|
} else if (LI.liveAt(SI)) {
|
|
LiveMask = MRI.getMaxLaneMaskForVReg(LI.reg());
|
|
}
|
|
return LiveMask;
|
|
}
|
|
|
|
GCNRPTracker::LiveRegSet llvm::getLiveRegs(SlotIndex SI,
|
|
const LiveIntervals &LIS,
|
|
const MachineRegisterInfo &MRI) {
|
|
GCNRPTracker::LiveRegSet LiveRegs;
|
|
for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
|
|
auto Reg = Register::index2VirtReg(I);
|
|
if (!LIS.hasInterval(Reg))
|
|
continue;
|
|
auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
|
|
if (LiveMask.any())
|
|
LiveRegs[Reg] = LiveMask;
|
|
}
|
|
return LiveRegs;
|
|
}
|
|
|
|
void GCNRPTracker::reset(const MachineInstr &MI,
|
|
const LiveRegSet *LiveRegsCopy,
|
|
bool After) {
|
|
const MachineFunction &MF = *MI.getMF();
|
|
MRI = &MF.getRegInfo();
|
|
if (LiveRegsCopy) {
|
|
if (&LiveRegs != LiveRegsCopy)
|
|
LiveRegs = *LiveRegsCopy;
|
|
} else {
|
|
LiveRegs = After ? getLiveRegsAfter(MI, LIS)
|
|
: getLiveRegsBefore(MI, LIS);
|
|
}
|
|
|
|
MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
// GCNUpwardRPTracker
|
|
|
|
void GCNUpwardRPTracker::reset(const MachineRegisterInfo &MRI_,
|
|
const LiveRegSet &LiveRegs_) {
|
|
MRI = &MRI_;
|
|
LiveRegs = LiveRegs_;
|
|
LastTrackedMI = nullptr;
|
|
MaxPressure = CurPressure = getRegPressure(MRI_, LiveRegs_);
|
|
}
|
|
|
|
void GCNUpwardRPTracker::recede(const MachineInstr &MI) {
|
|
assert(MRI && "call reset first");
|
|
|
|
LastTrackedMI = &MI;
|
|
|
|
if (MI.isDebugInstr())
|
|
return;
|
|
|
|
// Kill all defs.
|
|
GCNRegPressure DefPressure, ECDefPressure;
|
|
bool HasECDefs = false;
|
|
for (const MachineOperand &MO : MI.all_defs()) {
|
|
if (!MO.getReg().isVirtual())
|
|
continue;
|
|
|
|
Register Reg = MO.getReg();
|
|
LaneBitmask DefMask = getDefRegMask(MO, *MRI);
|
|
|
|
// Treat a def as fully live at the moment of definition: keep a record.
|
|
if (MO.isEarlyClobber()) {
|
|
ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
|
|
HasECDefs = true;
|
|
} else
|
|
DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);
|
|
|
|
auto I = LiveRegs.find(Reg);
|
|
if (I == LiveRegs.end())
|
|
continue;
|
|
|
|
LaneBitmask &LiveMask = I->second;
|
|
LaneBitmask PrevMask = LiveMask;
|
|
LiveMask &= ~DefMask;
|
|
CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
|
|
if (LiveMask.none())
|
|
LiveRegs.erase(I);
|
|
}
|
|
|
|
// Update MaxPressure with defs pressure.
|
|
DefPressure += CurPressure;
|
|
if (HasECDefs)
|
|
DefPressure += ECDefPressure;
|
|
MaxPressure = max(DefPressure, MaxPressure);
|
|
|
|
// Make uses alive.
|
|
SmallVector<RegisterMaskPair, 8> RegUses;
|
|
collectVirtualRegUses(RegUses, MI, LIS, *MRI);
|
|
for (const RegisterMaskPair &U : RegUses) {
|
|
LaneBitmask &LiveMask = LiveRegs[U.RegUnit];
|
|
LaneBitmask PrevMask = LiveMask;
|
|
LiveMask |= U.LaneMask;
|
|
CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI);
|
|
}
|
|
|
|
// Update MaxPressure with uses plus early-clobber defs pressure.
|
|
MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure)
|
|
: max(CurPressure, MaxPressure);
|
|
|
|
assert(CurPressure == getRegPressure(*MRI, LiveRegs));
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
// GCNDownwardRPTracker
|
|
|
|
bool GCNDownwardRPTracker::reset(const MachineInstr &MI,
|
|
const LiveRegSet *LiveRegsCopy) {
|
|
MRI = &MI.getParent()->getParent()->getRegInfo();
|
|
LastTrackedMI = nullptr;
|
|
MBBEnd = MI.getParent()->end();
|
|
NextMI = &MI;
|
|
NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
|
|
if (NextMI == MBBEnd)
|
|
return false;
|
|
GCNRPTracker::reset(*NextMI, LiveRegsCopy, false);
|
|
return true;
|
|
}
|
|
|
|
bool GCNDownwardRPTracker::advanceBeforeNext() {
|
|
assert(MRI && "call reset first");
|
|
if (!LastTrackedMI)
|
|
return NextMI == MBBEnd;
|
|
|
|
assert(NextMI == MBBEnd || !NextMI->isDebugInstr());
|
|
|
|
SlotIndex SI = NextMI == MBBEnd
|
|
? LIS.getInstructionIndex(*LastTrackedMI).getDeadSlot()
|
|
: LIS.getInstructionIndex(*NextMI).getBaseIndex();
|
|
assert(SI.isValid());
|
|
|
|
// Remove dead registers or mask bits.
|
|
SmallSet<Register, 8> SeenRegs;
|
|
for (auto &MO : LastTrackedMI->operands()) {
|
|
if (!MO.isReg() || !MO.getReg().isVirtual())
|
|
continue;
|
|
if (MO.isUse() && !MO.readsReg())
|
|
continue;
|
|
if (!SeenRegs.insert(MO.getReg()).second)
|
|
continue;
|
|
const LiveInterval &LI = LIS.getInterval(MO.getReg());
|
|
if (LI.hasSubRanges()) {
|
|
auto It = LiveRegs.end();
|
|
for (const auto &S : LI.subranges()) {
|
|
if (!S.liveAt(SI)) {
|
|
if (It == LiveRegs.end()) {
|
|
It = LiveRegs.find(MO.getReg());
|
|
if (It == LiveRegs.end())
|
|
llvm_unreachable("register isn't live");
|
|
}
|
|
auto PrevMask = It->second;
|
|
It->second &= ~S.LaneMask;
|
|
CurPressure.inc(MO.getReg(), PrevMask, It->second, *MRI);
|
|
}
|
|
}
|
|
if (It != LiveRegs.end() && It->second.none())
|
|
LiveRegs.erase(It);
|
|
} else if (!LI.liveAt(SI)) {
|
|
auto It = LiveRegs.find(MO.getReg());
|
|
if (It == LiveRegs.end())
|
|
llvm_unreachable("register isn't live");
|
|
CurPressure.inc(MO.getReg(), It->second, LaneBitmask::getNone(), *MRI);
|
|
LiveRegs.erase(It);
|
|
}
|
|
}
|
|
|
|
MaxPressure = max(MaxPressure, CurPressure);
|
|
|
|
LastTrackedMI = nullptr;
|
|
|
|
return NextMI == MBBEnd;
|
|
}
|
|
|
|
void GCNDownwardRPTracker::advanceToNext() {
|
|
LastTrackedMI = &*NextMI++;
|
|
NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);
|
|
|
|
// Add new registers or mask bits.
|
|
for (const auto &MO : LastTrackedMI->all_defs()) {
|
|
Register Reg = MO.getReg();
|
|
if (!Reg.isVirtual())
|
|
continue;
|
|
auto &LiveMask = LiveRegs[Reg];
|
|
auto PrevMask = LiveMask;
|
|
LiveMask |= getDefRegMask(MO, *MRI);
|
|
CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);
|
|
}
|
|
|
|
MaxPressure = max(MaxPressure, CurPressure);
|
|
}
|
|
|
|
bool GCNDownwardRPTracker::advance() {
|
|
if (NextMI == MBBEnd)
|
|
return false;
|
|
advanceBeforeNext();
|
|
advanceToNext();
|
|
return true;
|
|
}
|
|
|
|
bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {
|
|
while (NextMI != End)
|
|
if (!advance()) return false;
|
|
return true;
|
|
}
|
|
|
|
bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,
|
|
MachineBasicBlock::const_iterator End,
|
|
const LiveRegSet *LiveRegsCopy) {
|
|
reset(*Begin, LiveRegsCopy);
|
|
return advance(End);
|
|
}
|
|
|
|
Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,
|
|
const GCNRPTracker::LiveRegSet &TrackedLR,
|
|
const TargetRegisterInfo *TRI, StringRef Pfx) {
|
|
return Printable([&LISLR, &TrackedLR, TRI, Pfx](raw_ostream &OS) {
|
|
for (auto const &P : TrackedLR) {
|
|
auto I = LISLR.find(P.first);
|
|
if (I == LISLR.end()) {
|
|
OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)
|
|
<< " isn't found in LIS reported set\n";
|
|
} else if (I->second != P.second) {
|
|
OS << Pfx << printReg(P.first, TRI)
|
|
<< " masks doesn't match: LIS reported " << PrintLaneMask(I->second)
|
|
<< ", tracked " << PrintLaneMask(P.second) << '\n';
|
|
}
|
|
}
|
|
for (auto const &P : LISLR) {
|
|
auto I = TrackedLR.find(P.first);
|
|
if (I == TrackedLR.end()) {
|
|
OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)
|
|
<< " isn't found in tracked set\n";
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
bool GCNUpwardRPTracker::isValid() const {
|
|
const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex();
|
|
const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI);
|
|
const auto &TrackedLR = LiveRegs;
|
|
|
|
if (!isEqual(LISLR, TrackedLR)) {
|
|
dbgs() << "\nGCNUpwardRPTracker error: Tracked and"
|
|
" LIS reported livesets mismatch:\n"
|
|
<< print(LISLR, *MRI);
|
|
reportMismatch(LISLR, TrackedLR, MRI->getTargetRegisterInfo());
|
|
return false;
|
|
}
|
|
|
|
auto LISPressure = getRegPressure(*MRI, LISLR);
|
|
if (LISPressure != CurPressure) {
|
|
dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "
|
|
<< print(CurPressure) << "LIS rpt: " << print(LISPressure);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
Printable llvm::print(const GCNRPTracker::LiveRegSet &LiveRegs,
|
|
const MachineRegisterInfo &MRI) {
|
|
return Printable([&LiveRegs, &MRI](raw_ostream &OS) {
|
|
const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
|
|
for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
|
|
Register Reg = Register::index2VirtReg(I);
|
|
auto It = LiveRegs.find(Reg);
|
|
if (It != LiveRegs.end() && It->second.any())
|
|
OS << ' ' << printVRegOrUnit(Reg, TRI) << ':'
|
|
<< PrintLaneMask(It->second);
|
|
}
|
|
OS << '\n';
|
|
});
|
|
}
|
|
|
|
void GCNRegPressure::dump() const { dbgs() << print(*this); }
|
|
|
|
static cl::opt<bool> UseDownwardTracker(
|
|
"amdgpu-print-rp-downward",
|
|
cl::desc("Use GCNDownwardRPTracker for GCNRegPressurePrinter pass"),
|
|
cl::init(false), cl::Hidden);
|
|
|
|
char llvm::GCNRegPressurePrinter::ID = 0;
|
|
char &llvm::GCNRegPressurePrinterID = GCNRegPressurePrinter::ID;
|
|
|
|
INITIALIZE_PASS(GCNRegPressurePrinter, "amdgpu-print-rp", "", true, true)
|
|
|
|
// Return lanemask of Reg's subregs that are live-through at [Begin, End] and
|
|
// are fully covered by Mask.
|
|
static LaneBitmask
|
|
getRegLiveThroughMask(const MachineRegisterInfo &MRI, const LiveIntervals &LIS,
|
|
Register Reg, SlotIndex Begin, SlotIndex End,
|
|
LaneBitmask Mask = LaneBitmask::getAll()) {
|
|
|
|
auto IsInOneSegment = [Begin, End](const LiveRange &LR) -> bool {
|
|
auto *Segment = LR.getSegmentContaining(Begin);
|
|
return Segment && Segment->contains(End);
|
|
};
|
|
|
|
LaneBitmask LiveThroughMask;
|
|
const LiveInterval &LI = LIS.getInterval(Reg);
|
|
if (LI.hasSubRanges()) {
|
|
for (auto &SR : LI.subranges()) {
|
|
if ((SR.LaneMask & Mask) == SR.LaneMask && IsInOneSegment(SR))
|
|
LiveThroughMask |= SR.LaneMask;
|
|
}
|
|
} else {
|
|
LaneBitmask RegMask = MRI.getMaxLaneMaskForVReg(Reg);
|
|
if ((RegMask & Mask) == RegMask && IsInOneSegment(LI))
|
|
LiveThroughMask = RegMask;
|
|
}
|
|
|
|
return LiveThroughMask;
|
|
}
|
|
|
|
bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {
|
|
const MachineRegisterInfo &MRI = MF.getRegInfo();
|
|
const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
|
|
const LiveIntervals &LIS = getAnalysis<LiveIntervals>();
|
|
|
|
auto &OS = dbgs();
|
|
|
|
// Leading spaces are important for YAML syntax.
|
|
#define PFX " "
|
|
|
|
OS << "---\nname: " << MF.getName() << "\nbody: |\n";
|
|
|
|
auto printRP = [](const GCNRegPressure &RP) {
|
|
return Printable([&RP](raw_ostream &OS) {
|
|
OS << format(PFX " %-5d", RP.getSGPRNum())
|
|
<< format(" %-5d", RP.getVGPRNum(false));
|
|
});
|
|
};
|
|
|
|
auto ReportLISMismatchIfAny = [&](const GCNRPTracker::LiveRegSet &TrackedLR,
|
|
const GCNRPTracker::LiveRegSet &LISLR) {
|
|
if (LISLR != TrackedLR) {
|
|
OS << PFX " mis LIS: " << llvm::print(LISLR, MRI)
|
|
<< reportMismatch(LISLR, TrackedLR, TRI, PFX " ");
|
|
}
|
|
};
|
|
|
|
// Register pressure before and at an instruction (in program order).
|
|
SmallVector<std::pair<GCNRegPressure, GCNRegPressure>, 16> RP;
|
|
|
|
for (auto &MBB : MF) {
|
|
RP.clear();
|
|
RP.reserve(MBB.size());
|
|
|
|
OS << PFX;
|
|
MBB.printName(OS);
|
|
OS << ":\n";
|
|
|
|
SlotIndex MBBStartSlot = LIS.getSlotIndexes()->getMBBStartIdx(&MBB);
|
|
SlotIndex MBBEndSlot = LIS.getSlotIndexes()->getMBBEndIdx(&MBB);
|
|
|
|
GCNRPTracker::LiveRegSet LiveIn, LiveOut;
|
|
GCNRegPressure RPAtMBBEnd;
|
|
|
|
if (UseDownwardTracker) {
|
|
if (MBB.empty()) {
|
|
LiveIn = LiveOut = getLiveRegs(MBBStartSlot, LIS, MRI);
|
|
RPAtMBBEnd = getRegPressure(MRI, LiveIn);
|
|
} else {
|
|
GCNDownwardRPTracker RPT(LIS);
|
|
RPT.reset(MBB.front());
|
|
|
|
LiveIn = RPT.getLiveRegs();
|
|
|
|
while (!RPT.advanceBeforeNext()) {
|
|
GCNRegPressure RPBeforeMI = RPT.getPressure();
|
|
RPT.advanceToNext();
|
|
RP.emplace_back(RPBeforeMI, RPT.getPressure());
|
|
}
|
|
|
|
LiveOut = RPT.getLiveRegs();
|
|
RPAtMBBEnd = RPT.getPressure();
|
|
}
|
|
} else {
|
|
GCNUpwardRPTracker RPT(LIS);
|
|
RPT.reset(MRI, MBBEndSlot);
|
|
|
|
LiveOut = RPT.getLiveRegs();
|
|
RPAtMBBEnd = RPT.getPressure();
|
|
|
|
for (auto &MI : reverse(MBB)) {
|
|
RPT.resetMaxPressure();
|
|
RPT.recede(MI);
|
|
if (!MI.isDebugInstr())
|
|
RP.emplace_back(RPT.getPressure(), RPT.getMaxPressure());
|
|
}
|
|
|
|
LiveIn = RPT.getLiveRegs();
|
|
}
|
|
|
|
OS << PFX " Live-in: " << llvm::print(LiveIn, MRI);
|
|
if (!UseDownwardTracker)
|
|
ReportLISMismatchIfAny(LiveIn, getLiveRegs(MBBStartSlot, LIS, MRI));
|
|
|
|
OS << PFX " SGPR VGPR\n";
|
|
int I = 0;
|
|
for (auto &MI : MBB) {
|
|
if (!MI.isDebugInstr()) {
|
|
auto &[RPBeforeInstr, RPAtInstr] =
|
|
RP[UseDownwardTracker ? I : (RP.size() - 1 - I)];
|
|
++I;
|
|
OS << printRP(RPBeforeInstr) << '\n' << printRP(RPAtInstr) << " ";
|
|
} else
|
|
OS << PFX " ";
|
|
MI.print(OS);
|
|
}
|
|
OS << printRP(RPAtMBBEnd) << '\n';
|
|
|
|
OS << PFX " Live-out:" << llvm::print(LiveOut, MRI);
|
|
if (UseDownwardTracker)
|
|
ReportLISMismatchIfAny(LiveOut, getLiveRegs(MBBEndSlot, LIS, MRI));
|
|
|
|
GCNRPTracker::LiveRegSet LiveThrough;
|
|
for (auto [Reg, Mask] : LiveIn) {
|
|
LaneBitmask MaskIntersection = Mask & LiveOut.lookup(Reg);
|
|
if (MaskIntersection.any()) {
|
|
LaneBitmask LTMask = getRegLiveThroughMask(
|
|
MRI, LIS, Reg, MBBStartSlot, MBBEndSlot, MaskIntersection);
|
|
if (LTMask.any())
|
|
LiveThrough[Reg] = LTMask;
|
|
}
|
|
}
|
|
OS << PFX " Live-thr:" << llvm::print(LiveThrough, MRI);
|
|
OS << printRP(getRegPressure(MRI, LiveThrough)) << '\n';
|
|
}
|
|
OS << "...\n";
|
|
return false;
|
|
|
|
#undef PFX
|
|
} |