From 37e39667ccdbf96645515ff6ad8f86d499f472e8 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 19 Nov 2024 23:16:35 -0800 Subject: [PATCH] [ELF] Make ThunkCreator take ownership of thunks This removes many SpecificAlloc instantiations and makes my lld (x86-64 Release+Assertions) smaller by ~36k. --- lld/ELF/Relocations.cpp | 17 ++++--- lld/ELF/Relocations.h | 11 +++-- lld/ELF/Thunks.cpp | 104 +++++++++++++++++++++------------------- lld/ELF/Thunks.h | 5 +- lld/ELF/Writer.cpp | 5 +- 5 files changed, 79 insertions(+), 63 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 1f3c5708f925..e110adead5ad 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -1907,6 +1907,10 @@ static void forEachInputSectionDescription( } } +ThunkCreator::ThunkCreator(Ctx &ctx) : ctx(ctx) {} + +ThunkCreator::~ThunkCreator() {} + // Thunk Implementation // // Thunks (sometimes called stubs, veneers or branch islands) are small pieces @@ -2212,7 +2216,7 @@ static bool isThunkSectionCompatible(InputSection *source, std::pair ThunkCreator::getThunk(InputSection *isec, Relocation &rel, uint64_t src) { - std::vector *thunkVec = nullptr; + SmallVector, 0> *thunkVec = nullptr; // Arm and Thumb have a PC Bias of 8 and 4 respectively, this is cancelled // out in the relocation addend. We compensate for the PC bias so that // an Arm and Thumb relocation to the same destination get the same keyAddend, @@ -2233,17 +2237,16 @@ std::pair ThunkCreator::getThunk(InputSection *isec, thunkVec = &thunkedSymbols[{rel.sym, keyAddend}]; // Check existing Thunks for Sym to see if they can be reused - for (Thunk *t : *thunkVec) + for (auto &t : *thunkVec) if (isThunkSectionCompatible(isec, t->getThunkTargetSym()->section) && t->isCompatibleWith(*isec, rel) && ctx.target->inBranchRange(rel.type, src, t->getThunkTargetSym()->getVA(ctx, -pcBias))) - return std::make_pair(t, false); + return std::make_pair(t.get(), false); // No existing compatible Thunk in range, create a new one - Thunk *t = addThunk(ctx, *isec, rel); - thunkVec->push_back(t); - return std::make_pair(t, true); + thunkVec->push_back(addThunk(ctx, *isec, rel)); + return std::make_pair(thunkVec->back().get(), true); } std::pair ThunkCreator::getSyntheticLandingPad(Defined &d, @@ -2252,7 +2255,7 @@ std::pair ThunkCreator::getSyntheticLandingPad(Defined &d, {{d.section, d.value}, a}, nullptr); if (isNew) it->second = addLandingPadThunk(ctx, d, a); - return {it->second, isNew}; + return {it->second.get(), isNew}; } // Return true if the relocation target is an in range Thunk. diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h index f800b5db61e9..041bd4804858 100644 --- a/lld/ELF/Relocations.h +++ b/lld/ELF/Relocations.h @@ -163,7 +163,9 @@ class InputSectionDescription; class ThunkCreator { public: - ThunkCreator(Ctx &ctx) : ctx(ctx) {} + // Thunk may be incomplete. Avoid inline ctor/dtor. + ThunkCreator(Ctx &ctx); + ~ThunkCreator(); // Return true if Thunks have been added to OutputSections bool createThunks(uint32_t pass, ArrayRef outputSections); @@ -199,9 +201,10 @@ private: // original addend, so we cannot fold offset + addend. A nested pair is used // because DenseMapInfo is not specialized for std::tuple. llvm::DenseMap, int64_t>, - std::vector> + SmallVector, 0>> thunkedSymbolsBySectionAndAddend; - llvm::DenseMap, std::vector> + llvm::DenseMap, + SmallVector, 0>> thunkedSymbols; // Find a Thunk from the Thunks symbol definition, we can use this to find @@ -220,7 +223,7 @@ private: // to be reached via thunks that use indirect branches. A destination // needs at most one landing pad as that can be reused by all callers. llvm::DenseMap, int64_t>, - Thunk *> + std::unique_ptr> landingPadsBySectionAndAddend; // All the nonLandingPad thunks that have been created, in order of creation. diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp index c9625453b2eb..4f04c33f0e5c 100644 --- a/lld/ELF/Thunks.cpp +++ b/lld/ELF/Thunks.cpp @@ -1372,15 +1372,16 @@ Thunk::Thunk(Ctx &ctx, Symbol &d, int64_t a) Thunk::~Thunk() = default; -static Thunk *addThunkAArch64(Ctx &ctx, RelType type, Symbol &s, int64_t a) { +static std::unique_ptr addThunkAArch64(Ctx &ctx, RelType type, Symbol &s, + int64_t a) { assert(is_contained({R_AARCH64_CALL26, R_AARCH64_JUMP26, R_AARCH64_PLT32}, type)); bool mayNeedLandingPad = (ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) && !isAArch64BTILandingPad(ctx, s, a); if (ctx.arg.picThunk) - return make(ctx, s, a, mayNeedLandingPad); - return make(ctx, s, a, mayNeedLandingPad); + return std::make_unique(ctx, s, a, mayNeedLandingPad); + return std::make_unique(ctx, s, a, mayNeedLandingPad); } // Creates a thunk for long branches or Thumb-ARM interworking. @@ -1391,7 +1392,8 @@ static Thunk *addThunkAArch64(Ctx &ctx, RelType type, Symbol &s, int64_t a) { // // TODO: use B for short Thumb->Arm thunks instead of LDR (this doesn't work for // Arm->Thumb, as in Arm state no BX PC trick; it doesn't switch state). -static Thunk *addThunkArmv4(Ctx &ctx, RelType reloc, Symbol &s, int64_t a) { +static std::unique_ptr addThunkArmv4(Ctx &ctx, RelType reloc, Symbol &s, + int64_t a) { bool thumb_target = s.getVA(ctx, a) & 1; switch (reloc) { @@ -1401,21 +1403,21 @@ static Thunk *addThunkArmv4(Ctx &ctx, RelType reloc, Symbol &s, int64_t a) { case R_ARM_CALL: if (ctx.arg.picThunk) { if (thumb_target) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); } if (thumb_target) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); case R_ARM_THM_CALL: if (ctx.arg.picThunk) { if (thumb_target) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); } if (thumb_target) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); } Fatal(ctx) << "relocation " << reloc << " to " << &s << " not supported for Armv4 or Armv4T target"; @@ -1427,7 +1429,8 @@ static Thunk *addThunkArmv4(Ctx &ctx, RelType reloc, Symbol &s, int64_t a) { // - MOVT and MOVW instructions cannot be used // - Only Thumb relocation that can generate a Thunk is a BL, this can always // be transformed into a BLX -static Thunk *addThunkArmv5v6(Ctx &ctx, RelType reloc, Symbol &s, int64_t a) { +static std::unique_ptr addThunkArmv5v6(Ctx &ctx, RelType reloc, + Symbol &s, int64_t a) { switch (reloc) { case R_ARM_PC24: case R_ARM_PLT32: @@ -1435,8 +1438,8 @@ static Thunk *addThunkArmv5v6(Ctx &ctx, RelType reloc, Symbol &s, int64_t a) { case R_ARM_CALL: case R_ARM_THM_CALL: if (ctx.arg.picThunk) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); } Fatal(ctx) << "relocation " << reloc << " to " << &s << " not supported for Armv5 or Armv6 targets"; @@ -1448,8 +1451,8 @@ static Thunk *addThunkArmv5v6(Ctx &ctx, RelType reloc, Symbol &s, int64_t a) { // - MOVT and MOVW instructions cannot be used. // - Only a limited number of instructions can access registers r8 and above // - No interworking support is needed (all Thumb). -static Thunk *addThunkV6M(Ctx &ctx, const InputSection &isec, RelType reloc, - Symbol &s, int64_t a) { +static std::unique_ptr addThunkV6M(Ctx &ctx, const InputSection &isec, + RelType reloc, Symbol &s, int64_t a) { const bool isPureCode = isec.getParent()->flags & SHF_ARM_PURECODE; switch (reloc) { case R_ARM_THM_JUMP19: @@ -1457,7 +1460,7 @@ static Thunk *addThunkV6M(Ctx &ctx, const InputSection &isec, RelType reloc, case R_ARM_THM_CALL: if (ctx.arg.isPic) { if (!isPureCode) - return make(ctx, s, a); + return std::make_unique(ctx, s, a); Fatal(ctx) << "relocation " << reloc << " to " << &s @@ -1466,8 +1469,8 @@ static Thunk *addThunkV6M(Ctx &ctx, const InputSection &isec, RelType reloc, llvm_unreachable(""); } if (isPureCode) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); } Fatal(ctx) << "relocation " << reloc << " to " << &s << " not supported for Armv6-M targets"; @@ -1475,8 +1478,8 @@ static Thunk *addThunkV6M(Ctx &ctx, const InputSection &isec, RelType reloc, } // Creates a thunk for Thumb-ARM interworking or branch range extension. -static Thunk *addThunkArm(Ctx &ctx, const InputSection &isec, RelType reloc, - Symbol &s, int64_t a) { +static std::unique_ptr addThunkArm(Ctx &ctx, const InputSection &isec, + RelType reloc, Symbol &s, int64_t a) { // Decide which Thunk is needed based on: // Available instruction set // - An Arm Thunk can only be used if Arm state is available. @@ -1508,47 +1511,49 @@ static Thunk *addThunkArm(Ctx &ctx, const InputSection &isec, RelType reloc, case R_ARM_JUMP24: case R_ARM_CALL: if (ctx.arg.picThunk) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); case R_ARM_THM_JUMP19: case R_ARM_THM_JUMP24: case R_ARM_THM_CALL: if (ctx.arg.picThunk) - return make(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); + return std::make_unique(ctx, s, a); } llvm_unreachable(""); } -static Thunk *addThunkAVR(Ctx &ctx, RelType type, Symbol &s, int64_t a) { +static std::unique_ptr addThunkAVR(Ctx &ctx, RelType type, Symbol &s, + int64_t a) { switch (type) { case R_AVR_LO8_LDI_GS: case R_AVR_HI8_LDI_GS: - return make(ctx, s, a); + return std::make_unique(ctx, s, a); default: llvm_unreachable(""); } } -static Thunk *addThunkMips(Ctx &ctx, RelType type, Symbol &s) { +static std::unique_ptr addThunkMips(Ctx &ctx, RelType type, Symbol &s) { if ((s.stOther & STO_MIPS_MICROMIPS) && isMipsR6(ctx)) - return make(ctx, s); + return std::make_unique(ctx, s); if (s.stOther & STO_MIPS_MICROMIPS) - return make(ctx, s); - return make(ctx, s); + return std::make_unique(ctx, s); + return std::make_unique(ctx, s); } -static Thunk *addThunkPPC32(Ctx &ctx, const InputSection &isec, - const Relocation &rel, Symbol &s) { +static std::unique_ptr addThunkPPC32(Ctx &ctx, const InputSection &isec, + const Relocation &rel, Symbol &s) { assert((rel.type == R_PPC_LOCAL24PC || rel.type == R_PPC_REL24 || rel.type == R_PPC_PLTREL24) && "unexpected relocation type for thunk"); if (s.isInPlt(ctx)) - return make(ctx, isec, rel, s); - return make(ctx, s, rel.addend); + return std::make_unique(ctx, isec, rel, s); + return std::make_unique(ctx, s, rel.addend); } -static Thunk *addThunkPPC64(Ctx &ctx, RelType type, Symbol &s, int64_t a) { +static std::unique_ptr addThunkPPC64(Ctx &ctx, RelType type, Symbol &s, + int64_t a) { assert((type == R_PPC64_REL14 || type == R_PPC64_REL24 || type == R_PPC64_REL24_NOTOC) && "unexpected relocation type for thunk"); @@ -1558,27 +1563,30 @@ static Thunk *addThunkPPC64(Ctx &ctx, RelType type, Symbol &s, int64_t a) { if (type == R_PPC64_REL24_NOTOC) ctx.target->ppc64DynamicSectionOpt = 0x2; - if (s.isInPlt(ctx)) - return type == R_PPC64_REL24_NOTOC - ? (Thunk *)make(ctx, s, /*gotPlt=*/true) - : (Thunk *)make(ctx, s); + if (s.isInPlt(ctx)) { + if (type == R_PPC64_REL24_NOTOC) + return std::make_unique(ctx, s, + /*gotPlt=*/true); + return std::make_unique(ctx, s); + } // This check looks at the st_other bits of the callee. If the value is 1 // then the callee clobbers the TOC and we need an R2 save stub when RelType // is R_PPC64_REL14 or R_PPC64_REL24. if ((type == R_PPC64_REL14 || type == R_PPC64_REL24) && (s.stOther >> 5) == 1) - return make(ctx, s, a); + return std::make_unique(ctx, s, a); if (type == R_PPC64_REL24_NOTOC) - return make(ctx, s, /*gotPlt=*/false); + return std::make_unique(ctx, s, /*gotPlt=*/false); if (ctx.arg.picThunk) - return make(ctx, s, a); + return std::make_unique(ctx, s, a); - return make(ctx, s, a); + return std::make_unique(ctx, s, a); } -Thunk *elf::addThunk(Ctx &ctx, const InputSection &isec, Relocation &rel) { +std::unique_ptr elf::addThunk(Ctx &ctx, const InputSection &isec, + Relocation &rel) { Symbol &s = *rel.sym; int64_t a = rel.addend; @@ -1600,10 +1608,10 @@ Thunk *elf::addThunk(Ctx &ctx, const InputSection &isec, Relocation &rel) { } } -Thunk *elf::addLandingPadThunk(Ctx &ctx, Symbol &s, int64_t a) { +std::unique_ptr elf::addLandingPadThunk(Ctx &ctx, Symbol &s, int64_t a) { switch (ctx.arg.emachine) { case EM_AARCH64: - return make(ctx, s, a); + return std::make_unique(ctx, s, a); default: llvm_unreachable("add landing pad only supported for AArch64"); } diff --git a/lld/ELF/Thunks.h b/lld/ELF/Thunks.h index 247b50058032..446345b8517f 100644 --- a/lld/ELF/Thunks.h +++ b/lld/ELF/Thunks.h @@ -76,11 +76,12 @@ public: // For a Relocation to symbol S create a Thunk to be added to a synthetic // ThunkSection. -Thunk *addThunk(Ctx &, const InputSection &isec, Relocation &rel); +std::unique_ptr addThunk(Ctx &, const InputSection &isec, + Relocation &rel); // Create a landing pad Thunk for use when indirect branches from Thunks // are restricted. -Thunk *addLandingPadThunk(Ctx &, Symbol &s, int64_t a); +std::unique_ptr addLandingPadThunk(Ctx &, Symbol &s, int64_t a); void writePPC32PltCallStub(Ctx &, uint8_t *buf, uint64_t gotPltVA, const InputFile *file, int64_t addend); diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 26887d11a397..67497bad7cb2 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -49,7 +49,7 @@ template class Writer { public: LLVM_ELF_IMPORT_TYPES_ELFT(ELFT) - Writer(Ctx &ctx) : ctx(ctx), buffer(ctx.e.outputBuffer) {} + Writer(Ctx &ctx) : ctx(ctx), buffer(ctx.e.outputBuffer), tc(ctx) {} void run(); @@ -82,6 +82,8 @@ private: Ctx &ctx; std::unique_ptr &buffer; + // ThunkCreator holds Thunks that are used at writeTo time. + ThunkCreator tc; void addRelIpltSymbols(); void addStartEndSymbols(); @@ -1448,7 +1450,6 @@ static void finalizeSynthetic(Ctx &ctx, SyntheticSection *sec) { // in Writer::finalizeSections(). template void Writer::finalizeAddressDependentContent() { llvm::TimeTraceScope timeScope("Finalize address dependent content"); - ThunkCreator tc(ctx); AArch64Err843419Patcher a64p(ctx); ARMErr657417Patcher a32p(ctx); ctx.script->assignAddresses();