mirrors/llvm-project
0
0
Fork 0
mirror of https://github.com/llvm/llvm-project.git synced 2025-04-18 15:36:58 +00:00
Code Issues Projects Releases Wiki Activity

[NFC][clang][FMV][TargetInfo] Refactor API for FMV feature priority. (#116257)

Browse Source 
Currently we have code with target hooks in CodeGenModule shared between
X86 and AArch64 for sorting MultiVersionResolverOptions. Those are used
when generating IFunc resolvers for FMV. The RISCV target has different
criteria for sorting, therefore it repeats sorting after calling
CodeGenFunction::EmitMultiVersionResolver.

I am moving the FMV priority logic in TargetInfo, so that it can be
implemented by the TargetParser which then makes it possible to query it
from llvm. Here is an example why this is handy:
https://github.com/llvm/llvm-project/pull/87939
This commit is contained in:
Alexandros Lamprineas 2024-11-28 09:22:05 +00:00 committed by GitHub
parent 2c242b98c6
commit 88c2af80fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 216 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
clang/include/clang/Basic/Attr.td
View File

@ -3263,33 +3263,28 @@ def Target : InheritableAttr {
let Subjects = SubjectList<[Function], ErrorDiag>;
let Documentation = [TargetDocs];
let AdditionalMembers = [{
StringRef getArchitecture() const {
std::optional<StringRef> getX86Architecture() const {
StringRef Features = getFeaturesStr();
if (Features == "default") return {};
SmallVector<StringRef, 1> AttrFeatures;
Features.split(AttrFeatures, ",");
for (auto &Feature : AttrFeatures) {
SmallVector<StringRef, 4> AttrFeatures;
Features.split(AttrFeatures, ',');
for (StringRef Feature : AttrFeatures) {
Feature = Feature.trim();
if (Feature.starts_with("arch="))
return Feature.drop_front(sizeof("arch=") - 1);
}
return "";
return std::nullopt;
}
// Gets the list of features as simple string-refs with no +/- or 'no-'.
// Only adds the items to 'Out' that are additions.
void getAddedFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
void getX86AddedFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
if (isDefaultVersion())
return;
StringRef Features = getFeaturesStr();
if (Features == "default") return;
SmallVector<StringRef, 1> AttrFeatures;
Features.split(AttrFeatures, ",");
SmallVector<StringRef, 4> AttrFeatures;
Features.split(AttrFeatures, ',');
for (auto &Feature : AttrFeatures) {
Feature = Feature.trim();
if (!Feature.starts_with("no-") && !Feature.starts_with("arch=") &&
!Feature.starts_with("fpmath=") && !Feature.starts_with("tune="))
Out.push_back(Feature);
@ -3307,17 +3302,17 @@ def TargetVersion : InheritableAttr, TargetSpecificAttr<TargetArch<!listconcat(T
let Documentation = [TargetVersionDocs];
let AdditionalMembers = [{
StringRef getName() const { return getNamesStr().trim(); }
bool isDefaultVersion() const {
return getName() == "default";
}
void getFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
if (isDefaultVersion()) return;
bool isDefaultVersion() const { return getName() == "default"; }
void getFeatures(llvm::SmallVectorImpl<StringRef> &Out,
char Delim = '+') const {
if (isDefaultVersion())
return;
StringRef Features = getName();
SmallVector<StringRef, 8> AttrFeatures;
Features.split(AttrFeatures, "+");
for (auto &Feature : AttrFeatures) {
SmallVector<StringRef, 4> AttrFeatures;
Features.split(AttrFeatures, Delim);
for (StringRef Feature : AttrFeatures) {
Feature = Feature.trim();
Out.push_back(Feature);
}
@ -3334,20 +3329,40 @@ def TargetClones : InheritableAttr {
StringRef getFeatureStr(unsigned Index) const {
return *(featuresStrs_begin() + Index);
}
bool isDefaultVersion(unsigned Index) const {
return getFeatureStr(Index) == "default";
}
void getFeatures(llvm::SmallVectorImpl<StringRef> &Out,
unsigned Index) const {
if (isDefaultVersion(Index)) return;
unsigned Index, char Delim = '+') const {
if (isDefaultVersion(Index))
return;
StringRef Features = getFeatureStr(Index);
SmallVector<StringRef, 8> AttrFeatures;
Features.split(AttrFeatures, "+");
for (auto &Feature : AttrFeatures) {
SmallVector<StringRef, 4> AttrFeatures;
Features.split(AttrFeatures, Delim);
for (StringRef Feature : AttrFeatures) {
Feature = Feature.trim();
Out.push_back(Feature);
}
}
std::optional<StringRef> getX86Architecture(unsigned Index) const {
StringRef Feature = getFeatureStr(Index);
if (Feature.starts_with("arch="))
return Feature.drop_front(sizeof("arch=") - 1);
return std::nullopt;
}
void getX86Feature(llvm::SmallVectorImpl<StringRef> &Out,
unsigned Index) const {
if (isDefaultVersion(Index))
return;
if (getX86Architecture(Index))
return;
Out.push_back(getFeatureStr(Index));
}
// Given an index into the 'featuresStrs' sequence, compute a unique
// ID to be used with function name mangling for the associated variant.
// This mapping is necessary due to a requirement that the mangling ID

6
clang/include/clang/Basic/TargetInfo.h
View File

@ -1525,14 +1525,10 @@ public:
// Return the target-specific priority for features/cpus/vendors so
// that they can be properly sorted for checking.
virtual unsigned multiVersionSortPriority(StringRef Name) const {
virtual unsigned getFMVPriority(ArrayRef<StringRef> Features) const {
return 0;
}
// Return the target-specific cost for feature
// that taken into account in priority sorting.
virtual unsigned multiVersionFeatureCost() const { return 0; }
// Validate the contents of the __builtin_cpu_is(const char*)
// argument.
virtual bool validateCpuIs(StringRef Name) const { return false; }

14
clang/lib/Basic/Targets/AArch64.cpp
View File

@ -705,18 +705,8 @@ AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts) const {
return std::nullopt;
}
unsigned AArch64TargetInfo::multiVersionSortPriority(StringRef Name) const {
if (Name == "default")
return 0;
if (auto Ext = llvm::AArch64::parseFMVExtension(Name))
return Ext->Priority;
return 0;
}
unsigned AArch64TargetInfo::multiVersionFeatureCost() const {
// Take the maximum priority as per feature cost, so more features win.
constexpr unsigned MaxFMVPriority = 1000;
return MaxFMVPriority;
unsigned AArch64TargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const {
return llvm::AArch64::getFMVPriority(Features);
}
bool AArch64TargetInfo::doesFeatureAffectCodeGen(StringRef Name) const {

3
clang/lib/Basic/Targets/AArch64.h
View File

@ -137,8 +137,7 @@ public:
void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override;
bool setCPU(const std::string &Name) override;
unsigned multiVersionSortPriority(StringRef Name) const override;
unsigned multiVersionFeatureCost() const override;
unsigned getFMVPriority(ArrayRef<StringRef> Features) const override;
bool useFP16ConversionIntrinsics() const override {
return false;

57
clang/lib/Basic/Targets/RISCV.cpp
View File

@ -423,6 +423,24 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
Features.split(AttrFeatures, ";");
bool FoundArch = false;
auto handleArchExtension = [](StringRef AttrString,
std::vector<std::string> &Features) {
SmallVector<StringRef, 1> Exts;
AttrString.split(Exts, ",");
for (auto Ext : Exts) {
if (Ext.empty())
continue;
StringRef ExtName = Ext.substr(1);
std::string TargetFeature =
llvm::RISCVISAInfo::getTargetFeatureForExtension(ExtName);
if (!TargetFeature.empty())
Features.push_back(Ext.front() + TargetFeature);
else
Features.push_back(Ext.str());
}
};
for (auto &Feature : AttrFeatures) {
Feature = Feature.trim();
StringRef AttrString = Feature.split("=").second.trim();
@ -436,20 +454,7 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
if (AttrString.starts_with("+")) {
// EXTENSION like arch=+v,+zbb
SmallVector<StringRef, 1> Exts;
AttrString.split(Exts, ",");
for (auto Ext : Exts) {
if (Ext.empty())
continue;
StringRef ExtName = Ext.substr(1);
std::string TargetFeature =
llvm::RISCVISAInfo::getTargetFeatureForExtension(ExtName);
if (!TargetFeature.empty())
Ret.Features.push_back(Ext.front() + TargetFeature);
else
Ret.Features.push_back(Ext.str());
}
handleArchExtension(AttrString, Ret.Features);
} else {
// full-arch-string like arch=rv64gcv
handleFullArchString(AttrString, Ret.Features);
@ -475,11 +480,35 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
Ret.Tune = AttrString;
} else if (Feature.starts_with("priority")) {
// Skip because it only use for FMV.
} else if (Feature.starts_with("+")) {
// Handle target_version/target_clones attribute strings
// that are already delimited by ','
handleArchExtension(Feature, Ret.Features);
}
}
return Ret;
}
unsigned RISCVTargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const {
// Priority is explicitly specified on RISC-V unlike on other targets, where
// it is derived by all the features of a specific version. Therefore if a
// feature contains the priority string, then return it immediately.
for (StringRef Feature : Features) {
auto [LHS, RHS] = Feature.rsplit(';');
if (LHS.consume_front("priority="))
Feature = LHS;
else if (RHS.consume_front("priority="))
Feature = RHS;
else
continue;
unsigned Priority;
if (!Feature.getAsInteger(0, Priority))
return Priority;
}
// Default Priority is zero.
return 0;
}
TargetInfo::CallingConvCheckResult
RISCVTargetInfo::checkCallingConvention(CallingConv CC) const {
switch (CC) {

1
clang/lib/Basic/Targets/RISCV.h
View File

@ -122,6 +122,7 @@ public:
void fillValidTuneCPUList(SmallVectorImpl<StringRef> &Values) const override;
bool supportsTargetAttributeTune() const override { return true; }
ParsedTargetAttr parseTargetAttr(StringRef Str) const override;
unsigned getFMVPriority(ArrayRef<StringRef> Features) const override;
std::pair<unsigned, unsigned> hardwareInterferenceSizes() const override {
return std::make_pair(32, 32);

31
clang/lib/Basic/Targets/X86.cpp
View File

@ -1364,19 +1364,26 @@ static llvm::X86::ProcessorFeatures getFeature(StringRef Name) {
// correct, so it asserts if the value is out of range.
}
unsigned X86TargetInfo::multiVersionSortPriority(StringRef Name) const {
// Valid CPUs have a 'key feature' that compares just better than its key
// feature.
using namespace llvm::X86;
CPUKind Kind = parseArchX86(Name);
if (Kind != CK_None) {
ProcessorFeatures KeyFeature = getKeyFeature(Kind);
return (getFeaturePriority(KeyFeature) << 1) + 1;
}
unsigned X86TargetInfo::getFMVPriority(ArrayRef<StringRef> Features) const {
auto getPriority = [this](StringRef Feature) -> unsigned {
// Valid CPUs have a 'key feature' that compares just better than its key
// feature.
using namespace llvm::X86;
CPUKind Kind = parseArchX86(Feature);
if (Kind != CK_None) {
ProcessorFeatures KeyFeature = getKeyFeature(Kind);
return (getFeaturePriority(KeyFeature) << 1) + 1;
}
// Now we know we have a feature, so get its priority and shift it a few so
// that we have sufficient room for the CPUs (above).
return getFeaturePriority(getFeature(Feature)) << 1;
};
// Now we know we have a feature, so get its priority and shift it a few so
// that we have sufficient room for the CPUs (above).
return getFeaturePriority(getFeature(Name)) << 1;
unsigned Priority = 0;
for (StringRef Feature : Features)
if (!Feature.empty())
Priority = std::max(Priority, getPriority(Feature));
return Priority;
}
bool X86TargetInfo::validateCPUSpecificCPUDispatch(StringRef Name) const {

2
clang/lib/Basic/Targets/X86.h
View File

@ -384,7 +384,7 @@ public:
return CPU != llvm::X86::CK_None;
}
unsigned multiVersionSortPriority(StringRef Name) const override;
unsigned getFMVPriority(ArrayRef<StringRef> Features) const override;
bool setFPMath(StringRef Name) override;

4
clang/lib/CodeGen/ABIInfo.cpp
View File

@ -218,8 +218,8 @@ void ABIInfo::appendAttributeMangling(StringRef AttrStr,
// only have "+" prefixes here.
assert(LHS.starts_with("+") && RHS.starts_with("+") &&
"Features should always have a prefix.");
return TI.multiVersionSortPriority(LHS.substr(1)) >
TI.multiVersionSortPriority(RHS.substr(1));
return TI.getFMVPriority({LHS.substr(1)}) >
TI.getFMVPriority({RHS.substr(1)});
});
bool IsFirst = true;

109
clang/lib/CodeGen/CodeGenFunction.cpp
View File

@ -2828,23 +2828,17 @@ void CodeGenFunction::EmitKCFIOperandBundle(
Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar()));
}
llvm::Value *CodeGenFunction::FormAArch64ResolverCondition(
const MultiVersionResolverOption &RO) {
llvm::SmallVector<StringRef, 8> CondFeatures;
for (const StringRef &Feature : RO.Conditions.Features)
CondFeatures.push_back(Feature);
if (!CondFeatures.empty()) {
return EmitAArch64CpuSupports(CondFeatures);
}
return nullptr;
llvm::Value *
CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {
return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(RO.Features);
}
llvm::Value *CodeGenFunction::FormX86ResolverCondition(
const MultiVersionResolverOption &RO) {
llvm::Value *
CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {
llvm::Value *Condition = nullptr;
if (!RO.Conditions.Architecture.empty()) {
StringRef Arch = RO.Conditions.Architecture;
if (RO.Architecture) {
StringRef Arch = *RO.Architecture;
// If arch= specifies an x86-64 micro-architecture level, test the feature
// with __builtin_cpu_supports, otherwise use __builtin_cpu_is.
if (Arch.starts_with("x86-64"))
@ -2853,8 +2847,8 @@ llvm::Value *CodeGenFunction::FormX86ResolverCondition(
Condition = EmitX86CpuIs(Arch);
}
if (!RO.Conditions.Features.empty()) {
llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
if (!RO.Features.empty()) {
llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Features);
Condition =
Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
}
@ -2884,7 +2878,7 @@ static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
}
void CodeGenFunction::EmitMultiVersionResolver(
llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
llvm::Triple::ArchType ArchType =
getContext().getTargetInfo().getTriple().getArch();
@ -2907,26 +2901,8 @@ void CodeGenFunction::EmitMultiVersionResolver(
}
}
static unsigned getPriorityFromAttrString(StringRef AttrStr) {
SmallVector<StringRef, 8> Attrs;
AttrStr.split(Attrs, ';');
// Default Priority is zero.
unsigned Priority = 0;
for (auto Attr : Attrs) {
if (Attr.consume_front("priority=")) {
unsigned Result;
if (!Attr.getAsInteger(0, Result))
Priority = Result;
}
}
return Priority;
}
void CodeGenFunction::EmitRISCVMultiVersionResolver(
llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
if (getContext().getTargetInfo().getTriple().getOS() !=
llvm::Triple::OSType::Linux) {
@ -2942,20 +2918,10 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
bool HasDefault = false;
unsigned DefaultIndex = 0;
SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> CurrOptions(
Options);
llvm::stable_sort(
CurrOptions, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
const CodeGenFunction::MultiVersionResolverOption &RHS) {
return getPriorityFromAttrString(LHS.Conditions.Features[0]) >
getPriorityFromAttrString(RHS.Conditions.Features[0]);
});
// Check the each candidate function.
for (unsigned Index = 0; Index < CurrOptions.size(); Index++) {
for (unsigned Index = 0; Index < Options.size(); Index++) {
if (CurrOptions[Index].Conditions.Features[0].starts_with("default")) {
if (Options[Index].Features.empty()) {
HasDefault = true;
DefaultIndex = Index;
continue;
@ -2963,15 +2929,6 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
Builder.SetInsertPoint(CurBlock);
std::vector<std::string> TargetAttrFeats =
getContext()
.getTargetInfo()
.parseTargetAttr(CurrOptions[Index].Conditions.Features[0])
.Features;
if (TargetAttrFeats.empty())
continue;
// FeaturesCondition: The bitmask of the required extension has been
// enabled by the runtime object.
// (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==
@ -2994,20 +2951,32 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
// Without checking the length first, we may access an incorrect memory
// address when using different versions.
llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;
llvm::SmallVector<std::string, 8> TargetAttrFeats;
for (auto &Feat : TargetAttrFeats) {
StringRef CurrFeat = Feat;
if (CurrFeat.starts_with('+'))
CurrTargetAttrFeats.push_back(CurrFeat.substr(1));
for (StringRef Feat : Options[Index].Features) {
std::vector<std::string> FeatStr =
getContext().getTargetInfo().parseTargetAttr(Feat).Features;
assert(FeatStr.size() == 1 && "Feature string not delimited");
std::string &CurrFeat = FeatStr.front();
if (CurrFeat[0] == '+')
TargetAttrFeats.push_back(CurrFeat.substr(1));
}
if (TargetAttrFeats.empty())
continue;
for (std::string &Feat : TargetAttrFeats)
CurrTargetAttrFeats.push_back(Feat);
Builder.SetInsertPoint(CurBlock);
llvm::Value *FeatsCondition = EmitRISCVCpuSupports(CurrTargetAttrFeats);
llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
CGBuilderTy RetBuilder(*this, RetBlock);
CreateMultiVersionResolverReturn(
CGM, Resolver, RetBuilder, CurrOptions[Index].Function, SupportsIFunc);
CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder,
Options[Index].Function, SupportsIFunc);
llvm::BasicBlock *ElseBlock = createBasicBlock("resolver_else", Resolver);
Builder.SetInsertPoint(CurBlock);
@ -3019,9 +2988,8 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
// Finally, emit the default one.
if (HasDefault) {
Builder.SetInsertPoint(CurBlock);
CreateMultiVersionResolverReturn(CGM, Resolver, Builder,
CurrOptions[DefaultIndex].Function,
SupportsIFunc);
CreateMultiVersionResolverReturn(
CGM, Resolver, Builder, Options[DefaultIndex].Function, SupportsIFunc);
return;
}
@ -3035,17 +3003,16 @@ void CodeGenFunction::EmitRISCVMultiVersionResolver(
}
void CodeGenFunction::EmitAArch64MultiVersionResolver(
llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
assert(!Options.empty() && "No multiversion resolver options found");
assert(Options.back().Conditions.Features.size() == 0 &&
"Default case must be last");
assert(Options.back().Features.size() == 0 && "Default case must be last");
bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
assert(SupportsIFunc &&
"Multiversion resolver requires target IFUNC support");
bool AArch64CpuInitialized = false;
llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
for (const MultiVersionResolverOption &RO : Options) {
for (const FMVResolverOption &RO : Options) {
Builder.SetInsertPoint(CurBlock);
llvm::Value *Condition = FormAArch64ResolverCondition(RO);
@ -3081,7 +3048,7 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
}
void CodeGenFunction::EmitX86MultiVersionResolver(
llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {
bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
@ -3090,7 +3057,7 @@ void CodeGenFunction::EmitX86MultiVersionResolver(
Builder.SetInsertPoint(CurBlock);
EmitX86CpuInit();
for (const MultiVersionResolverOption &RO : Options) {
for (const FMVResolverOption &RO : Options) {
Builder.SetInsertPoint(CurBlock);
llvm::Value *Condition = FormX86ResolverCondition(RO);

39
clang/lib/CodeGen/CodeGenFunction.h
View File

@ -5334,35 +5334,27 @@ public:
void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
struct MultiVersionResolverOption {
struct FMVResolverOption {
llvm::Function *Function;
struct Conds {
StringRef Architecture;
llvm::SmallVector<StringRef, 8> Features;
llvm::SmallVector<StringRef, 8> Features;
std::optional<StringRef> Architecture;
Conds(StringRef Arch, ArrayRef<StringRef> Feats)
: Architecture(Arch), Features(Feats) {}
} Conditions;
MultiVersionResolverOption(llvm::Function *F, StringRef Arch,
ArrayRef<StringRef> Feats)
: Function(F), Conditions(Arch, Feats) {}
FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,
std::optional<StringRef> Arch = std::nullopt)
: Function(F), Features(Feats), Architecture(Arch) {}
};
// Emits the body of a multiversion function's resolver. Assumes that the
// options are already sorted in the proper order, with the 'default' option
// last (if it exists).
void EmitMultiVersionResolver(llvm::Function *Resolver,
ArrayRef<MultiVersionResolverOption> Options);
void
EmitX86MultiVersionResolver(llvm::Function *Resolver,
ArrayRef<MultiVersionResolverOption> Options);
void
EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
ArrayRef<MultiVersionResolverOption> Options);
void
EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
ArrayRef<MultiVersionResolverOption> Options);
ArrayRef<FMVResolverOption> Options);
void EmitX86MultiVersionResolver(llvm::Function *Resolver,
ArrayRef<FMVResolverOption> Options);
void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,
ArrayRef<FMVResolverOption> Options);
void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,
ArrayRef<FMVResolverOption> Options);
private:
QualType getVarArgType(const Expr *Arg);
@ -5381,10 +5373,9 @@ private:
llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);
llvm::Value *EmitX86CpuInit();
llvm::Value *FormX86ResolverCondition(const MultiVersionResolverOption &RO);
llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);
llvm::Value *EmitAArch64CpuInit();
llvm::Value *
FormAArch64ResolverCondition(const MultiVersionResolverOption &RO);
llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);
llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);
llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);
};

86
clang/lib/CodeGen/CodeGenModule.cpp
View File

@ -4223,23 +4223,12 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
llvm::Function *NewFn);
static unsigned
TargetMVPriority(const TargetInfo &TI,
const CodeGenFunction::MultiVersionResolverOption &RO) {
unsigned Priority = 0;
unsigned NumFeatures = 0;
for (StringRef Feat : RO.Conditions.Features) {
Priority = std::max(Priority, TI.multiVersionSortPriority(Feat));
NumFeatures++;
}
if (!RO.Conditions.Architecture.empty())
Priority = std::max(
Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture));
Priority += TI.multiVersionFeatureCost() * NumFeatures;
return Priority;
static unsigned getFMVPriority(const TargetInfo &TI,
const CodeGenFunction::FMVResolverOption &RO) {
llvm::SmallVector<StringRef, 8> Features{RO.Features};
if (RO.Architecture)
Features.push_back(*RO.Architecture);
return TI.getFMVPriority(Features);
}
// Multiversion functions should be at most 'WeakODRLinkage' so that a different
@ -4285,7 +4274,7 @@ void CodeGenModule::emitMultiVersionFunctions() {
// target_version("default")) or target_clones() is present and defined
// in this TU. For other architectures it is always emitted.
bool ShouldEmitResolver = !getTarget().getTriple().isAArch64();
SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
getContext().forEachMultiversionedFunctionVersion(
FD, [&](const FunctionDecl *CurFD) {
@ -4293,20 +4282,17 @@ void CodeGenModule::emitMultiVersionFunctions() {
bool IsDefined = CurFD->doesThisDeclarationHaveABody();
if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
TA->getAddedFeatures(Feats);
assert(getTarget().getTriple().isX86() && "Unsupported target");
TA->getX86AddedFeatures(Feats);
llvm::Function *Func = createFunction(CurFD);
Options.emplace_back(Func, TA->getArchitecture(), Feats);
Options.emplace_back(Func, Feats, TA->getX86Architecture());
} else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
if (TVA->isDefaultVersion() && IsDefined)
ShouldEmitResolver = true;
llvm::Function *Func = createFunction(CurFD);
if (getTarget().getTriple().isRISCV()) {
Feats.push_back(TVA->getName());
} else {
assert(getTarget().getTriple().isAArch64());
TVA->getFeatures(Feats);
}
Options.emplace_back(Func, /*Architecture*/ "", Feats);
char Delim = getTarget().getTriple().isAArch64() ? '+' : ',';
TVA->getFeatures(Feats, Delim);
Options.emplace_back(Func, Feats);
} else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
if (IsDefined)
ShouldEmitResolver = true;
@ -4315,21 +4301,15 @@ void CodeGenModule::emitMultiVersionFunctions() {
continue;
llvm::Function *Func = createFunction(CurFD, I);
StringRef Architecture;
Feats.clear();
if (getTarget().getTriple().isAArch64())
TC->getFeatures(Feats, I);
else if (getTarget().getTriple().isRISCV()) {
StringRef Version = TC->getFeatureStr(I);
Feats.push_back(Version);
if (getTarget().getTriple().isX86()) {
TC->getX86Feature(Feats, I);
Options.emplace_back(Func, Feats, TC->getX86Architecture(I));
} else {
StringRef Version = TC->getFeatureStr(I);
if (Version.starts_with("arch="))
Architecture = Version.drop_front(sizeof("arch=") - 1);
else if (Version != "default")
Feats.push_back(Version);
char Delim = getTarget().getTriple().isAArch64() ? '+' : ',';
TC->getFeatures(Feats, I, Delim);
Options.emplace_back(Func, Feats);
}
Options.emplace_back(Func, Architecture, Feats);
}
} else
llvm_unreachable("unexpected MultiVersionKind");
@ -4368,9 +4348,9 @@ void CodeGenModule::emitMultiVersionFunctions() {
const TargetInfo &TI = getTarget();
llvm::stable_sort(
Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
const CodeGenFunction::MultiVersionResolverOption &RHS) {
return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
Options, [&TI](const CodeGenFunction::FMVResolverOption &LHS,
const CodeGenFunction::FMVResolverOption &RHS) {
return getFMVPriority(TI, LHS) > getFMVPriority(TI, RHS);
});
CodeGenFunction CGF(*this);
CGF.EmitMultiVersionResolver(ResolverFunc, Options);
@ -4429,7 +4409,7 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
ResolverFunc->setComdat(
getModule().getOrInsertComdat(ResolverFunc->getName()));
SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
SmallVector<CodeGenFunction::FMVResolverOption, 10> Options;
const TargetInfo &Target = getTarget();
unsigned Index = 0;
for (const IdentifierInfo *II : DD->cpus()) {
@ -4463,25 +4443,23 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
llvm::erase_if(Features, [&Target](StringRef Feat) {
return !Target.validateCpuSupports(Feat);
});
Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
Options.emplace_back(cast<llvm::Function>(Func), Features);
++Index;
}
llvm::stable_sort(
Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
const CodeGenFunction::MultiVersionResolverOption &RHS) {
return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) >
llvm::X86::getCpuSupportsMask(RHS.Conditions.Features);
});
llvm::stable_sort(Options, [](const CodeGenFunction::FMVResolverOption &LHS,
const CodeGenFunction::FMVResolverOption &RHS) {
return llvm::X86::getCpuSupportsMask(LHS.Features) >
llvm::X86::getCpuSupportsMask(RHS.Features);
});
// If the list contains multiple 'default' versions, such as when it contains
// 'pentium' and 'generic', don't emit the call to the generic one (since we
// always run on at least a 'pentium'). We do this by deleting the 'least
// advanced' (read, lowest mangling letter).
while (Options.size() > 1 &&
llvm::all_of(llvm::X86::getCpuSupportsMask(
(Options.end() - 2)->Conditions.Features),
[](auto X) { return X == 0; })) {
while (Options.size() > 1 && llvm::all_of(llvm::X86::getCpuSupportsMask(
(Options.end() - 2)->Features),
[](auto X) { return X == 0; })) {
StringRef LHSName = (Options.end() - 2)->Function->getName();
StringRef RHSName = (Options.end() - 1)->Function->getName();
if (LHSName.compare(RHSName) < 0)

3
llvm/include/llvm/TargetParser/AArch64TargetParser.h
View File

@ -268,6 +268,9 @@ void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
bool isX18ReservedByDefault(const Triple &TT);
// Return the priority for a given set of FMV features.
unsigned getFMVPriority(ArrayRef<StringRef> Features);
// For given feature names, return a bitmask corresponding to the entries of
// AArch64::CPUFeatures. The values in CPUFeatures are not bitmasks
// themselves, they are sequential (0, 1, 2, 3, ...).

13
llvm/lib/TargetParser/AArch64TargetParser.cpp
View File

@ -48,6 +48,19 @@ std::optional<AArch64::ArchInfo> AArch64::ArchInfo::findBySubArch(StringRef SubA
return {};
}
unsigned AArch64::getFMVPriority(ArrayRef<StringRef> Features) {
constexpr unsigned MaxFMVPriority = 1000;
unsigned Priority = 0;
unsigned NumFeatures = 0;
for (StringRef Feature : Features) {
if (auto Ext = parseFMVExtension(Feature)) {
Priority = std::max(Priority, Ext->Priority);
NumFeatures++;
}
}
return Priority + MaxFMVPriority * NumFeatures;
}
uint64_t AArch64::getCpuSupportsMask(ArrayRef<StringRef> FeatureStrs) {
uint64_t FeaturesMask = 0;
for (const StringRef &FeatureStr : FeatureStrs) {