mirrors/llvm-project
0
0
Fork 0
mirror of https://github.com/llvm/llvm-project.git synced 2025-05-17 20:06:06 +00:00
Code Issues Projects Releases Wiki Activity
llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
Joseph Huber 4e80bc7d71
[Clang] Introduce scoped variants of GNU atomic functions (#72280) 
Summary:
The standard GNU atomic operations are a very common way to target
hardware atomics on the device. With more heterogenous devices being
introduced, the concept of memory scopes has been in the LLVM language
for awhile via the `syncscope` modifier. For targets, such as the GPU,
this can change code generation depending on whether or not we only need
to be consistent with the memory ordering with the entire system, the
single GPU device, or lower.

Previously these scopes were only exported via the `opencl` and `hip`
variants of these functions. However, this made it difficult to use
outside of those languages and the semantics were different from the
standard GNU versions. This patch introduces a `__scoped_atomic` variant
for the common functions. There was some discussion over whether or not
these should be overloads of the existing ones, or simply new variants.
I leant towards new variants to be less disruptive.

The scope here can be one of the following

```
__MEMORY_SCOPE_SYSTEM // All devices and systems
__MEMORY_SCOPE_DEVICE // Just this device
__MEMORY_SCOPE_WRKGRP // A 'work-group' AKA CUDA block
__MEMORY_SCOPE_WVFRNT // A 'wavefront' AKA CUDA warp
__MEMORY_SCOPE_SINGLE // A single thread.
```
Naming consistency was attempted, but it is difficult to capture to full
spectrum with no many names. Suggestions appreciated.
2023-12-07 13:40:25 -06:00

655 lines
25 KiB
C++
Raw Blame History

//===- AMDGPU.cpp ---------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "ABIInfoImpl.h"
#include "TargetInfo.h"
#include "clang/Basic/TargetOptions.h"
using namespace clang;
using namespace clang::CodeGen;
//===----------------------------------------------------------------------===//
// AMDGPU ABI Implementation
//===----------------------------------------------------------------------===//
namespace {
class AMDGPUABIInfo final : public DefaultABIInfo {
private:
static const unsigned MaxNumRegsForArgsRet = 16;
unsigned numRegsForType(QualType Ty) const;
bool isHomogeneousAggregateBaseType(QualType Ty) const override;
bool isHomogeneousAggregateSmallEnough(const Type *Base,
uint64_t Members) const override;
// Coerce HIP scalar pointer arguments from generic pointers to global ones.
llvm::Type *coerceKernelArgumentType(llvm::Type *Ty, unsigned FromAS,
unsigned ToAS) const {
// Single value types.
auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Ty);
if (PtrTy && PtrTy->getAddressSpace() == FromAS)
return llvm::PointerType::get(Ty->getContext(), ToAS);
return Ty;
}
public:
explicit AMDGPUABIInfo(CodeGen::CodeGenTypes &CGT) :
DefaultABIInfo(CGT) {}
ABIArgInfo classifyReturnType(QualType RetTy) const;
ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
ABIArgInfo classifyArgumentType(QualType Ty, unsigned &NumRegsLeft) const;
void computeInfo(CGFunctionInfo &FI) const override;
Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
QualType Ty) const override;
};
bool AMDGPUABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
return true;
}
bool AMDGPUABIInfo::isHomogeneousAggregateSmallEnough(
const Type *Base, uint64_t Members) const {
uint32_t NumRegs = (getContext().getTypeSize(Base) + 31) / 32;
// Homogeneous Aggregates may occupy at most 16 registers.
return Members * NumRegs <= MaxNumRegsForArgsRet;
}
/// Estimate number of registers the type will use when passed in registers.
unsigned AMDGPUABIInfo::numRegsForType(QualType Ty) const {
unsigned NumRegs = 0;
if (const VectorType *VT = Ty->getAs<VectorType>()) {
// Compute from the number of elements. The reported size is based on the
// in-memory size, which includes the padding 4th element for 3-vectors.
QualType EltTy = VT->getElementType();
unsigned EltSize = getContext().getTypeSize(EltTy);
// 16-bit element vectors should be passed as packed.
if (EltSize == 16)
return (VT->getNumElements() + 1) / 2;
unsigned EltNumRegs = (EltSize + 31) / 32;
return EltNumRegs * VT->getNumElements();
}
if (const RecordType *RT = Ty->getAs<RecordType>()) {
const RecordDecl *RD = RT->getDecl();
assert(!RD->hasFlexibleArrayMember());
for (const FieldDecl *Field : RD->fields()) {
QualType FieldTy = Field->getType();
NumRegs += numRegsForType(FieldTy);
}
return NumRegs;
}
return (getContext().getTypeSize(Ty) + 31) / 32;
}
void AMDGPUABIInfo::computeInfo(CGFunctionInfo &FI) const {
llvm::CallingConv::ID CC = FI.getCallingConvention();
if (!getCXXABI().classifyReturnType(FI))
FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
unsigned NumRegsLeft = MaxNumRegsForArgsRet;
for (auto &Arg : FI.arguments()) {
if (CC == llvm::CallingConv::AMDGPU_KERNEL) {
Arg.info = classifyKernelArgumentType(Arg.type);
} else {
Arg.info = classifyArgumentType(Arg.type, NumRegsLeft);
}
}
}
Address AMDGPUABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
QualType Ty) const {
llvm_unreachable("AMDGPU does not support varargs");
}
ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const {
if (isAggregateTypeForABI(RetTy)) {
// Records with non-trivial destructors/copy-constructors should not be
// returned by value.
if (!getRecordArgABI(RetTy, getCXXABI())) {
// Ignore empty structs/unions.
if (isEmptyRecord(getContext(), RetTy, true))
return ABIArgInfo::getIgnore();
// Lower single-element structs to just return a regular value.
if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
if (const RecordType *RT = RetTy->getAs<RecordType>()) {
const RecordDecl *RD = RT->getDecl();
if (RD->hasFlexibleArrayMember())
return DefaultABIInfo::classifyReturnType(RetTy);
}
// Pack aggregates <= 4 bytes into single VGPR or pair.
uint64_t Size = getContext().getTypeSize(RetTy);
if (Size <= 16)
return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
if (Size <= 32)
return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
if (Size <= 64) {
llvm::Type *I32Ty = llvm::Type::getInt32Ty(getVMContext());
return ABIArgInfo::getDirect(llvm::ArrayType::get(I32Ty, 2));
}
if (numRegsForType(RetTy) <= MaxNumRegsForArgsRet)
return ABIArgInfo::getDirect();
}
}
// Otherwise just do the default thing.
return DefaultABIInfo::classifyReturnType(RetTy);
}
/// For kernels all parameters are really passed in a special buffer. It doesn't
/// make sense to pass anything byval, so everything must be direct.
ABIArgInfo AMDGPUABIInfo::classifyKernelArgumentType(QualType Ty) const {
Ty = useFirstFieldIfTransparentUnion(Ty);
// TODO: Can we omit empty structs?
if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
Ty = QualType(SeltTy, 0);
llvm::Type *OrigLTy = CGT.ConvertType(Ty);
llvm::Type *LTy = OrigLTy;
if (getContext().getLangOpts().HIP) {
LTy = coerceKernelArgumentType(
OrigLTy, /*FromAS=*/getContext().getTargetAddressSpace(LangAS::Default),
/*ToAS=*/getContext().getTargetAddressSpace(LangAS::cuda_device));
}
// FIXME: Should also use this for OpenCL, but it requires addressing the
// problem of kernels being called.
//
// FIXME: This doesn't apply the optimization of coercing pointers in structs
// to global address space when using byref. This would require implementing a
// new kind of coercion of the in-memory type when for indirect arguments.
if (!getContext().getLangOpts().OpenCL && LTy == OrigLTy &&
isAggregateTypeForABI(Ty)) {
return ABIArgInfo::getIndirectAliased(
getContext().getTypeAlignInChars(Ty),
getContext().getTargetAddressSpace(LangAS::opencl_constant),
false /*Realign*/, nullptr /*Padding*/);
}
// If we set CanBeFlattened to true, CodeGen will expand the struct to its
// individual elements, which confuses the Clover OpenCL backend; therefore we
// have to set it to false here. Other args of getDirect() are just defaults.
return ABIArgInfo::getDirect(LTy, 0, nullptr, false);
}
ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty,
unsigned &NumRegsLeft) const {
assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow");
Ty = useFirstFieldIfTransparentUnion(Ty);
if (isAggregateTypeForABI(Ty)) {
// Records with non-trivial destructors/copy-constructors should not be
// passed by value.
if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
// Ignore empty structs/unions.
if (isEmptyRecord(getContext(), Ty, true))
return ABIArgInfo::getIgnore();
// Lower single-element structs to just pass a regular value. TODO: We
// could do reasonable-size multiple-element structs too, using getExpand(),
// though watch out for things like bitfields.
if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
if (const RecordType *RT = Ty->getAs<RecordType>()) {
const RecordDecl *RD = RT->getDecl();
if (RD->hasFlexibleArrayMember())
return DefaultABIInfo::classifyArgumentType(Ty);
}
// Pack aggregates <= 8 bytes into single VGPR or pair.
uint64_t Size = getContext().getTypeSize(Ty);
if (Size <= 64) {
unsigned NumRegs = (Size + 31) / 32;
NumRegsLeft -= std::min(NumRegsLeft, NumRegs);
if (Size <= 16)
return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
if (Size <= 32)
return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
// XXX: Should this be i64 instead, and should the limit increase?
llvm::Type *I32Ty = llvm::Type::getInt32Ty(getVMContext());
return ABIArgInfo::getDirect(llvm::ArrayType::get(I32Ty, 2));
}
if (NumRegsLeft > 0) {
unsigned NumRegs = numRegsForType(Ty);
if (NumRegsLeft >= NumRegs) {
NumRegsLeft -= NumRegs;
return ABIArgInfo::getDirect();
}
}
// Use pass-by-reference in stead of pass-by-value for struct arguments in
// function ABI.
return ABIArgInfo::getIndirectAliased(
getContext().getTypeAlignInChars(Ty),
getContext().getTargetAddressSpace(LangAS::opencl_private));
}
// Otherwise just do the default thing.
ABIArgInfo ArgInfo = DefaultABIInfo::classifyArgumentType(Ty);
if (!ArgInfo.isIndirect()) {
unsigned NumRegs = numRegsForType(Ty);
NumRegsLeft -= std::min(NumRegs, NumRegsLeft);
}
return ArgInfo;
}
class AMDGPUTargetCodeGenInfo : public TargetCodeGenInfo {
public:
AMDGPUTargetCodeGenInfo(CodeGenTypes &CGT)
: TargetCodeGenInfo(std::make_unique<AMDGPUABIInfo>(CGT)) {}
void setFunctionDeclAttributes(const FunctionDecl *FD, llvm::Function *F,
CodeGenModule &CGM) const;
void emitTargetGlobals(CodeGen::CodeGenModule &CGM) const override;
void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
CodeGen::CodeGenModule &M) const override;
unsigned getOpenCLKernelCallingConv() const override;
llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
llvm::PointerType *T, QualType QT) const override;
LangAS getASTAllocaAddressSpace() const override {
return getLangASFromTargetAS(
getABIInfo().getDataLayout().getAllocaAddrSpace());
}
LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
const VarDecl *D) const override;
llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts,
SyncScope Scope,
llvm::AtomicOrdering Ordering,
llvm::LLVMContext &Ctx) const override;
llvm::Value *createEnqueuedBlockKernel(CodeGenFunction &CGF,
llvm::Function *BlockInvokeFunc,
llvm::Type *BlockTy) const override;
bool shouldEmitStaticExternCAliases() const override;
bool shouldEmitDWARFBitFieldSeparators() const override;
void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
};
}
static bool requiresAMDGPUProtectedVisibility(const Decl *D,
llvm::GlobalValue *GV) {
if (GV->getVisibility() != llvm::GlobalValue::HiddenVisibility)
return false;
return !D->hasAttr<OMPDeclareTargetDeclAttr>() &&
(D->hasAttr<OpenCLKernelAttr>() ||
(isa<FunctionDecl>(D) && D->hasAttr<CUDAGlobalAttr>()) ||
(isa<VarDecl>(D) &&
(D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinSurfaceType() ||
cast<VarDecl>(D)->getType()->isCUDADeviceBuiltinTextureType())));
}
void AMDGPUTargetCodeGenInfo::setFunctionDeclAttributes(
const FunctionDecl *FD, llvm::Function *F, CodeGenModule &M) const {
const auto *ReqdWGS =
M.getLangOpts().OpenCL ? FD->getAttr<ReqdWorkGroupSizeAttr>() : nullptr;
const bool IsOpenCLKernel =
M.getLangOpts().OpenCL && FD->hasAttr<OpenCLKernelAttr>();
const bool IsHIPKernel = M.getLangOpts().HIP && FD->hasAttr<CUDAGlobalAttr>();
const auto *FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>();
if (ReqdWGS || FlatWGS) {
M.handleAMDGPUFlatWorkGroupSizeAttr(F, FlatWGS, ReqdWGS);
} else if (IsOpenCLKernel || IsHIPKernel) {
// By default, restrict the maximum size to a value specified by
// --gpu-max-threads-per-block=n or its default value for HIP.
const unsigned OpenCLDefaultMaxWorkGroupSize = 256;
const unsigned DefaultMaxWorkGroupSize =
IsOpenCLKernel ? OpenCLDefaultMaxWorkGroupSize
: M.getLangOpts().GPUMaxThreadsPerBlock;
std::string AttrVal =
std::string("1,") + llvm::utostr(DefaultMaxWorkGroupSize);
F->addFnAttr("amdgpu-flat-work-group-size", AttrVal);
}
if (const auto *Attr = FD->getAttr<AMDGPUWavesPerEUAttr>())
M.handleAMDGPUWavesPerEUAttr(F, Attr);
if (const auto *Attr = FD->getAttr<AMDGPUNumSGPRAttr>()) {
unsigned NumSGPR = Attr->getNumSGPR();
if (NumSGPR != 0)
F->addFnAttr("amdgpu-num-sgpr", llvm::utostr(NumSGPR));
}
if (const auto *Attr = FD->getAttr<AMDGPUNumVGPRAttr>()) {
uint32_t NumVGPR = Attr->getNumVGPR();
if (NumVGPR != 0)
F->addFnAttr("amdgpu-num-vgpr", llvm::utostr(NumVGPR));
}
}
/// Emits control constants used to change per-architecture behaviour in the
/// AMDGPU ROCm device libraries.
void AMDGPUTargetCodeGenInfo::emitTargetGlobals(
CodeGen::CodeGenModule &CGM) const {
StringRef Name = "__oclc_ABI_version";
llvm::GlobalVariable *OriginalGV = CGM.getModule().getNamedGlobal(Name);
if (OriginalGV && !llvm::GlobalVariable::isExternalLinkage(OriginalGV->getLinkage()))
return;
if (CGM.getTarget().getTargetOpts().CodeObjectVersion ==
llvm::CodeObjectVersionKind::COV_None)
return;
auto *Type = llvm::IntegerType::getIntNTy(CGM.getModule().getContext(), 32);
llvm::Constant *COV = llvm::ConstantInt::get(
Type, CGM.getTarget().getTargetOpts().CodeObjectVersion);
// It needs to be constant weak_odr without externally_initialized so that
// the load instuction can be eliminated by the IPSCCP.
auto *GV = new llvm::GlobalVariable(
CGM.getModule(), Type, true, llvm::GlobalValue::WeakODRLinkage, COV, Name,
nullptr, llvm::GlobalValue::ThreadLocalMode::NotThreadLocal,
CGM.getContext().getTargetAddressSpace(LangAS::opencl_constant));
GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Local);
GV->setVisibility(llvm::GlobalValue::VisibilityTypes::HiddenVisibility);
// Replace any external references to this variable with the new global.
if (OriginalGV) {
OriginalGV->replaceAllUsesWith(GV);
GV->takeName(OriginalGV);
OriginalGV->eraseFromParent();
}
}
void AMDGPUTargetCodeGenInfo::setTargetAttributes(
const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
if (requiresAMDGPUProtectedVisibility(D, GV)) {
GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
GV->setDSOLocal(true);
}
if (GV->isDeclaration())
return;
llvm::Function *F = dyn_cast<llvm::Function>(GV);
if (!F)
return;
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);
if (FD)
setFunctionDeclAttributes(FD, F, M);
if (M.getContext().getTargetInfo().allowAMDGPUUnsafeFPAtomics())
F->addFnAttr("amdgpu-unsafe-fp-atomics", "true");
if (!getABIInfo().getCodeGenOpts().EmitIEEENaNCompliantInsts)
F->addFnAttr("amdgpu-ieee", "false");
}
unsigned AMDGPUTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
return llvm::CallingConv::AMDGPU_KERNEL;
}
// Currently LLVM assumes null pointers always have value 0,
// which results in incorrectly transformed IR. Therefore, instead of
// emitting null pointers in private and local address spaces, a null
// pointer in generic address space is emitted which is casted to a
// pointer in local or private address space.
llvm::Constant *AMDGPUTargetCodeGenInfo::getNullPointer(
const CodeGen::CodeGenModule &CGM, llvm::PointerType *PT,
QualType QT) const {
if (CGM.getContext().getTargetNullPointerValue(QT) == 0)
return llvm::ConstantPointerNull::get(PT);
auto &Ctx = CGM.getContext();
auto NPT = llvm::PointerType::get(
PT->getContext(), Ctx.getTargetAddressSpace(LangAS::opencl_generic));
return llvm::ConstantExpr::getAddrSpaceCast(
llvm::ConstantPointerNull::get(NPT), PT);
}
LangAS
AMDGPUTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
const VarDecl *D) const {
assert(!CGM.getLangOpts().OpenCL &&
!(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
"Address space agnostic languages only");
LangAS DefaultGlobalAS = getLangASFromTargetAS(
CGM.getContext().getTargetAddressSpace(LangAS::opencl_global));
if (!D)
return DefaultGlobalAS;
LangAS AddrSpace = D->getType().getAddressSpace();
if (AddrSpace != LangAS::Default)
return AddrSpace;
// Only promote to address space 4 if VarDecl has constant initialization.
if (D->getType().isConstantStorage(CGM.getContext(), false, false) &&
D->hasConstantInitialization()) {
if (auto ConstAS = CGM.getTarget().getConstantAddressSpace())
return *ConstAS;
}
return DefaultGlobalAS;
}
llvm::SyncScope::ID
AMDGPUTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
SyncScope Scope,
llvm::AtomicOrdering Ordering,
llvm::LLVMContext &Ctx) const {
std::string Name;
switch (Scope) {
case SyncScope::HIPSingleThread:
case SyncScope::SingleScope:
Name = "singlethread";
break;
case SyncScope::HIPWavefront:
case SyncScope::OpenCLSubGroup:
case SyncScope::WavefrontScope:
Name = "wavefront";
break;
case SyncScope::HIPWorkgroup:
case SyncScope::OpenCLWorkGroup:
case SyncScope::WorkgroupScope:
Name = "workgroup";
break;
case SyncScope::HIPAgent:
case SyncScope::OpenCLDevice:
case SyncScope::DeviceScope:
Name = "agent";
break;
case SyncScope::SystemScope:
case SyncScope::HIPSystem:
case SyncScope::OpenCLAllSVMDevices:
Name = "";
break;
}
if (Ordering != llvm::AtomicOrdering::SequentiallyConsistent) {
if (!Name.empty())
Name = Twine(Twine(Name) + Twine("-")).str();
Name = Twine(Twine(Name) + Twine("one-as")).str();
}
return Ctx.getOrInsertSyncScopeID(Name);
}
bool AMDGPUTargetCodeGenInfo::shouldEmitStaticExternCAliases() const {
return false;
}
bool AMDGPUTargetCodeGenInfo::shouldEmitDWARFBitFieldSeparators() const {
return true;
}
void AMDGPUTargetCodeGenInfo::setCUDAKernelCallingConvention(
const FunctionType *&FT) const {
FT = getABIInfo().getContext().adjustFunctionType(
FT, FT->getExtInfo().withCallingConv(CC_OpenCLKernel));
}
/// Create an OpenCL kernel for an enqueued block.
///
/// The type of the first argument (the block literal) is the struct type
/// of the block literal instead of a pointer type. The first argument
/// (block literal) is passed directly by value to the kernel. The kernel
/// allocates the same type of struct on stack and stores the block literal
/// to it and passes its pointer to the block invoke function. The kernel
/// has "enqueued-block" function attribute and kernel argument metadata.
llvm::Value *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
CodeGenFunction &CGF, llvm::Function *Invoke, llvm::Type *BlockTy) const {
auto &Builder = CGF.Builder;
auto &C = CGF.getLLVMContext();
auto *InvokeFT = Invoke->getFunctionType();
llvm::SmallVector<llvm::Type *, 2> ArgTys;
llvm::SmallVector<llvm::Metadata *, 8> AddressQuals;
llvm::SmallVector<llvm::Metadata *, 8> AccessQuals;
llvm::SmallVector<llvm::Metadata *, 8> ArgTypeNames;
llvm::SmallVector<llvm::Metadata *, 8> ArgBaseTypeNames;
llvm::SmallVector<llvm::Metadata *, 8> ArgTypeQuals;
llvm::SmallVector<llvm::Metadata *, 8> ArgNames;
ArgTys.push_back(BlockTy);
ArgTypeNames.push_back(llvm::MDString::get(C, "__block_literal"));
AddressQuals.push_back(llvm::ConstantAsMetadata::get(Builder.getInt32(0)));
ArgBaseTypeNames.push_back(llvm::MDString::get(C, "__block_literal"));
ArgTypeQuals.push_back(llvm::MDString::get(C, ""));
AccessQuals.push_back(llvm::MDString::get(C, "none"));
ArgNames.push_back(llvm::MDString::get(C, "block_literal"));
for (unsigned I = 1, E = InvokeFT->getNumParams(); I < E; ++I) {
ArgTys.push_back(InvokeFT->getParamType(I));
ArgTypeNames.push_back(llvm::MDString::get(C, "void*"));
AddressQuals.push_back(llvm::ConstantAsMetadata::get(Builder.getInt32(3)));
AccessQuals.push_back(llvm::MDString::get(C, "none"));
ArgBaseTypeNames.push_back(llvm::MDString::get(C, "void*"));
ArgTypeQuals.push_back(llvm::MDString::get(C, ""));
ArgNames.push_back(
llvm::MDString::get(C, (Twine("local_arg") + Twine(I)).str()));
}
std::string Name = Invoke->getName().str() + "_kernel";
auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), ArgTys, false);
auto *F = llvm::Function::Create(FT, llvm::GlobalValue::InternalLinkage, Name,
&CGF.CGM.getModule());
F->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
llvm::AttrBuilder KernelAttrs(C);
// FIXME: The invoke isn't applying the right attributes either
// FIXME: This is missing setTargetAttributes
CGF.CGM.addDefaultFunctionDefinitionAttributes(KernelAttrs);
KernelAttrs.addAttribute("enqueued-block");
F->addFnAttrs(KernelAttrs);
auto IP = CGF.Builder.saveIP();
auto *BB = llvm::BasicBlock::Create(C, "entry", F);
Builder.SetInsertPoint(BB);
const auto BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(BlockTy);
auto *BlockPtr = Builder.CreateAlloca(BlockTy, nullptr);
BlockPtr->setAlignment(BlockAlign);
Builder.CreateAlignedStore(F->arg_begin(), BlockPtr, BlockAlign);
auto *Cast = Builder.CreatePointerCast(BlockPtr, InvokeFT->getParamType(0));
llvm::SmallVector<llvm::Value *, 2> Args;
Args.push_back(Cast);
for (llvm::Argument &A : llvm::drop_begin(F->args()))
Args.push_back(&A);
llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
call->setCallingConv(Invoke->getCallingConv());
Builder.CreateRetVoid();
Builder.restoreIP(IP);
F->setMetadata("kernel_arg_addr_space", llvm::MDNode::get(C, AddressQuals));
F->setMetadata("kernel_arg_access_qual", llvm::MDNode::get(C, AccessQuals));
F->setMetadata("kernel_arg_type", llvm::MDNode::get(C, ArgTypeNames));
F->setMetadata("kernel_arg_base_type",
llvm::MDNode::get(C, ArgBaseTypeNames));
F->setMetadata("kernel_arg_type_qual", llvm::MDNode::get(C, ArgTypeQuals));
if (CGF.CGM.getCodeGenOpts().EmitOpenCLArgMetadata)
F->setMetadata("kernel_arg_name", llvm::MDNode::get(C, ArgNames));
return F;
}
void CodeGenModule::handleAMDGPUFlatWorkGroupSizeAttr(
llvm::Function *F, const AMDGPUFlatWorkGroupSizeAttr *FlatWGS,
const ReqdWorkGroupSizeAttr *ReqdWGS, int32_t *MinThreadsVal,
int32_t *MaxThreadsVal) {
unsigned Min = 0;
unsigned Max = 0;
if (FlatWGS) {
Min = FlatWGS->getMin()->EvaluateKnownConstInt(getContext()).getExtValue();
Max = FlatWGS->getMax()->EvaluateKnownConstInt(getContext()).getExtValue();
}
if (ReqdWGS && Min == 0 && Max == 0)
Min = Max = ReqdWGS->getXDim() * ReqdWGS->getYDim() * ReqdWGS->getZDim();
if (Min != 0) {
assert(Min <= Max && "Min must be less than or equal Max");
if (MinThreadsVal)
*MinThreadsVal = Min;
if (MaxThreadsVal)
*MaxThreadsVal = Max;
std::string AttrVal = llvm::utostr(Min) + "," + llvm::utostr(Max);
if (F)
F->addFnAttr("amdgpu-flat-work-group-size", AttrVal);
} else
assert(Max == 0 && "Max must be zero");
}
void CodeGenModule::handleAMDGPUWavesPerEUAttr(
llvm::Function *F, const AMDGPUWavesPerEUAttr *Attr) {
unsigned Min =
Attr->getMin()->EvaluateKnownConstInt(getContext()).getExtValue();
unsigned Max =
Attr->getMax()
? Attr->getMax()->EvaluateKnownConstInt(getContext()).getExtValue()
: 0;
if (Min != 0) {
assert((Max == 0 || Min <= Max) && "Min must be less than or equal Max");
std::string AttrVal = llvm::utostr(Min);
if (Max != 0)
AttrVal = AttrVal + "," + llvm::utostr(Max);
F->addFnAttr("amdgpu-waves-per-eu", AttrVal);
} else
assert(Max == 0 && "Max must be zero");
}
std::unique_ptr<TargetCodeGenInfo>
CodeGen::createAMDGPUTargetCodeGenInfo(CodeGenModule &CGM) {
return std::make_unique<AMDGPUTargetCodeGenInfo>(CGM.getTypes());
}
Reference in New Issue View Git Blame Copy Permalink