mirror of
https://github.com/llvm/llvm-project.git
synced 2025-05-16 02:16:06 +00:00
338be9c595
Currently Clang does not respect -fno-unroll-loops during LTO. During D76916 it was suggested to respect -fno-unroll-loops on a TU basis. This patch uses the existing llvm.loop.unroll.disable metadata to disable loop unrolling explicitly for each loop in the TU if unrolling is disabled. This should ensure that loops from TUs compiled with -fno-unroll-loops are skipped by the unroller during LTO. This also means that if a loop from a TU with -fno-unroll-loops gets inlined into a TU without this option, the loop won't be unrolled. Due to the fact that some transforms might drop loop metadata, there potentially are cases in which we still unroll loops from TUs with -fno-unroll-loops. I think we should fix those issues rather than introducing a function attribute to disable loop unrolling during LTO. Improving the metadata handling will benefit other use cases, like various loop pragmas, too. And it is an improvement to clang completely ignoring -fno-unroll-loops during LTO. If that direction looks good, we can use a similar approach to also respect -fno-vectorize during LTO, at least for LoopVectorize. In the future, this might also allow us to remove the UnrollLoops option LLVM's PassManagerBuilder. Reviewers: Meinersbur, hfinkel, dexonsmith, tejohnson Reviewed By: Meinersbur, tejohnson Differential Revision: https://reviews.llvm.org/D77058
808 lines
29 KiB
C++
808 lines
29 KiB
C++
//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "CGLoopInfo.h"
|
|
#include "clang/AST/ASTContext.h"
|
|
#include "clang/AST/Attr.h"
|
|
#include "clang/AST/Expr.h"
|
|
#include "clang/Basic/CodeGenOptions.h"
|
|
#include "llvm/IR/BasicBlock.h"
|
|
#include "llvm/IR/CFG.h"
|
|
#include "llvm/IR/Constants.h"
|
|
#include "llvm/IR/InstrTypes.h"
|
|
#include "llvm/IR/Instructions.h"
|
|
#include "llvm/IR/Metadata.h"
|
|
using namespace clang::CodeGen;
|
|
using namespace llvm;
|
|
|
|
MDNode *
|
|
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
SmallVector<Metadata *, 4> NewLoopProperties;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
NewLoopProperties.push_back(TempNode.get());
|
|
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
|
|
ArrayRef<Metadata *> LoopProperties,
|
|
bool &HasUserTransforms) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
Optional<bool> Enabled;
|
|
if (Attrs.PipelineDisabled)
|
|
Enabled = false;
|
|
else if (Attrs.PipelineInitiationInterval != 0)
|
|
Enabled = true;
|
|
|
|
if (Enabled != true) {
|
|
SmallVector<Metadata *, 4> NewLoopProperties;
|
|
if (Enabled == false) {
|
|
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
NewLoopProperties.push_back(
|
|
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt1Ty(Ctx), 1))}));
|
|
LoopProperties = NewLoopProperties;
|
|
}
|
|
return createLoopPropertiesMetadata(LoopProperties);
|
|
}
|
|
|
|
SmallVector<Metadata *, 4> Args;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
Args.push_back(TempNode.get());
|
|
Args.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
if (Attrs.PipelineInitiationInterval > 0) {
|
|
Metadata *Vals[] = {
|
|
MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
// No follow-up: This is the last transformation.
|
|
|
|
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
HasUserTransforms = true;
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *
|
|
LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
|
|
ArrayRef<Metadata *> LoopProperties,
|
|
bool &HasUserTransforms) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
Optional<bool> Enabled;
|
|
if (Attrs.UnrollEnable == LoopAttributes::Disable)
|
|
Enabled = false;
|
|
else if (Attrs.UnrollEnable == LoopAttributes::Full)
|
|
Enabled = None;
|
|
else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
|
|
Attrs.UnrollCount != 0)
|
|
Enabled = true;
|
|
|
|
if (Enabled != true) {
|
|
// createFullUnrollMetadata will already have added llvm.loop.unroll.disable
|
|
// if unrolling is disabled.
|
|
return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
|
|
}
|
|
|
|
SmallVector<Metadata *, 4> FollowupLoopProperties;
|
|
|
|
// Apply all loop properties to the unrolled loop.
|
|
FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
// Don't unroll an already unrolled loop.
|
|
FollowupLoopProperties.push_back(
|
|
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
|
|
|
|
bool FollowupHasTransforms = false;
|
|
MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
|
|
FollowupHasTransforms);
|
|
|
|
SmallVector<Metadata *, 4> Args;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
Args.push_back(TempNode.get());
|
|
Args.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
// Setting unroll.count
|
|
if (Attrs.UnrollCount > 0) {
|
|
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
// Setting unroll.full or unroll.disable
|
|
if (Attrs.UnrollEnable == LoopAttributes::Enable) {
|
|
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
if (FollowupHasTransforms)
|
|
Args.push_back(MDNode::get(
|
|
Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
|
|
|
|
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
HasUserTransforms = true;
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *
|
|
LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
|
|
ArrayRef<Metadata *> LoopProperties,
|
|
bool &HasUserTransforms) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
Optional<bool> Enabled;
|
|
if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
|
|
Enabled = false;
|
|
else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
|
|
Attrs.UnrollAndJamCount != 0)
|
|
Enabled = true;
|
|
|
|
if (Enabled != true) {
|
|
SmallVector<Metadata *, 4> NewLoopProperties;
|
|
if (Enabled == false) {
|
|
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
NewLoopProperties.push_back(MDNode::get(
|
|
Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
|
|
LoopProperties = NewLoopProperties;
|
|
}
|
|
return createPartialUnrollMetadata(Attrs, LoopProperties,
|
|
HasUserTransforms);
|
|
}
|
|
|
|
SmallVector<Metadata *, 4> FollowupLoopProperties;
|
|
FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
FollowupLoopProperties.push_back(
|
|
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
|
|
|
|
bool FollowupHasTransforms = false;
|
|
MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
|
|
FollowupHasTransforms);
|
|
|
|
SmallVector<Metadata *, 4> Args;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
Args.push_back(TempNode.get());
|
|
Args.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
// Setting unroll_and_jam.count
|
|
if (Attrs.UnrollAndJamCount > 0) {
|
|
Metadata *Vals[] = {
|
|
MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
|
|
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
|
|
Attrs.UnrollAndJamCount))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
|
|
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
if (FollowupHasTransforms)
|
|
Args.push_back(MDNode::get(
|
|
Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
|
|
Followup}));
|
|
|
|
if (UnrollAndJamInnerFollowup)
|
|
Args.push_back(MDNode::get(
|
|
Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
|
|
UnrollAndJamInnerFollowup}));
|
|
|
|
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
HasUserTransforms = true;
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *
|
|
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
|
|
ArrayRef<Metadata *> LoopProperties,
|
|
bool &HasUserTransforms) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
Optional<bool> Enabled;
|
|
if (Attrs.VectorizeEnable == LoopAttributes::Disable)
|
|
Enabled = false;
|
|
else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
|
|
Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
|
|
Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
|
|
Enabled = true;
|
|
|
|
if (Enabled != true) {
|
|
SmallVector<Metadata *, 4> NewLoopProperties;
|
|
if (Enabled == false) {
|
|
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
NewLoopProperties.push_back(
|
|
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt1Ty(Ctx), 0))}));
|
|
LoopProperties = NewLoopProperties;
|
|
}
|
|
return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
|
|
}
|
|
|
|
// Apply all loop properties to the vectorized loop.
|
|
SmallVector<Metadata *, 4> FollowupLoopProperties;
|
|
FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
// Don't vectorize an already vectorized loop.
|
|
FollowupLoopProperties.push_back(
|
|
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
|
|
|
|
bool FollowupHasTransforms = false;
|
|
MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
|
|
FollowupHasTransforms);
|
|
|
|
SmallVector<Metadata *, 4> Args;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
Args.push_back(TempNode.get());
|
|
Args.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
// Setting vectorize.predicate
|
|
bool IsVectorPredicateEnabled = false;
|
|
if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
|
|
Attrs.VectorizeEnable != LoopAttributes::Disable &&
|
|
Attrs.VectorizeWidth < 1) {
|
|
|
|
IsVectorPredicateEnabled =
|
|
(Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
|
|
|
|
Metadata *Vals[] = {
|
|
MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
|
|
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
|
|
IsVectorPredicateEnabled))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
// Setting vectorize.width
|
|
if (Attrs.VectorizeWidth > 0) {
|
|
Metadata *Vals[] = {
|
|
MDString::get(Ctx, "llvm.loop.vectorize.width"),
|
|
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
|
|
Attrs.VectorizeWidth))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
// Setting interleave.count
|
|
if (Attrs.InterleaveCount > 0) {
|
|
Metadata *Vals[] = {
|
|
MDString::get(Ctx, "llvm.loop.interleave.count"),
|
|
ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
|
|
Attrs.InterleaveCount))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
}
|
|
|
|
// vectorize.enable is set if:
|
|
// 1) loop hint vectorize.enable is set, or
|
|
// 2) it is implied when vectorize.predicate is set, or
|
|
// 3) it is implied when vectorize.width is set.
|
|
if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
|
|
IsVectorPredicateEnabled ||
|
|
Attrs.VectorizeWidth > 1 ) {
|
|
bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
|
|
Args.push_back(
|
|
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt1Ty(Ctx), AttrVal))}));
|
|
}
|
|
|
|
if (FollowupHasTransforms)
|
|
Args.push_back(MDNode::get(
|
|
Ctx,
|
|
{MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
|
|
|
|
MDNode *LoopID = MDNode::get(Ctx, Args);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
HasUserTransforms = true;
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *
|
|
LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
|
|
ArrayRef<Metadata *> LoopProperties,
|
|
bool &HasUserTransforms) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
Optional<bool> Enabled;
|
|
if (Attrs.DistributeEnable == LoopAttributes::Disable)
|
|
Enabled = false;
|
|
if (Attrs.DistributeEnable == LoopAttributes::Enable)
|
|
Enabled = true;
|
|
|
|
if (Enabled != true) {
|
|
SmallVector<Metadata *, 4> NewLoopProperties;
|
|
if (Enabled == false) {
|
|
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
NewLoopProperties.push_back(
|
|
MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt1Ty(Ctx), 0))}));
|
|
LoopProperties = NewLoopProperties;
|
|
}
|
|
return createLoopVectorizeMetadata(Attrs, LoopProperties,
|
|
HasUserTransforms);
|
|
}
|
|
|
|
bool FollowupHasTransforms = false;
|
|
MDNode *Followup =
|
|
createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
|
|
|
|
SmallVector<Metadata *, 4> Args;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
Args.push_back(TempNode.get());
|
|
Args.append(LoopProperties.begin(), LoopProperties.end());
|
|
|
|
Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
|
|
ConstantAsMetadata::get(ConstantInt::get(
|
|
llvm::Type::getInt1Ty(Ctx),
|
|
(Attrs.DistributeEnable == LoopAttributes::Enable)))};
|
|
Args.push_back(MDNode::get(Ctx, Vals));
|
|
|
|
if (FollowupHasTransforms)
|
|
Args.push_back(MDNode::get(
|
|
Ctx,
|
|
{MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
|
|
|
|
MDNode *LoopID = MDNode::get(Ctx, Args);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
HasUserTransforms = true;
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
|
|
ArrayRef<Metadata *> LoopProperties,
|
|
bool &HasUserTransforms) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
Optional<bool> Enabled;
|
|
if (Attrs.UnrollEnable == LoopAttributes::Disable)
|
|
Enabled = false;
|
|
else if (Attrs.UnrollEnable == LoopAttributes::Full)
|
|
Enabled = true;
|
|
|
|
if (Enabled != true) {
|
|
SmallVector<Metadata *, 4> NewLoopProperties;
|
|
if (Enabled == false) {
|
|
NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
|
|
NewLoopProperties.push_back(
|
|
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
|
|
LoopProperties = NewLoopProperties;
|
|
}
|
|
return createLoopDistributeMetadata(Attrs, LoopProperties,
|
|
HasUserTransforms);
|
|
}
|
|
|
|
SmallVector<Metadata *, 4> Args;
|
|
TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
|
|
Args.push_back(TempNode.get());
|
|
Args.append(LoopProperties.begin(), LoopProperties.end());
|
|
Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
|
|
|
|
// No follow-up: there is no loop after full unrolling.
|
|
// TODO: Warn if there are transformations after full unrolling.
|
|
|
|
MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
|
|
LoopID->replaceOperandWith(0, LoopID);
|
|
HasUserTransforms = true;
|
|
return LoopID;
|
|
}
|
|
|
|
MDNode *LoopInfo::createMetadata(
|
|
const LoopAttributes &Attrs,
|
|
llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
|
|
bool &HasUserTransforms) {
|
|
SmallVector<Metadata *, 3> LoopProperties;
|
|
|
|
// If we have a valid start debug location for the loop, add it.
|
|
if (StartLoc) {
|
|
LoopProperties.push_back(StartLoc.getAsMDNode());
|
|
|
|
// If we also have a valid end debug location for the loop, add it.
|
|
if (EndLoc)
|
|
LoopProperties.push_back(EndLoc.getAsMDNode());
|
|
}
|
|
|
|
assert(!!AccGroup == Attrs.IsParallel &&
|
|
"There must be an access group iff the loop is parallel");
|
|
if (Attrs.IsParallel) {
|
|
LLVMContext &Ctx = Header->getContext();
|
|
LoopProperties.push_back(MDNode::get(
|
|
Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
|
|
}
|
|
|
|
LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
|
|
AdditionalLoopProperties.end());
|
|
return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
|
|
}
|
|
|
|
LoopAttributes::LoopAttributes(bool IsParallel)
|
|
: IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
|
|
UnrollEnable(LoopAttributes::Unspecified),
|
|
UnrollAndJamEnable(LoopAttributes::Unspecified),
|
|
VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
|
|
InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
|
|
DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
|
|
PipelineInitiationInterval(0) {}
|
|
|
|
void LoopAttributes::clear() {
|
|
IsParallel = false;
|
|
VectorizeWidth = 0;
|
|
InterleaveCount = 0;
|
|
UnrollCount = 0;
|
|
UnrollAndJamCount = 0;
|
|
VectorizeEnable = LoopAttributes::Unspecified;
|
|
UnrollEnable = LoopAttributes::Unspecified;
|
|
UnrollAndJamEnable = LoopAttributes::Unspecified;
|
|
VectorizePredicateEnable = LoopAttributes::Unspecified;
|
|
DistributeEnable = LoopAttributes::Unspecified;
|
|
PipelineDisabled = false;
|
|
PipelineInitiationInterval = 0;
|
|
}
|
|
|
|
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
|
|
const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
|
|
LoopInfo *Parent)
|
|
: Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
|
|
Parent(Parent) {
|
|
|
|
if (Attrs.IsParallel) {
|
|
// Create an access group for this loop.
|
|
LLVMContext &Ctx = Header->getContext();
|
|
AccGroup = MDNode::getDistinct(Ctx, {});
|
|
}
|
|
|
|
if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
|
|
Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
|
|
Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
|
|
Attrs.PipelineInitiationInterval == 0 &&
|
|
Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
|
|
Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
|
|
Attrs.UnrollEnable == LoopAttributes::Unspecified &&
|
|
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
|
|
Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
|
|
!EndLoc)
|
|
return;
|
|
|
|
TempLoopID = MDNode::getTemporary(Header->getContext(), None);
|
|
}
|
|
|
|
void LoopInfo::finish() {
|
|
// We did not annotate the loop body instructions because there are no
|
|
// attributes for this loop.
|
|
if (!TempLoopID)
|
|
return;
|
|
|
|
MDNode *LoopID;
|
|
LoopAttributes CurLoopAttr = Attrs;
|
|
LLVMContext &Ctx = Header->getContext();
|
|
|
|
if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
|
|
Parent->Attrs.UnrollAndJamCount != 0)) {
|
|
// Parent unroll-and-jams this loop.
|
|
// Split the transformations in those that happens before the unroll-and-jam
|
|
// and those after.
|
|
|
|
LoopAttributes BeforeJam, AfterJam;
|
|
|
|
BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
|
|
|
|
BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
|
|
BeforeJam.InterleaveCount = Attrs.InterleaveCount;
|
|
BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
|
|
BeforeJam.DistributeEnable = Attrs.DistributeEnable;
|
|
BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
|
|
|
|
switch (Attrs.UnrollEnable) {
|
|
case LoopAttributes::Unspecified:
|
|
case LoopAttributes::Disable:
|
|
BeforeJam.UnrollEnable = Attrs.UnrollEnable;
|
|
AfterJam.UnrollEnable = Attrs.UnrollEnable;
|
|
break;
|
|
case LoopAttributes::Full:
|
|
BeforeJam.UnrollEnable = LoopAttributes::Full;
|
|
break;
|
|
case LoopAttributes::Enable:
|
|
AfterJam.UnrollEnable = LoopAttributes::Enable;
|
|
break;
|
|
}
|
|
|
|
AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
|
|
AfterJam.UnrollCount = Attrs.UnrollCount;
|
|
AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
|
|
AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
|
|
|
|
// If this loop is subject of an unroll-and-jam by the parent loop, and has
|
|
// an unroll-and-jam annotation itself, we have to decide whether to first
|
|
// apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
|
|
// UnrollAndJam pass processes loops from inner to outer, so we apply the
|
|
// inner first.
|
|
BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
|
|
BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
|
|
|
|
// Set the inner followup metadata to process by the outer loop. Only
|
|
// consider the first inner loop.
|
|
if (!Parent->UnrollAndJamInnerFollowup) {
|
|
// Splitting the attributes into a BeforeJam and an AfterJam part will
|
|
// stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
|
|
// to be forwarded to the AfterJam part. We detect the situation here and
|
|
// add it manually.
|
|
SmallVector<Metadata *, 1> BeforeLoopProperties;
|
|
if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
|
|
BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
|
|
BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
|
|
BeforeLoopProperties.push_back(
|
|
MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
|
|
|
|
bool InnerFollowupHasTransform = false;
|
|
MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
|
|
InnerFollowupHasTransform);
|
|
if (InnerFollowupHasTransform)
|
|
Parent->UnrollAndJamInnerFollowup = InnerFollowup;
|
|
}
|
|
|
|
CurLoopAttr = BeforeJam;
|
|
}
|
|
|
|
bool HasUserTransforms = false;
|
|
LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
|
|
TempLoopID->replaceAllUsesWith(LoopID);
|
|
}
|
|
|
|
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
|
|
const llvm::DebugLoc &EndLoc) {
|
|
Active.emplace_back(
|
|
new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
|
|
Active.empty() ? nullptr : Active.back().get()));
|
|
// Clear the attributes so nested loops do not inherit them.
|
|
StagedAttrs.clear();
|
|
}
|
|
|
|
void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
|
|
const clang::CodeGenOptions &CGOpts,
|
|
ArrayRef<const clang::Attr *> Attrs,
|
|
const llvm::DebugLoc &StartLoc,
|
|
const llvm::DebugLoc &EndLoc) {
|
|
|
|
// Identify loop hint attributes from Attrs.
|
|
for (const auto *Attr : Attrs) {
|
|
const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
|
|
const OpenCLUnrollHintAttr *OpenCLHint =
|
|
dyn_cast<OpenCLUnrollHintAttr>(Attr);
|
|
|
|
// Skip non loop hint attributes
|
|
if (!LH && !OpenCLHint) {
|
|
continue;
|
|
}
|
|
|
|
LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
|
|
LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
|
|
unsigned ValueInt = 1;
|
|
// Translate opencl_unroll_hint attribute argument to
|
|
// equivalent LoopHintAttr enums.
|
|
// OpenCL v2.0 s6.11.5:
|
|
// 0 - enable unroll (no argument).
|
|
// 1 - disable unroll.
|
|
// other positive integer n - unroll by n.
|
|
if (OpenCLHint) {
|
|
ValueInt = OpenCLHint->getUnrollHint();
|
|
if (ValueInt == 0) {
|
|
State = LoopHintAttr::Enable;
|
|
} else if (ValueInt != 1) {
|
|
Option = LoopHintAttr::UnrollCount;
|
|
State = LoopHintAttr::Numeric;
|
|
}
|
|
} else if (LH) {
|
|
auto *ValueExpr = LH->getValue();
|
|
if (ValueExpr) {
|
|
llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
|
|
ValueInt = ValueAPS.getSExtValue();
|
|
}
|
|
|
|
Option = LH->getOption();
|
|
State = LH->getState();
|
|
}
|
|
switch (State) {
|
|
case LoopHintAttr::Disable:
|
|
switch (Option) {
|
|
case LoopHintAttr::Vectorize:
|
|
// Disable vectorization by specifying a width of 1.
|
|
setVectorizeWidth(1);
|
|
break;
|
|
case LoopHintAttr::Interleave:
|
|
// Disable interleaving by speciyfing a count of 1.
|
|
setInterleaveCount(1);
|
|
break;
|
|
case LoopHintAttr::Unroll:
|
|
setUnrollState(LoopAttributes::Disable);
|
|
break;
|
|
case LoopHintAttr::UnrollAndJam:
|
|
setUnrollAndJamState(LoopAttributes::Disable);
|
|
break;
|
|
case LoopHintAttr::VectorizePredicate:
|
|
setVectorizePredicateState(LoopAttributes::Disable);
|
|
break;
|
|
case LoopHintAttr::Distribute:
|
|
setDistributeState(false);
|
|
break;
|
|
case LoopHintAttr::PipelineDisabled:
|
|
setPipelineDisabled(true);
|
|
break;
|
|
case LoopHintAttr::UnrollCount:
|
|
case LoopHintAttr::UnrollAndJamCount:
|
|
case LoopHintAttr::VectorizeWidth:
|
|
case LoopHintAttr::InterleaveCount:
|
|
case LoopHintAttr::PipelineInitiationInterval:
|
|
llvm_unreachable("Options cannot be disabled.");
|
|
break;
|
|
}
|
|
break;
|
|
case LoopHintAttr::Enable:
|
|
switch (Option) {
|
|
case LoopHintAttr::Vectorize:
|
|
case LoopHintAttr::Interleave:
|
|
setVectorizeEnable(true);
|
|
break;
|
|
case LoopHintAttr::Unroll:
|
|
setUnrollState(LoopAttributes::Enable);
|
|
break;
|
|
case LoopHintAttr::UnrollAndJam:
|
|
setUnrollAndJamState(LoopAttributes::Enable);
|
|
break;
|
|
case LoopHintAttr::VectorizePredicate:
|
|
setVectorizePredicateState(LoopAttributes::Enable);
|
|
break;
|
|
case LoopHintAttr::Distribute:
|
|
setDistributeState(true);
|
|
break;
|
|
case LoopHintAttr::UnrollCount:
|
|
case LoopHintAttr::UnrollAndJamCount:
|
|
case LoopHintAttr::VectorizeWidth:
|
|
case LoopHintAttr::InterleaveCount:
|
|
case LoopHintAttr::PipelineDisabled:
|
|
case LoopHintAttr::PipelineInitiationInterval:
|
|
llvm_unreachable("Options cannot enabled.");
|
|
break;
|
|
}
|
|
break;
|
|
case LoopHintAttr::AssumeSafety:
|
|
switch (Option) {
|
|
case LoopHintAttr::Vectorize:
|
|
case LoopHintAttr::Interleave:
|
|
// Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
|
|
setParallel(true);
|
|
setVectorizeEnable(true);
|
|
break;
|
|
case LoopHintAttr::Unroll:
|
|
case LoopHintAttr::UnrollAndJam:
|
|
case LoopHintAttr::VectorizePredicate:
|
|
case LoopHintAttr::UnrollCount:
|
|
case LoopHintAttr::UnrollAndJamCount:
|
|
case LoopHintAttr::VectorizeWidth:
|
|
case LoopHintAttr::InterleaveCount:
|
|
case LoopHintAttr::Distribute:
|
|
case LoopHintAttr::PipelineDisabled:
|
|
case LoopHintAttr::PipelineInitiationInterval:
|
|
llvm_unreachable("Options cannot be used to assume mem safety.");
|
|
break;
|
|
}
|
|
break;
|
|
case LoopHintAttr::Full:
|
|
switch (Option) {
|
|
case LoopHintAttr::Unroll:
|
|
setUnrollState(LoopAttributes::Full);
|
|
break;
|
|
case LoopHintAttr::UnrollAndJam:
|
|
setUnrollAndJamState(LoopAttributes::Full);
|
|
break;
|
|
case LoopHintAttr::Vectorize:
|
|
case LoopHintAttr::Interleave:
|
|
case LoopHintAttr::UnrollCount:
|
|
case LoopHintAttr::UnrollAndJamCount:
|
|
case LoopHintAttr::VectorizeWidth:
|
|
case LoopHintAttr::InterleaveCount:
|
|
case LoopHintAttr::Distribute:
|
|
case LoopHintAttr::PipelineDisabled:
|
|
case LoopHintAttr::PipelineInitiationInterval:
|
|
case LoopHintAttr::VectorizePredicate:
|
|
llvm_unreachable("Options cannot be used with 'full' hint.");
|
|
break;
|
|
}
|
|
break;
|
|
case LoopHintAttr::Numeric:
|
|
switch (Option) {
|
|
case LoopHintAttr::VectorizeWidth:
|
|
setVectorizeWidth(ValueInt);
|
|
break;
|
|
case LoopHintAttr::InterleaveCount:
|
|
setInterleaveCount(ValueInt);
|
|
break;
|
|
case LoopHintAttr::UnrollCount:
|
|
setUnrollCount(ValueInt);
|
|
break;
|
|
case LoopHintAttr::UnrollAndJamCount:
|
|
setUnrollAndJamCount(ValueInt);
|
|
break;
|
|
case LoopHintAttr::PipelineInitiationInterval:
|
|
setPipelineInitiationInterval(ValueInt);
|
|
break;
|
|
case LoopHintAttr::Unroll:
|
|
case LoopHintAttr::UnrollAndJam:
|
|
case LoopHintAttr::VectorizePredicate:
|
|
case LoopHintAttr::Vectorize:
|
|
case LoopHintAttr::Interleave:
|
|
case LoopHintAttr::Distribute:
|
|
case LoopHintAttr::PipelineDisabled:
|
|
llvm_unreachable("Options cannot be assigned a value.");
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (CGOpts.OptimizationLevel > 0)
|
|
// Disable unrolling for the loop, if unrolling is disabled (via
|
|
// -fno-unroll-loops) and no pragmas override the decision.
|
|
if (!CGOpts.UnrollLoops &&
|
|
(StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
|
|
StagedAttrs.UnrollCount == 0))
|
|
setUnrollState(LoopAttributes::Disable);
|
|
|
|
/// Stage the attributes.
|
|
push(Header, StartLoc, EndLoc);
|
|
}
|
|
|
|
void LoopInfoStack::pop() {
|
|
assert(!Active.empty() && "No active loops to pop");
|
|
Active.back()->finish();
|
|
Active.pop_back();
|
|
}
|
|
|
|
void LoopInfoStack::InsertHelper(Instruction *I) const {
|
|
if (I->mayReadOrWriteMemory()) {
|
|
SmallVector<Metadata *, 4> AccessGroups;
|
|
for (const auto &AL : Active) {
|
|
// Here we assume that every loop that has an access group is parallel.
|
|
if (MDNode *Group = AL->getAccessGroup())
|
|
AccessGroups.push_back(Group);
|
|
}
|
|
MDNode *UnionMD = nullptr;
|
|
if (AccessGroups.size() == 1)
|
|
UnionMD = cast<MDNode>(AccessGroups[0]);
|
|
else if (AccessGroups.size() >= 2)
|
|
UnionMD = MDNode::get(I->getContext(), AccessGroups);
|
|
I->setMetadata("llvm.access.group", UnionMD);
|
|
}
|
|
|
|
if (!hasInfo())
|
|
return;
|
|
|
|
const LoopInfo &L = getInfo();
|
|
if (!L.getLoopID())
|
|
return;
|
|
|
|
if (I->isTerminator()) {
|
|
for (BasicBlock *Succ : successors(I))
|
|
if (Succ == L.getHeader()) {
|
|
I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
|
|
break;
|
|
}
|
|
return;
|
|
}
|
|
}
|