2017-02-17 00:00:09 +00:00
|
|
|
//===- Attributes.cpp - Implement AttributesList --------------------------===//
|
2008-01-02 23:42:30 +00:00
|
|
|
//
|
2019-01-19 08:50:56 +00:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2008-01-02 23:42:30 +00:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
2013-01-28 21:55:20 +00:00
|
|
|
// \file
|
2018-05-01 15:54:18 +00:00
|
|
|
// This file implements the Attribute, AttributeImpl, AttrBuilder,
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// AttributeListImpl, and AttributeList classes.
|
2008-01-02 23:42:30 +00:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2017-06-06 11:49:48 +00:00
|
|
|
#include "llvm/IR/Attributes.h"
|
2012-12-20 01:36:59 +00:00
|
|
|
#include "AttributeImpl.h"
|
2012-09-26 21:07:29 +00:00
|
|
|
#include "LLVMContextImpl.h"
|
2017-02-17 00:00:09 +00:00
|
|
|
#include "llvm/ADT/ArrayRef.h"
|
|
|
|
#include "llvm/ADT/FoldingSet.h"
|
|
|
|
#include "llvm/ADT/Optional.h"
|
2014-04-12 16:15:53 +00:00
|
|
|
#include "llvm/ADT/STLExtras.h"
|
2017-06-06 11:49:48 +00:00
|
|
|
#include "llvm/ADT/SmallVector.h"
|
2012-12-03 16:50:05 +00:00
|
|
|
#include "llvm/ADT/StringExtras.h"
|
2017-02-17 00:00:09 +00:00
|
|
|
#include "llvm/ADT/StringRef.h"
|
2020-02-29 10:23:54 -08:00
|
|
|
#include "llvm/ADT/StringSwitch.h"
|
2018-04-30 14:59:11 +00:00
|
|
|
#include "llvm/Config/llvm-config.h"
|
2017-02-17 00:00:09 +00:00
|
|
|
#include "llvm/IR/Function.h"
|
|
|
|
#include "llvm/IR/LLVMContext.h"
|
2013-01-02 11:36:10 +00:00
|
|
|
#include "llvm/IR/Type.h"
|
2017-02-17 00:00:09 +00:00
|
|
|
#include "llvm/Support/Compiler.h"
|
|
|
|
#include "llvm/Support/ErrorHandling.h"
|
2009-08-23 11:37:21 +00:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2013-01-24 00:06:56 +00:00
|
|
|
#include <algorithm>
|
2017-02-17 00:00:09 +00:00
|
|
|
#include <cassert>
|
2017-05-15 21:57:41 +00:00
|
|
|
#include <cstddef>
|
2017-02-17 00:00:09 +00:00
|
|
|
#include <cstdint>
|
|
|
|
#include <limits>
|
|
|
|
#include <string>
|
|
|
|
#include <tuple>
|
|
|
|
#include <utility>
|
|
|
|
|
2008-01-02 23:42:30 +00:00
|
|
|
using namespace llvm;
|
|
|
|
|
2008-03-12 17:45:29 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
2013-01-29 00:48:16 +00:00
|
|
|
// Attribute Construction Methods
|
2008-03-12 17:45:29 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
2008-01-14 19:52:09 +00:00
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
// allocsize has two integer arguments, but because they're both 32 bits, we can
|
|
|
|
// pack them into one 64-bit value, at the cost of making said value
|
|
|
|
// nonsensical.
|
|
|
|
//
|
|
|
|
// In order to do this, we need to reserve one value of the second (optional)
|
|
|
|
// allocsize argument to signify "not present."
|
2016-08-25 01:05:08 +00:00
|
|
|
static const unsigned AllocSizeNumElemsNotPresent = -1;
|
2016-04-12 01:05:35 +00:00
|
|
|
|
|
|
|
static uint64_t packAllocSizeArgs(unsigned ElemSizeArg,
|
|
|
|
const Optional<unsigned> &NumElemsArg) {
|
2022-06-20 11:49:10 -07:00
|
|
|
assert((!NumElemsArg || *NumElemsArg != AllocSizeNumElemsNotPresent) &&
|
2016-04-12 01:05:35 +00:00
|
|
|
"Attempting to pack a reserved value");
|
|
|
|
|
|
|
|
return uint64_t(ElemSizeArg) << 32 |
|
2022-06-18 23:07:11 -07:00
|
|
|
NumElemsArg.value_or(AllocSizeNumElemsNotPresent);
|
2016-04-12 01:05:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static std::pair<unsigned, Optional<unsigned>>
|
|
|
|
unpackAllocSizeArgs(uint64_t Num) {
|
|
|
|
unsigned NumElems = Num & std::numeric_limits<unsigned>::max();
|
|
|
|
unsigned ElemSizeArg = Num >> 32;
|
|
|
|
|
|
|
|
Optional<unsigned> NumElemsArg;
|
|
|
|
if (NumElems != AllocSizeNumElemsNotPresent)
|
|
|
|
NumElemsArg = NumElems;
|
|
|
|
return std::make_pair(ElemSizeArg, NumElemsArg);
|
|
|
|
}
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
static uint64_t packVScaleRangeArgs(unsigned MinValue,
|
|
|
|
Optional<unsigned> MaxValue) {
|
2022-06-18 23:07:11 -07:00
|
|
|
return uint64_t(MinValue) << 32 | MaxValue.value_or(0);
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
static std::pair<unsigned, Optional<unsigned>>
|
|
|
|
unpackVScaleRangeArgs(uint64_t Value) {
|
2021-03-03 13:53:30 +00:00
|
|
|
unsigned MaxValue = Value & std::numeric_limits<unsigned>::max();
|
|
|
|
unsigned MinValue = Value >> 32;
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
return std::make_pair(MinValue,
|
|
|
|
MaxValue > 0 ? MaxValue : Optional<unsigned>());
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
|
|
|
|
uint64_t Val) {
|
2021-07-12 20:54:58 +02:00
|
|
|
if (Val)
|
|
|
|
assert(Attribute::isIntAttrKind(Kind) && "Not an int attribute");
|
|
|
|
else
|
|
|
|
assert(Attribute::isEnumAttrKind(Kind) && "Not an enum attribute");
|
|
|
|
|
2012-10-08 21:47:17 +00:00
|
|
|
LLVMContextImpl *pImpl = Context.pImpl;
|
|
|
|
FoldingSetNodeID ID;
|
2013-02-05 22:37:24 +00:00
|
|
|
ID.AddInteger(Kind);
|
2014-09-03 23:38:05 +00:00
|
|
|
if (Val) ID.AddInteger(Val);
|
2012-10-08 21:47:17 +00:00
|
|
|
|
|
|
|
void *InsertPoint;
|
2012-12-20 01:36:59 +00:00
|
|
|
AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
|
2012-10-08 21:47:17 +00:00
|
|
|
|
|
|
|
if (!PA) {
|
|
|
|
// If we didn't find any existing attributes of the same shape then create a
|
|
|
|
// new one and insert it.
|
2014-09-03 23:38:05 +00:00
|
|
|
if (!Val)
|
2020-05-01 14:12:17 +02:00
|
|
|
PA = new (pImpl->Alloc) EnumAttributeImpl(Kind);
|
2014-09-03 23:38:05 +00:00
|
|
|
else
|
2020-05-01 14:12:17 +02:00
|
|
|
PA = new (pImpl->Alloc) IntAttributeImpl(Kind, Val);
|
2012-10-08 21:47:17 +00:00
|
|
|
pImpl->AttrsSet.InsertNode(PA, InsertPoint);
|
|
|
|
}
|
|
|
|
|
2013-02-05 08:09:32 +00:00
|
|
|
// Return the Attribute that we found or created.
|
2012-12-19 07:18:57 +00:00
|
|
|
return Attribute(PA);
|
2012-10-08 21:47:17 +00:00
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
Attribute Attribute::get(LLVMContext &Context, StringRef Kind, StringRef Val) {
|
|
|
|
LLVMContextImpl *pImpl = Context.pImpl;
|
|
|
|
FoldingSetNodeID ID;
|
|
|
|
ID.AddString(Kind);
|
|
|
|
if (!Val.empty()) ID.AddString(Val);
|
|
|
|
|
|
|
|
void *InsertPoint;
|
|
|
|
AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
|
|
|
|
|
|
|
|
if (!PA) {
|
|
|
|
// If we didn't find any existing attributes of the same shape then create a
|
|
|
|
// new one and insert it.
|
2020-05-01 14:12:17 +02:00
|
|
|
void *Mem =
|
|
|
|
pImpl->Alloc.Allocate(StringAttributeImpl::totalSizeToAlloc(Kind, Val),
|
|
|
|
alignof(StringAttributeImpl));
|
|
|
|
PA = new (Mem) StringAttributeImpl(Kind, Val);
|
2013-02-05 22:37:24 +00:00
|
|
|
pImpl->AttrsSet.InsertNode(PA, InsertPoint);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return the Attribute that we found or created.
|
|
|
|
return Attribute(PA);
|
2013-01-31 23:16:25 +00:00
|
|
|
}
|
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind,
|
|
|
|
Type *Ty) {
|
2021-07-10 18:36:00 +02:00
|
|
|
assert(Attribute::isTypeAttrKind(Kind) && "Not a type attribute");
|
2019-05-30 18:48:23 +00:00
|
|
|
LLVMContextImpl *pImpl = Context.pImpl;
|
|
|
|
FoldingSetNodeID ID;
|
|
|
|
ID.AddInteger(Kind);
|
|
|
|
ID.AddPointer(Ty);
|
|
|
|
|
|
|
|
void *InsertPoint;
|
|
|
|
AttributeImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
|
|
|
|
|
|
|
|
if (!PA) {
|
|
|
|
// If we didn't find any existing attributes of the same shape then create a
|
|
|
|
// new one and insert it.
|
2020-05-01 14:12:17 +02:00
|
|
|
PA = new (pImpl->Alloc) TypeAttributeImpl(Kind, Ty);
|
2019-05-30 18:48:23 +00:00
|
|
|
pImpl->AttrsSet.InsertNode(PA, InsertPoint);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return the Attribute that we found or created.
|
|
|
|
return Attribute(PA);
|
|
|
|
}
|
|
|
|
|
2019-10-15 12:56:24 +00:00
|
|
|
Attribute Attribute::getWithAlignment(LLVMContext &Context, Align A) {
|
[IR] Attribute/AttrBuilder: use Value::MaximumAlignment magic constant
Summary:
I initially encountered those assertions when trying to create
this IR `alignment` attribute from clang's `__attribute__((assume_aligned(imm)))`,
because until D72994 there is no sanity checking for the value of `imm`.
But even then, we have `llvm::Value::MaximumAlignment` constant (which is `536870912`),
which is enforced for clang attributes, and then there are some other magical constant
(`0x40000000` i.e. `1073741824` i.e. `2 * 536870912`) in
`Attribute::getWithAlignment()`/`AttrBuilder::addAlignmentAttr()`.
I strongly suspect that `0x40000000` is incorrect,
and that also should be `llvm::Value::MaximumAlignment`.
Reviewers: erichkeane, hfinkel, jdoerfert, gchatelet, courbet
Reviewed By: erichkeane
Subscribers: hiraditya, cfe-commits, llvm-commits
Tags: #llvm, #clang
Differential Revision: https://reviews.llvm.org/D72998
2020-01-23 22:50:06 +03:00
|
|
|
assert(A <= llvm::Value::MaximumAlignment && "Alignment too large.");
|
2019-10-15 12:56:24 +00:00
|
|
|
return get(Context, Alignment, A.value());
|
2013-01-27 22:43:04 +00:00
|
|
|
}
|
|
|
|
|
2019-10-15 12:56:24 +00:00
|
|
|
Attribute Attribute::getWithStackAlignment(LLVMContext &Context, Align A) {
|
|
|
|
assert(A <= 0x100 && "Alignment too large.");
|
|
|
|
return get(Context, StackAlignment, A.value());
|
2013-01-27 22:43:04 +00:00
|
|
|
}
|
|
|
|
|
2014-07-18 15:51:28 +00:00
|
|
|
Attribute Attribute::getWithDereferenceableBytes(LLVMContext &Context,
|
|
|
|
uint64_t Bytes) {
|
|
|
|
assert(Bytes && "Bytes must be non-zero.");
|
|
|
|
return get(Context, Dereferenceable, Bytes);
|
|
|
|
}
|
|
|
|
|
2015-04-16 20:29:50 +00:00
|
|
|
Attribute Attribute::getWithDereferenceableOrNullBytes(LLVMContext &Context,
|
|
|
|
uint64_t Bytes) {
|
|
|
|
assert(Bytes && "Bytes must be non-zero.");
|
|
|
|
return get(Context, DereferenceableOrNull, Bytes);
|
|
|
|
}
|
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
Attribute Attribute::getWithByValType(LLVMContext &Context, Type *Ty) {
|
|
|
|
return get(Context, ByVal, Ty);
|
|
|
|
}
|
|
|
|
|
2020-09-29 09:33:55 -04:00
|
|
|
Attribute Attribute::getWithStructRetType(LLVMContext &Context, Type *Ty) {
|
|
|
|
return get(Context, StructRet, Ty);
|
|
|
|
}
|
|
|
|
|
IR: Define byref parameter attribute
This allows tracking the in-memory type of a pointer argument to a
function for ABI purposes. This is essentially a stripped down version
of byval to remove some of the stack-copy implications in its
definition.
This includes the base IR changes, and some tests for places where it
should be treated similarly to byval. Codegen support will be in a
future patch.
My original attempt at solving some of these problems was to repurpose
byval with a different address space from the stack. However, it is
technically permitted for the callee to introduce a write to the
argument, although nothing does this in reality. There is also talk of
removing and replacing the byval attribute, so a new attribute would
need to take its place anyway.
This is intended avoid some optimization issues with the current
handling of aggregate arguments, as well as fixes inflexibilty in how
frontends can specify the kernel ABI. The most honest representation
of the amdgpu_kernel convention is to expose all kernel arguments as
loads from constant memory. Today, these are raw, SSA Argument values
and codegen is responsible for turning these into loads.
Background:
There currently isn't a satisfactory way to represent how arguments
for the amdgpu_kernel calling convention are passed. In reality,
arguments are passed in a single, flat, constant memory buffer
implicitly passed to the function. It is also illegal to call this
function in the IR, and this is only ever invoked by a driver of some
kind.
It does not make sense to have a stack passed parameter in this
context as is implied by byval. It is never valid to write to the
kernel arguments, as this would corrupt the inputs seen by other
dispatches of the kernel. These argumets are also not in the same
address space as the stack, so a copy is needed to an alloca. From a
source C-like language, the kernel parameters are invisible.
Semantically, a copy is always required from the constant argument
memory to a mutable variable.
The current clang calling convention lowering emits raw values,
including aggregates into the function argument list, since using
byval would not make sense. This has some unfortunate consequences for
the optimizer. In the aggregate case, we end up with an aggregate
store to alloca, which both SROA and instcombine turn into a store of
each aggregate field. The optimizer never pieces this back together to
see that this is really just a copy from constant memory, so we end up
stuck with expensive stack usage.
This also means the backend dictates the alignment of arguments, and
arbitrarily picks the LLVM IR ABI type alignment. By allowing an
explicit alignment, frontends can make better decisions. For example,
there's real no advantage to an aligment higher than 4, so a frontend
could choose to compact the argument layout. Similarly, there is a
high penalty to using an alignment lower than 4, so a frontend could
opt into more padding for small arguments.
Another design consideration is when it is appropriate to expose the
fact that these arguments are all really passed in adjacent
memory. Currently we have a late IR optimization pass in codegen to
rewrite the kernel argument values into explicit loads to enable
vectorization. In most programs, unrelated argument loads can be
merged together. However, exposing this property directly from the
frontend has some disadvantages. We still need a way to track the
original argument sizes and alignments to report to the driver. I find
using some side-channel, metadata mechanism to track this
unappealing. If the kernel arguments were exposed as a single buffer
to begin with, alias analysis would be unaware that the padding bits
betewen arguments are meaningless. Another family of problems is there
are still some gaps in replacing all of the available parameter
attributes with metadata equivalents once lowered to loads.
The immediate plan is to start using this new attribute to handle all
aggregate argumets for kernels. Long term, it makes sense to migrate
all kernel arguments, including scalars, to be passed indirectly in
the same manner.
Additional context is in D79744.
2020-06-05 16:58:47 -04:00
|
|
|
Attribute Attribute::getWithByRefType(LLVMContext &Context, Type *Ty) {
|
|
|
|
return get(Context, ByRef, Ty);
|
|
|
|
}
|
|
|
|
|
2020-02-14 14:16:53 -08:00
|
|
|
Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) {
|
|
|
|
return get(Context, Preallocated, Ty);
|
|
|
|
}
|
|
|
|
|
2021-03-29 08:42:23 -04:00
|
|
|
Attribute Attribute::getWithInAllocaType(LLVMContext &Context, Type *Ty) {
|
|
|
|
return get(Context, InAlloca, Ty);
|
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
Attribute Attribute::getWithUWTableKind(LLVMContext &Context,
|
|
|
|
UWTableKind Kind) {
|
|
|
|
return get(Context, UWTable, uint64_t(Kind));
|
|
|
|
}
|
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
Attribute
|
|
|
|
Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg,
|
|
|
|
const Optional<unsigned> &NumElemsArg) {
|
|
|
|
assert(!(ElemSizeArg == 0 && NumElemsArg && *NumElemsArg == 0) &&
|
|
|
|
"Invalid allocsize arguments -- given allocsize(0, 0)");
|
|
|
|
return get(Context, AllocSize, packAllocSizeArgs(ElemSizeArg, NumElemsArg));
|
|
|
|
}
|
|
|
|
|
2021-03-03 13:53:30 +00:00
|
|
|
Attribute Attribute::getWithVScaleRangeArgs(LLVMContext &Context,
|
|
|
|
unsigned MinValue,
|
|
|
|
unsigned MaxValue) {
|
|
|
|
return get(Context, VScaleRange, packVScaleRangeArgs(MinValue, MaxValue));
|
|
|
|
}
|
|
|
|
|
2020-02-02 14:46:59 +01:00
|
|
|
Attribute::AttrKind Attribute::getAttrKindFromName(StringRef AttrName) {
|
|
|
|
return StringSwitch<Attribute::AttrKind>(AttrName)
|
|
|
|
#define GET_ATTR_NAMES
|
|
|
|
#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \
|
|
|
|
.Case(#DISPLAY_NAME, Attribute::ENUM_NAME)
|
|
|
|
#include "llvm/IR/Attributes.inc"
|
|
|
|
.Default(Attribute::None);
|
|
|
|
}
|
|
|
|
|
|
|
|
StringRef Attribute::getNameFromAttrKind(Attribute::AttrKind AttrKind) {
|
|
|
|
switch (AttrKind) {
|
|
|
|
#define GET_ATTR_NAMES
|
|
|
|
#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \
|
|
|
|
case Attribute::ENUM_NAME: \
|
|
|
|
return #DISPLAY_NAME;
|
|
|
|
#include "llvm/IR/Attributes.inc"
|
|
|
|
case Attribute::None:
|
|
|
|
return "none";
|
|
|
|
default:
|
|
|
|
llvm_unreachable("invalid Kind");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-11 23:21:44 +01:00
|
|
|
bool Attribute::isExistingAttribute(StringRef Name) {
|
|
|
|
return StringSwitch<bool>(Name)
|
|
|
|
#define GET_ATTR_NAMES
|
|
|
|
#define ATTRIBUTE_ALL(ENUM_NAME, DISPLAY_NAME) .Case(#DISPLAY_NAME, true)
|
|
|
|
#include "llvm/IR/Attributes.inc"
|
|
|
|
.Default(false);
|
|
|
|
}
|
|
|
|
|
2013-01-29 00:48:16 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// Attribute Accessor Methods
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
bool Attribute::isEnumAttribute() const {
|
|
|
|
return pImpl && pImpl->isEnumAttribute();
|
2012-10-05 06:44:41 +00:00
|
|
|
}
|
|
|
|
|
2014-07-18 06:51:55 +00:00
|
|
|
bool Attribute::isIntAttribute() const {
|
|
|
|
return pImpl && pImpl->isIntAttribute();
|
2013-01-29 20:45:34 +00:00
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
bool Attribute::isStringAttribute() const {
|
|
|
|
return pImpl && pImpl->isStringAttribute();
|
|
|
|
}
|
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
bool Attribute::isTypeAttribute() const {
|
|
|
|
return pImpl && pImpl->isTypeAttribute();
|
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
Attribute::AttrKind Attribute::getKindAsEnum() const {
|
2013-07-25 00:34:29 +00:00
|
|
|
if (!pImpl) return None;
|
2019-05-30 18:48:23 +00:00
|
|
|
assert((isEnumAttribute() || isIntAttribute() || isTypeAttribute()) &&
|
2013-02-05 22:37:24 +00:00
|
|
|
"Invalid attribute type to get the kind as an enum!");
|
2015-12-16 05:21:02 +00:00
|
|
|
return pImpl->getKindAsEnum();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t Attribute::getValueAsInt() const {
|
2013-07-25 00:34:29 +00:00
|
|
|
if (!pImpl) return 0;
|
2014-07-18 06:51:55 +00:00
|
|
|
assert(isIntAttribute() &&
|
|
|
|
"Expected the attribute to be an integer attribute!");
|
2015-12-16 05:21:02 +00:00
|
|
|
return pImpl->getValueAsInt();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
|
|
|
|
2021-03-24 16:45:04 -04:00
|
|
|
bool Attribute::getValueAsBool() const {
|
|
|
|
if (!pImpl) return false;
|
|
|
|
assert(isStringAttribute() &&
|
|
|
|
"Expected the attribute to be a string attribute!");
|
|
|
|
return pImpl->getValueAsBool();
|
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
StringRef Attribute::getKindAsString() const {
|
2018-03-30 00:47:31 +00:00
|
|
|
if (!pImpl) return {};
|
2013-02-05 22:37:24 +00:00
|
|
|
assert(isStringAttribute() &&
|
|
|
|
"Invalid attribute type to get the kind as a string!");
|
2015-12-16 05:21:02 +00:00
|
|
|
return pImpl->getKindAsString();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
StringRef Attribute::getValueAsString() const {
|
2018-03-30 00:47:31 +00:00
|
|
|
if (!pImpl) return {};
|
2013-02-05 22:37:24 +00:00
|
|
|
assert(isStringAttribute() &&
|
|
|
|
"Invalid attribute type to get the value as a string!");
|
2015-12-16 05:21:02 +00:00
|
|
|
return pImpl->getValueAsString();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
Type *Attribute::getValueAsType() const {
|
|
|
|
if (!pImpl) return {};
|
|
|
|
assert(isTypeAttribute() &&
|
|
|
|
"Invalid attribute type to get the value as a type!");
|
|
|
|
return pImpl->getValueAsType();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2013-02-05 23:48:36 +00:00
|
|
|
bool Attribute::hasAttribute(AttrKind Kind) const {
|
|
|
|
return (pImpl && pImpl->hasAttribute(Kind)) || (!pImpl && Kind == None);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Attribute::hasAttribute(StringRef Kind) const {
|
|
|
|
if (!isStringAttribute()) return false;
|
|
|
|
return pImpl && pImpl->hasAttribute(Kind);
|
2013-01-29 20:45:34 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign Attribute::getAlignment() const {
|
2013-02-01 01:04:27 +00:00
|
|
|
assert(hasAttribute(Attribute::Alignment) &&
|
|
|
|
"Trying to get alignment from non-alignment attribute!");
|
2019-10-22 09:51:06 +00:00
|
|
|
return MaybeAlign(pImpl->getValueAsInt());
|
2012-10-05 06:44:41 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign Attribute::getStackAlignment() const {
|
2013-02-01 01:04:27 +00:00
|
|
|
assert(hasAttribute(Attribute::StackAlignment) &&
|
|
|
|
"Trying to get alignment from non-alignment attribute!");
|
2019-10-22 09:51:06 +00:00
|
|
|
return MaybeAlign(pImpl->getValueAsInt());
|
2012-10-05 06:44:41 +00:00
|
|
|
}
|
|
|
|
|
2014-07-18 15:51:28 +00:00
|
|
|
uint64_t Attribute::getDereferenceableBytes() const {
|
|
|
|
assert(hasAttribute(Attribute::Dereferenceable) &&
|
|
|
|
"Trying to get dereferenceable bytes from "
|
|
|
|
"non-dereferenceable attribute!");
|
|
|
|
return pImpl->getValueAsInt();
|
|
|
|
}
|
|
|
|
|
2015-04-16 20:29:50 +00:00
|
|
|
uint64_t Attribute::getDereferenceableOrNullBytes() const {
|
|
|
|
assert(hasAttribute(Attribute::DereferenceableOrNull) &&
|
|
|
|
"Trying to get dereferenceable bytes from "
|
|
|
|
"non-dereferenceable attribute!");
|
|
|
|
return pImpl->getValueAsInt();
|
|
|
|
}
|
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
std::pair<unsigned, Optional<unsigned>> Attribute::getAllocSizeArgs() const {
|
|
|
|
assert(hasAttribute(Attribute::AllocSize) &&
|
|
|
|
"Trying to get allocsize args from non-allocsize attribute");
|
|
|
|
return unpackAllocSizeArgs(pImpl->getValueAsInt());
|
|
|
|
}
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
unsigned Attribute::getVScaleRangeMin() const {
|
2021-03-03 13:53:30 +00:00
|
|
|
assert(hasAttribute(Attribute::VScaleRange) &&
|
|
|
|
"Trying to get vscale args from non-vscale attribute");
|
2021-12-07 09:53:16 +00:00
|
|
|
return unpackVScaleRangeArgs(pImpl->getValueAsInt()).first;
|
|
|
|
}
|
|
|
|
|
|
|
|
Optional<unsigned> Attribute::getVScaleRangeMax() const {
|
|
|
|
assert(hasAttribute(Attribute::VScaleRange) &&
|
|
|
|
"Trying to get vscale args from non-vscale attribute");
|
|
|
|
return unpackVScaleRangeArgs(pImpl->getValueAsInt()).second;
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
UWTableKind Attribute::getUWTableKind() const {
|
|
|
|
assert(hasAttribute(Attribute::UWTable) &&
|
|
|
|
"Trying to get unwind table kind from non-uwtable attribute");
|
|
|
|
return UWTableKind(pImpl->getValueAsInt());
|
|
|
|
}
|
|
|
|
|
2022-03-29 11:14:07 -04:00
|
|
|
AllocFnKind Attribute::getAllocKind() const {
|
|
|
|
assert(hasAttribute(Attribute::AllocKind) &&
|
|
|
|
"Trying to get allockind value from non-allockind attribute");
|
|
|
|
return AllocFnKind(pImpl->getValueAsInt());
|
|
|
|
}
|
|
|
|
|
2013-02-11 08:43:33 +00:00
|
|
|
std::string Attribute::getAsString(bool InAttrGrp) const {
|
2018-03-30 00:47:31 +00:00
|
|
|
if (!pImpl) return {};
|
2013-01-31 20:59:05 +00:00
|
|
|
|
2021-07-07 22:41:12 +02:00
|
|
|
if (isEnumAttribute())
|
|
|
|
return getNameFromAttrKind(getKindAsEnum()).str();
|
2013-01-31 20:59:05 +00:00
|
|
|
|
2021-03-29 08:42:23 -04:00
|
|
|
if (isTypeAttribute()) {
|
2021-07-07 22:41:12 +02:00
|
|
|
std::string Result = getNameFromAttrKind(getKindAsEnum()).str();
|
2020-02-14 14:16:53 -08:00
|
|
|
Result += '(';
|
2021-07-07 22:41:12 +02:00
|
|
|
raw_string_ostream OS(Result);
|
2020-02-14 14:16:53 -08:00
|
|
|
getValueAsType()->print(OS, false, true);
|
|
|
|
OS.flush();
|
|
|
|
Result += ')';
|
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
|
2013-01-31 20:59:05 +00:00
|
|
|
// FIXME: These should be output like this:
|
|
|
|
//
|
|
|
|
// align=4
|
|
|
|
// alignstack=8
|
|
|
|
//
|
2022-01-31 10:48:14 +00:00
|
|
|
if (hasAttribute(Attribute::Alignment))
|
|
|
|
return (InAttrGrp ? "align=" + Twine(getValueAsInt())
|
|
|
|
: "align " + Twine(getValueAsInt()))
|
|
|
|
.str();
|
2013-02-11 08:43:33 +00:00
|
|
|
|
2015-04-16 20:29:50 +00:00
|
|
|
auto AttrWithBytesToString = [&](const char *Name) {
|
2022-01-31 10:48:14 +00:00
|
|
|
return (InAttrGrp ? Name + ("=" + Twine(getValueAsInt()))
|
|
|
|
: Name + ("(" + Twine(getValueAsInt())) + ")")
|
|
|
|
.str();
|
2015-04-16 20:29:50 +00:00
|
|
|
};
|
2013-01-31 20:59:05 +00:00
|
|
|
|
2015-04-16 20:29:50 +00:00
|
|
|
if (hasAttribute(Attribute::StackAlignment))
|
|
|
|
return AttrWithBytesToString("alignstack");
|
|
|
|
|
|
|
|
if (hasAttribute(Attribute::Dereferenceable))
|
|
|
|
return AttrWithBytesToString("dereferenceable");
|
|
|
|
|
|
|
|
if (hasAttribute(Attribute::DereferenceableOrNull))
|
|
|
|
return AttrWithBytesToString("dereferenceable_or_null");
|
2014-07-18 15:51:28 +00:00
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
if (hasAttribute(Attribute::AllocSize)) {
|
|
|
|
unsigned ElemSize;
|
|
|
|
Optional<unsigned> NumElems;
|
|
|
|
std::tie(ElemSize, NumElems) = getAllocSizeArgs();
|
|
|
|
|
2022-01-31 10:48:14 +00:00
|
|
|
return (NumElems
|
|
|
|
? "allocsize(" + Twine(ElemSize) + "," + Twine(*NumElems) + ")"
|
|
|
|
: "allocsize(" + Twine(ElemSize) + ")")
|
|
|
|
.str();
|
2016-04-12 01:05:35 +00:00
|
|
|
}
|
|
|
|
|
2021-03-03 13:53:30 +00:00
|
|
|
if (hasAttribute(Attribute::VScaleRange)) {
|
2021-12-07 09:53:16 +00:00
|
|
|
unsigned MinValue = getVScaleRangeMin();
|
|
|
|
Optional<unsigned> MaxValue = getVScaleRangeMax();
|
2022-01-31 10:48:14 +00:00
|
|
|
return ("vscale_range(" + Twine(MinValue) + "," +
|
2022-06-18 23:07:11 -07:00
|
|
|
Twine(MaxValue.value_or(0)) + ")")
|
2022-01-31 10:48:14 +00:00
|
|
|
.str();
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
if (hasAttribute(Attribute::UWTable)) {
|
|
|
|
UWTableKind Kind = getUWTableKind();
|
|
|
|
if (Kind != UWTableKind::None) {
|
|
|
|
return Kind == UWTableKind::Default
|
|
|
|
? "uwtable"
|
|
|
|
: ("uwtable(" +
|
|
|
|
Twine(Kind == UWTableKind::Sync ? "sync" : "async") + ")")
|
|
|
|
.str();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-29 11:14:07 -04:00
|
|
|
if (hasAttribute(Attribute::AllocKind)) {
|
|
|
|
AllocFnKind Kind = getAllocKind();
|
|
|
|
SmallVector<StringRef> parts;
|
|
|
|
if ((Kind & AllocFnKind::Alloc) != AllocFnKind::Unknown)
|
|
|
|
parts.push_back("alloc");
|
|
|
|
if ((Kind & AllocFnKind::Realloc) != AllocFnKind::Unknown)
|
|
|
|
parts.push_back("realloc");
|
|
|
|
if ((Kind & AllocFnKind::Free) != AllocFnKind::Unknown)
|
|
|
|
parts.push_back("free");
|
|
|
|
if ((Kind & AllocFnKind::Uninitialized) != AllocFnKind::Unknown)
|
|
|
|
parts.push_back("uninitialized");
|
|
|
|
if ((Kind & AllocFnKind::Zeroed) != AllocFnKind::Unknown)
|
|
|
|
parts.push_back("zeroed");
|
|
|
|
if ((Kind & AllocFnKind::Aligned) != AllocFnKind::Unknown)
|
|
|
|
parts.push_back("aligned");
|
|
|
|
return ("allockind(\"" +
|
|
|
|
Twine(llvm::join(parts.begin(), parts.end(), ",")) + "\")")
|
|
|
|
.str();
|
|
|
|
}
|
|
|
|
|
2013-01-31 20:59:05 +00:00
|
|
|
// Convert target-dependent attributes to strings of the form:
|
|
|
|
//
|
|
|
|
// "kind"
|
|
|
|
// "kind" = "value"
|
|
|
|
//
|
2013-02-05 22:37:24 +00:00
|
|
|
if (isStringAttribute()) {
|
2013-01-31 20:59:05 +00:00
|
|
|
std::string Result;
|
[IR] Properly handle escape characters in Attribute::getAsString()
If an attribute name has special characters such as '\01', it is not
properly printed in LLVM assembly language format. Since the format
expects the special characters are printed as it is, it has to contain
escape characters to make it printable.
Before:
attributes #0 = { ... "counting-function"="^A__gnu_mcount_nc" ...
After:
attributes #0 = { ... "counting-function"="\01__gnu_mcount_nc" ...
Reviewers: hfinkel, rengolin, rjmccall, compnerd
Subscribers: nemanjai, mcrosier, hans, shenhan, majnemer, llvm-commits
Differential Revision: https://reviews.llvm.org/D23792
llvm-svn: 280357
2016-09-01 11:44:06 +00:00
|
|
|
{
|
|
|
|
raw_string_ostream OS(Result);
|
2020-04-26 13:06:50 +02:00
|
|
|
OS << '"' << getKindAsString() << '"';
|
|
|
|
|
|
|
|
// Since some attribute strings contain special characters that cannot be
|
|
|
|
// printable, those have to be escaped to make the attribute value
|
|
|
|
// printable as is. e.g. "\01__gnu_mcount_nc"
|
|
|
|
const auto &AttrVal = pImpl->getValueAsString();
|
|
|
|
if (!AttrVal.empty()) {
|
|
|
|
OS << "=\"";
|
|
|
|
printEscapedString(AttrVal, OS);
|
|
|
|
OS << "\"";
|
|
|
|
}
|
[IR] Properly handle escape characters in Attribute::getAsString()
If an attribute name has special characters such as '\01', it is not
properly printed in LLVM assembly language format. Since the format
expects the special characters are printed as it is, it has to contain
escape characters to make it printable.
Before:
attributes #0 = { ... "counting-function"="^A__gnu_mcount_nc" ...
After:
attributes #0 = { ... "counting-function"="\01__gnu_mcount_nc" ...
Reviewers: hfinkel, rengolin, rjmccall, compnerd
Subscribers: nemanjai, mcrosier, hans, shenhan, majnemer, llvm-commits
Differential Revision: https://reviews.llvm.org/D23792
llvm-svn: 280357
2016-09-01 11:44:06 +00:00
|
|
|
}
|
2013-02-01 01:04:27 +00:00
|
|
|
return Result;
|
2013-01-31 20:59:05 +00:00
|
|
|
}
|
2013-01-29 03:20:31 +00:00
|
|
|
|
|
|
|
llvm_unreachable("Unknown attribute");
|
2008-01-02 23:42:30 +00:00
|
|
|
}
|
|
|
|
|
2021-03-24 16:13:29 -07:00
|
|
|
bool Attribute::hasParentContext(LLVMContext &C) const {
|
|
|
|
assert(isValid() && "invalid Attribute doesn't refer to any context");
|
|
|
|
FoldingSetNodeID ID;
|
|
|
|
pImpl->Profile(ID);
|
|
|
|
void *Unused;
|
|
|
|
return C.pImpl->AttrsSet.FindNodeOrInsertPos(ID, Unused) == pImpl;
|
|
|
|
}
|
|
|
|
|
2013-01-29 00:34:06 +00:00
|
|
|
bool Attribute::operator<(Attribute A) const {
|
|
|
|
if (!pImpl && !A.pImpl) return false;
|
|
|
|
if (!pImpl) return true;
|
|
|
|
if (!A.pImpl) return false;
|
|
|
|
return *pImpl < *A.pImpl;
|
2012-10-08 23:27:46 +00:00
|
|
|
}
|
|
|
|
|
2020-01-17 16:04:02 -08:00
|
|
|
void Attribute::Profile(FoldingSetNodeID &ID) const {
|
|
|
|
ID.AddPointer(pImpl);
|
|
|
|
}
|
|
|
|
|
2021-07-11 16:54:03 +02:00
|
|
|
enum AttributeProperty {
|
|
|
|
FnAttr = (1 << 0),
|
|
|
|
ParamAttr = (1 << 1),
|
|
|
|
RetAttr = (1 << 2),
|
|
|
|
};
|
|
|
|
|
|
|
|
#define GET_ATTR_PROP_TABLE
|
|
|
|
#include "llvm/IR/Attributes.inc"
|
|
|
|
|
|
|
|
static bool hasAttributeProperty(Attribute::AttrKind Kind,
|
|
|
|
AttributeProperty Prop) {
|
|
|
|
unsigned Index = Kind - 1;
|
|
|
|
assert(Index < sizeof(AttrPropTable) / sizeof(AttrPropTable[0]) &&
|
|
|
|
"Invalid attribute kind");
|
|
|
|
return AttrPropTable[Index] & Prop;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Attribute::canUseAsFnAttr(AttrKind Kind) {
|
|
|
|
return hasAttributeProperty(Kind, AttributeProperty::FnAttr);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Attribute::canUseAsParamAttr(AttrKind Kind) {
|
|
|
|
return hasAttributeProperty(Kind, AttributeProperty::ParamAttr);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Attribute::canUseAsRetAttr(AttrKind Kind) {
|
|
|
|
return hasAttributeProperty(Kind, AttributeProperty::RetAttr);
|
|
|
|
}
|
|
|
|
|
2012-09-26 21:07:29 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// AttributeImpl Definition
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
bool AttributeImpl::hasAttribute(Attribute::AttrKind A) const {
|
|
|
|
if (isStringAttribute()) return false;
|
|
|
|
return getKindAsEnum() == A;
|
2012-12-30 01:38:39 +00:00
|
|
|
}
|
2013-01-24 00:06:56 +00:00
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
bool AttributeImpl::hasAttribute(StringRef Kind) const {
|
|
|
|
if (!isStringAttribute()) return false;
|
|
|
|
return getKindAsString() == Kind;
|
2013-01-04 20:54:35 +00:00
|
|
|
}
|
2012-12-30 01:38:39 +00:00
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
Attribute::AttrKind AttributeImpl::getKindAsEnum() const {
|
2019-05-30 18:48:23 +00:00
|
|
|
assert(isEnumAttribute() || isIntAttribute() || isTypeAttribute());
|
2013-07-11 12:13:16 +00:00
|
|
|
return static_cast<const EnumAttributeImpl *>(this)->getEnumKind();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
2013-02-01 01:04:27 +00:00
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
uint64_t AttributeImpl::getValueAsInt() const {
|
2014-07-18 06:51:55 +00:00
|
|
|
assert(isIntAttribute());
|
|
|
|
return static_cast<const IntAttributeImpl *>(this)->getValue();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
2013-01-24 00:06:56 +00:00
|
|
|
|
2021-03-24 16:45:04 -04:00
|
|
|
bool AttributeImpl::getValueAsBool() const {
|
|
|
|
assert(getValueAsString().empty() || getValueAsString() == "false" || getValueAsString() == "true");
|
|
|
|
return getValueAsString() == "true";
|
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
StringRef AttributeImpl::getKindAsString() const {
|
2013-07-11 12:13:16 +00:00
|
|
|
assert(isStringAttribute());
|
|
|
|
return static_cast<const StringAttributeImpl *>(this)->getStringKind();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
2013-01-24 00:06:56 +00:00
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
StringRef AttributeImpl::getValueAsString() const {
|
2013-07-11 12:13:16 +00:00
|
|
|
assert(isStringAttribute());
|
|
|
|
return static_cast<const StringAttributeImpl *>(this)->getStringValue();
|
2013-02-05 22:37:24 +00:00
|
|
|
}
|
2013-01-24 00:06:56 +00:00
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
Type *AttributeImpl::getValueAsType() const {
|
|
|
|
assert(isTypeAttribute());
|
|
|
|
return static_cast<const TypeAttributeImpl *>(this)->getTypeValue();
|
|
|
|
}
|
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
bool AttributeImpl::operator<(const AttributeImpl &AI) const {
|
2020-06-23 14:43:02 +03:00
|
|
|
if (this == &AI)
|
|
|
|
return false;
|
2021-07-12 21:08:41 +02:00
|
|
|
|
2013-02-05 22:37:24 +00:00
|
|
|
// This sorts the attributes with Attribute::AttrKinds coming first (sorted
|
|
|
|
// relative to their enum value) and then strings.
|
2021-07-12 21:08:41 +02:00
|
|
|
if (!isStringAttribute()) {
|
|
|
|
if (AI.isStringAttribute())
|
|
|
|
return true;
|
|
|
|
if (getKindAsEnum() != AI.getKindAsEnum())
|
2019-05-30 18:48:23 +00:00
|
|
|
return getKindAsEnum() < AI.getKindAsEnum();
|
2021-07-12 21:08:41 +02:00
|
|
|
assert(!AI.isEnumAttribute() && "Non-unique attribute");
|
|
|
|
assert(!AI.isTypeAttribute() && "Comparison of types would be unstable");
|
|
|
|
// TODO: Is this actually needed?
|
|
|
|
assert(AI.isIntAttribute() && "Only possibility left");
|
|
|
|
return getValueAsInt() < AI.getValueAsInt();
|
2013-02-15 04:15:55 +00:00
|
|
|
}
|
|
|
|
|
2021-07-12 21:08:41 +02:00
|
|
|
if (!AI.isStringAttribute())
|
|
|
|
return false;
|
2013-02-15 05:25:26 +00:00
|
|
|
if (getKindAsString() == AI.getKindAsString())
|
|
|
|
return getValueAsString() < AI.getValueAsString();
|
|
|
|
return getKindAsString() < AI.getKindAsString();
|
2013-01-24 00:06:56 +00:00
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// AttributeSet Definition
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::get(LLVMContext &C, const AttrBuilder &B) {
|
|
|
|
return AttributeSet(AttributeSetNode::get(C, B));
|
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::get(LLVMContext &C, ArrayRef<Attribute> Attrs) {
|
|
|
|
return AttributeSet(AttributeSetNode::get(C, Attrs));
|
|
|
|
}
|
|
|
|
|
2017-05-11 12:28:08 +00:00
|
|
|
AttributeSet AttributeSet::addAttribute(LLVMContext &C,
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
Attribute::AttrKind Kind) const {
|
2017-05-11 12:28:08 +00:00
|
|
|
if (hasAttribute(Kind)) return *this;
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2017-05-11 12:28:08 +00:00
|
|
|
B.addAttribute(Kind);
|
|
|
|
return addAttributes(C, AttributeSet::get(C, B));
|
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::addAttribute(LLVMContext &C, StringRef Kind,
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
StringRef Value) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2017-05-11 12:28:08 +00:00
|
|
|
B.addAttribute(Kind, Value);
|
|
|
|
return addAttributes(C, AttributeSet::get(C, B));
|
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::addAttributes(LLVMContext &C,
|
|
|
|
const AttributeSet AS) const {
|
|
|
|
if (!hasAttributes())
|
|
|
|
return AS;
|
|
|
|
|
|
|
|
if (!AS.hasAttributes())
|
|
|
|
return *this;
|
|
|
|
|
[Attributes] Make attribute addition behavior consistent
Currently, the behavior when adding an attribute with the same key
as an existing attribute is inconsistent, depending on the type of
the attribute and the method used to add it. When going through
AttrBuilder::addAttribute(), the new attribute always overwrites
the old one. When going through AttrBuilder::merge() the new
attribute overwrites the existing one if it is a string attribute,
but keeps the existing one for int and type attributes. One
particular API also asserts that you can't overwrite an align
attribute, but does not handle any of the other int, type or string
attributes.
This patch makes the behavior consistent by always overwriting with
the new attribute, which is the behavior I would intuitively expect.
Two tests are affected, which now make a different (but equally
valid) choice. Those tests could be improved by taking the maximum
deref bytes, but I haven't bothered with that, since this is testing
a degenerate case -- the important bit is that it doesn't crash.
Differential Revision: https://reviews.llvm.org/D117552
2022-01-18 11:55:28 +01:00
|
|
|
AttrBuilder B(C, *this);
|
|
|
|
B.merge(AttrBuilder(C, AS));
|
|
|
|
return get(C, B);
|
2017-05-11 12:28:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::removeAttribute(LLVMContext &C,
|
|
|
|
Attribute::AttrKind Kind) const {
|
|
|
|
if (!hasAttribute(Kind)) return *this;
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C, *this);
|
2018-01-17 19:15:21 +00:00
|
|
|
B.removeAttribute(Kind);
|
|
|
|
return get(C, B);
|
2017-05-11 12:28:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::removeAttribute(LLVMContext &C,
|
|
|
|
StringRef Kind) const {
|
|
|
|
if (!hasAttribute(Kind)) return *this;
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C, *this);
|
2018-01-17 19:15:21 +00:00
|
|
|
B.removeAttribute(Kind);
|
|
|
|
return get(C, B);
|
2017-05-11 12:28:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet AttributeSet::removeAttributes(LLVMContext &C,
|
2022-01-04 09:44:47 +01:00
|
|
|
const AttributeMask &Attrs) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C, *this);
|
2021-05-22 18:53:17 +02:00
|
|
|
// If there is nothing to remove, directly return the original set.
|
|
|
|
if (!B.overlaps(Attrs))
|
|
|
|
return *this;
|
|
|
|
|
2017-05-11 12:28:08 +00:00
|
|
|
B.remove(Attrs);
|
|
|
|
return get(C, B);
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
unsigned AttributeSet::getNumAttributes() const {
|
|
|
|
return SetNode ? SetNode->getNumAttributes() : 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool AttributeSet::hasAttribute(Attribute::AttrKind Kind) const {
|
2017-05-15 21:57:41 +00:00
|
|
|
return SetNode ? SetNode->hasAttribute(Kind) : false;
|
2017-04-12 00:38:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool AttributeSet::hasAttribute(StringRef Kind) const {
|
2017-05-15 21:57:41 +00:00
|
|
|
return SetNode ? SetNode->hasAttribute(Kind) : false;
|
2017-04-12 00:38:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Attribute AttributeSet::getAttribute(Attribute::AttrKind Kind) const {
|
|
|
|
return SetNode ? SetNode->getAttribute(Kind) : Attribute();
|
|
|
|
}
|
|
|
|
|
|
|
|
Attribute AttributeSet::getAttribute(StringRef Kind) const {
|
|
|
|
return SetNode ? SetNode->getAttribute(Kind) : Attribute();
|
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign AttributeSet::getAlignment() const {
|
|
|
|
return SetNode ? SetNode->getAlignment() : None;
|
2017-04-12 00:38:00 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign AttributeSet::getStackAlignment() const {
|
|
|
|
return SetNode ? SetNode->getStackAlignment() : None;
|
2017-04-12 00:38:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t AttributeSet::getDereferenceableBytes() const {
|
|
|
|
return SetNode ? SetNode->getDereferenceableBytes() : 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t AttributeSet::getDereferenceableOrNullBytes() const {
|
|
|
|
return SetNode ? SetNode->getDereferenceableOrNullBytes() : 0;
|
|
|
|
}
|
|
|
|
|
IR: Define byref parameter attribute
This allows tracking the in-memory type of a pointer argument to a
function for ABI purposes. This is essentially a stripped down version
of byval to remove some of the stack-copy implications in its
definition.
This includes the base IR changes, and some tests for places where it
should be treated similarly to byval. Codegen support will be in a
future patch.
My original attempt at solving some of these problems was to repurpose
byval with a different address space from the stack. However, it is
technically permitted for the callee to introduce a write to the
argument, although nothing does this in reality. There is also talk of
removing and replacing the byval attribute, so a new attribute would
need to take its place anyway.
This is intended avoid some optimization issues with the current
handling of aggregate arguments, as well as fixes inflexibilty in how
frontends can specify the kernel ABI. The most honest representation
of the amdgpu_kernel convention is to expose all kernel arguments as
loads from constant memory. Today, these are raw, SSA Argument values
and codegen is responsible for turning these into loads.
Background:
There currently isn't a satisfactory way to represent how arguments
for the amdgpu_kernel calling convention are passed. In reality,
arguments are passed in a single, flat, constant memory buffer
implicitly passed to the function. It is also illegal to call this
function in the IR, and this is only ever invoked by a driver of some
kind.
It does not make sense to have a stack passed parameter in this
context as is implied by byval. It is never valid to write to the
kernel arguments, as this would corrupt the inputs seen by other
dispatches of the kernel. These argumets are also not in the same
address space as the stack, so a copy is needed to an alloca. From a
source C-like language, the kernel parameters are invisible.
Semantically, a copy is always required from the constant argument
memory to a mutable variable.
The current clang calling convention lowering emits raw values,
including aggregates into the function argument list, since using
byval would not make sense. This has some unfortunate consequences for
the optimizer. In the aggregate case, we end up with an aggregate
store to alloca, which both SROA and instcombine turn into a store of
each aggregate field. The optimizer never pieces this back together to
see that this is really just a copy from constant memory, so we end up
stuck with expensive stack usage.
This also means the backend dictates the alignment of arguments, and
arbitrarily picks the LLVM IR ABI type alignment. By allowing an
explicit alignment, frontends can make better decisions. For example,
there's real no advantage to an aligment higher than 4, so a frontend
could choose to compact the argument layout. Similarly, there is a
high penalty to using an alignment lower than 4, so a frontend could
opt into more padding for small arguments.
Another design consideration is when it is appropriate to expose the
fact that these arguments are all really passed in adjacent
memory. Currently we have a late IR optimization pass in codegen to
rewrite the kernel argument values into explicit loads to enable
vectorization. In most programs, unrelated argument loads can be
merged together. However, exposing this property directly from the
frontend has some disadvantages. We still need a way to track the
original argument sizes and alignments to report to the driver. I find
using some side-channel, metadata mechanism to track this
unappealing. If the kernel arguments were exposed as a single buffer
to begin with, alias analysis would be unaware that the padding bits
betewen arguments are meaningless. Another family of problems is there
are still some gaps in replacing all of the available parameter
attributes with metadata equivalents once lowered to loads.
The immediate plan is to start using this new attribute to handle all
aggregate argumets for kernels. Long term, it makes sense to migrate
all kernel arguments, including scalars, to be passed indirectly in
the same manner.
Additional context is in D79744.
2020-06-05 16:58:47 -04:00
|
|
|
Type *AttributeSet::getByRefType() const {
|
2021-07-14 21:09:06 +02:00
|
|
|
return SetNode ? SetNode->getAttributeType(Attribute::ByRef) : nullptr;
|
IR: Define byref parameter attribute
This allows tracking the in-memory type of a pointer argument to a
function for ABI purposes. This is essentially a stripped down version
of byval to remove some of the stack-copy implications in its
definition.
This includes the base IR changes, and some tests for places where it
should be treated similarly to byval. Codegen support will be in a
future patch.
My original attempt at solving some of these problems was to repurpose
byval with a different address space from the stack. However, it is
technically permitted for the callee to introduce a write to the
argument, although nothing does this in reality. There is also talk of
removing and replacing the byval attribute, so a new attribute would
need to take its place anyway.
This is intended avoid some optimization issues with the current
handling of aggregate arguments, as well as fixes inflexibilty in how
frontends can specify the kernel ABI. The most honest representation
of the amdgpu_kernel convention is to expose all kernel arguments as
loads from constant memory. Today, these are raw, SSA Argument values
and codegen is responsible for turning these into loads.
Background:
There currently isn't a satisfactory way to represent how arguments
for the amdgpu_kernel calling convention are passed. In reality,
arguments are passed in a single, flat, constant memory buffer
implicitly passed to the function. It is also illegal to call this
function in the IR, and this is only ever invoked by a driver of some
kind.
It does not make sense to have a stack passed parameter in this
context as is implied by byval. It is never valid to write to the
kernel arguments, as this would corrupt the inputs seen by other
dispatches of the kernel. These argumets are also not in the same
address space as the stack, so a copy is needed to an alloca. From a
source C-like language, the kernel parameters are invisible.
Semantically, a copy is always required from the constant argument
memory to a mutable variable.
The current clang calling convention lowering emits raw values,
including aggregates into the function argument list, since using
byval would not make sense. This has some unfortunate consequences for
the optimizer. In the aggregate case, we end up with an aggregate
store to alloca, which both SROA and instcombine turn into a store of
each aggregate field. The optimizer never pieces this back together to
see that this is really just a copy from constant memory, so we end up
stuck with expensive stack usage.
This also means the backend dictates the alignment of arguments, and
arbitrarily picks the LLVM IR ABI type alignment. By allowing an
explicit alignment, frontends can make better decisions. For example,
there's real no advantage to an aligment higher than 4, so a frontend
could choose to compact the argument layout. Similarly, there is a
high penalty to using an alignment lower than 4, so a frontend could
opt into more padding for small arguments.
Another design consideration is when it is appropriate to expose the
fact that these arguments are all really passed in adjacent
memory. Currently we have a late IR optimization pass in codegen to
rewrite the kernel argument values into explicit loads to enable
vectorization. In most programs, unrelated argument loads can be
merged together. However, exposing this property directly from the
frontend has some disadvantages. We still need a way to track the
original argument sizes and alignments to report to the driver. I find
using some side-channel, metadata mechanism to track this
unappealing. If the kernel arguments were exposed as a single buffer
to begin with, alias analysis would be unaware that the padding bits
betewen arguments are meaningless. Another family of problems is there
are still some gaps in replacing all of the available parameter
attributes with metadata equivalents once lowered to loads.
The immediate plan is to start using this new attribute to handle all
aggregate argumets for kernels. Long term, it makes sense to migrate
all kernel arguments, including scalars, to be passed indirectly in
the same manner.
Additional context is in D79744.
2020-06-05 16:58:47 -04:00
|
|
|
}
|
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
Type *AttributeSet::getByValType() const {
|
2021-07-14 21:09:06 +02:00
|
|
|
return SetNode ? SetNode->getAttributeType(Attribute::ByVal) : nullptr;
|
2019-05-30 18:48:23 +00:00
|
|
|
}
|
|
|
|
|
2020-09-29 09:33:55 -04:00
|
|
|
Type *AttributeSet::getStructRetType() const {
|
2021-07-14 21:09:06 +02:00
|
|
|
return SetNode ? SetNode->getAttributeType(Attribute::StructRet) : nullptr;
|
2020-09-29 09:33:55 -04:00
|
|
|
}
|
|
|
|
|
2020-02-14 14:16:53 -08:00
|
|
|
Type *AttributeSet::getPreallocatedType() const {
|
2021-07-14 21:09:06 +02:00
|
|
|
return SetNode ? SetNode->getAttributeType(Attribute::Preallocated) : nullptr;
|
2020-02-14 14:16:53 -08:00
|
|
|
}
|
|
|
|
|
2021-03-29 08:42:23 -04:00
|
|
|
Type *AttributeSet::getInAllocaType() const {
|
2021-07-14 21:09:06 +02:00
|
|
|
return SetNode ? SetNode->getAttributeType(Attribute::InAlloca) : nullptr;
|
2021-03-29 08:42:23 -04:00
|
|
|
}
|
|
|
|
|
2021-07-07 22:29:43 +02:00
|
|
|
Type *AttributeSet::getElementType() const {
|
|
|
|
return SetNode ? SetNode->getAttributeType(Attribute::ElementType) : nullptr;
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
std::pair<unsigned, Optional<unsigned>> AttributeSet::getAllocSizeArgs() const {
|
2017-04-12 23:57:37 +00:00
|
|
|
return SetNode ? SetNode->getAllocSizeArgs()
|
|
|
|
: std::pair<unsigned, Optional<unsigned>>(0, 0);
|
2017-04-12 00:38:00 +00:00
|
|
|
}
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
unsigned AttributeSet::getVScaleRangeMin() const {
|
|
|
|
return SetNode ? SetNode->getVScaleRangeMin() : 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
Optional<unsigned> AttributeSet::getVScaleRangeMax() const {
|
|
|
|
return SetNode ? SetNode->getVScaleRangeMax() : None;
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
UWTableKind AttributeSet::getUWTableKind() const {
|
|
|
|
return SetNode ? SetNode->getUWTableKind() : UWTableKind::None;
|
|
|
|
}
|
|
|
|
|
2022-03-29 11:14:07 -04:00
|
|
|
AllocFnKind AttributeSet::getAllocKind() const {
|
|
|
|
return SetNode ? SetNode->getAllocKind() : AllocFnKind::Unknown;
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
std::string AttributeSet::getAsString(bool InAttrGrp) const {
|
|
|
|
return SetNode ? SetNode->getAsString(InAttrGrp) : "";
|
|
|
|
}
|
|
|
|
|
2021-03-24 16:13:29 -07:00
|
|
|
bool AttributeSet::hasParentContext(LLVMContext &C) const {
|
|
|
|
assert(hasAttributes() && "empty AttributeSet doesn't refer to any context");
|
|
|
|
FoldingSetNodeID ID;
|
|
|
|
SetNode->Profile(ID);
|
|
|
|
void *Unused;
|
|
|
|
return C.pImpl->AttrsSetNodes.FindNodeOrInsertPos(ID, Unused) == SetNode;
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
AttributeSet::iterator AttributeSet::begin() const {
|
|
|
|
return SetNode ? SetNode->begin() : nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSet::iterator AttributeSet::end() const {
|
|
|
|
return SetNode ? SetNode->end() : nullptr;
|
|
|
|
}
|
|
|
|
|
2017-10-15 14:32:27 +00:00
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
2017-05-11 12:28:08 +00:00
|
|
|
LLVM_DUMP_METHOD void AttributeSet::dump() const {
|
|
|
|
dbgs() << "AS =\n";
|
|
|
|
dbgs() << " { ";
|
|
|
|
dbgs() << getAsString(true) << " }\n";
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2013-01-24 00:06:56 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// AttributeSetNode Definition
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2017-04-10 23:46:08 +00:00
|
|
|
AttributeSetNode::AttributeSetNode(ArrayRef<Attribute> Attrs)
|
2019-07-13 00:29:03 +00:00
|
|
|
: NumAttrs(Attrs.size()) {
|
2017-04-10 23:46:08 +00:00
|
|
|
// There's memory after the node where we can store the entries in.
|
2018-11-17 01:44:25 +00:00
|
|
|
llvm::copy(Attrs, getTrailingObjects<Attribute>());
|
2017-04-10 23:46:08 +00:00
|
|
|
|
2019-12-17 21:57:58 +01:00
|
|
|
for (const auto &I : *this) {
|
2020-06-14 23:46:18 +02:00
|
|
|
if (I.isStringAttribute())
|
2020-04-25 12:21:21 +02:00
|
|
|
StringAttrs.insert({ I.getKindAsString(), I });
|
2020-06-14 23:46:18 +02:00
|
|
|
else
|
|
|
|
AvailableAttrs.addAttribute(I.getKindAsEnum());
|
2017-04-10 23:46:08 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-01-24 00:06:56 +00:00
|
|
|
AttributeSetNode *AttributeSetNode::get(LLVMContext &C,
|
|
|
|
ArrayRef<Attribute> Attrs) {
|
2020-04-26 16:52:53 +02:00
|
|
|
SmallVector<Attribute, 8> SortedAttrs(Attrs.begin(), Attrs.end());
|
|
|
|
llvm::sort(SortedAttrs);
|
|
|
|
return getSorted(C, SortedAttrs);
|
|
|
|
}
|
|
|
|
|
|
|
|
AttributeSetNode *AttributeSetNode::getSorted(LLVMContext &C,
|
|
|
|
ArrayRef<Attribute> SortedAttrs) {
|
|
|
|
if (SortedAttrs.empty())
|
2014-04-09 06:08:46 +00:00
|
|
|
return nullptr;
|
2013-01-24 00:06:56 +00:00
|
|
|
|
2020-04-26 16:52:53 +02:00
|
|
|
// Build a key to look up the existing attributes.
|
2013-01-24 00:06:56 +00:00
|
|
|
LLVMContextImpl *pImpl = C.pImpl;
|
|
|
|
FoldingSetNodeID ID;
|
|
|
|
|
2020-04-26 16:52:53 +02:00
|
|
|
assert(llvm::is_sorted(SortedAttrs) && "Expected sorted attributes!");
|
2019-12-17 21:57:58 +01:00
|
|
|
for (const auto &Attr : SortedAttrs)
|
2015-12-16 05:21:02 +00:00
|
|
|
Attr.Profile(ID);
|
2013-01-24 00:06:56 +00:00
|
|
|
|
|
|
|
void *InsertPoint;
|
|
|
|
AttributeSetNode *PA =
|
|
|
|
pImpl->AttrsSetNodes.FindNodeOrInsertPos(ID, InsertPoint);
|
|
|
|
|
|
|
|
// If we didn't find any existing attributes of the same shape then create a
|
|
|
|
// new one and insert it.
|
|
|
|
if (!PA) {
|
2013-07-11 12:13:16 +00:00
|
|
|
// Coallocate entries after the AttributeSetNode itself.
|
2015-08-05 22:57:34 +00:00
|
|
|
void *Mem = ::operator new(totalSizeToAlloc<Attribute>(SortedAttrs.size()));
|
2013-07-11 12:13:16 +00:00
|
|
|
PA = new (Mem) AttributeSetNode(SortedAttrs);
|
2013-01-24 00:06:56 +00:00
|
|
|
pImpl->AttrsSetNodes.InsertNode(PA, InsertPoint);
|
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// Return the AttributeSetNode that we found or created.
|
2013-01-24 00:06:56 +00:00
|
|
|
return PA;
|
|
|
|
}
|
|
|
|
|
2017-04-10 23:31:05 +00:00
|
|
|
AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) {
|
2022-01-15 10:05:22 +01:00
|
|
|
return getSorted(C, B.attrs());
|
2017-04-10 23:31:05 +00:00
|
|
|
}
|
|
|
|
|
2013-02-13 08:42:21 +00:00
|
|
|
bool AttributeSetNode::hasAttribute(StringRef Kind) const {
|
2020-04-25 12:21:21 +02:00
|
|
|
return StringAttrs.count(Kind);
|
2013-02-13 08:42:21 +00:00
|
|
|
}
|
|
|
|
|
2020-04-26 20:03:29 +02:00
|
|
|
Optional<Attribute>
|
|
|
|
AttributeSetNode::findEnumAttribute(Attribute::AttrKind Kind) const {
|
|
|
|
// Do a quick presence check.
|
|
|
|
if (!hasAttribute(Kind))
|
|
|
|
return None;
|
|
|
|
|
|
|
|
// Attributes in a set are sorted by enum value, followed by string
|
|
|
|
// attributes. Binary search the one we want.
|
|
|
|
const Attribute *I =
|
|
|
|
std::lower_bound(begin(), end() - StringAttrs.size(), Kind,
|
|
|
|
[](Attribute A, Attribute::AttrKind Kind) {
|
|
|
|
return A.getKindAsEnum() < Kind;
|
|
|
|
});
|
|
|
|
assert(I != end() && I->hasAttribute(Kind) && "Presence check failed?");
|
|
|
|
return *I;
|
|
|
|
}
|
|
|
|
|
2013-02-13 08:42:21 +00:00
|
|
|
Attribute AttributeSetNode::getAttribute(Attribute::AttrKind Kind) const {
|
2020-04-26 20:03:29 +02:00
|
|
|
if (auto A = findEnumAttribute(Kind))
|
|
|
|
return *A;
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
2013-02-13 08:42:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Attribute AttributeSetNode::getAttribute(StringRef Kind) const {
|
2020-04-25 12:21:21 +02:00
|
|
|
return StringAttrs.lookup(Kind);
|
2013-02-13 08:42:21 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign AttributeSetNode::getAlignment() const {
|
2020-04-26 20:03:29 +02:00
|
|
|
if (auto A = findEnumAttribute(Attribute::Alignment))
|
|
|
|
return A->getAlignment();
|
2019-10-22 09:51:06 +00:00
|
|
|
return None;
|
2013-01-29 03:20:31 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign AttributeSetNode::getStackAlignment() const {
|
2020-04-26 20:03:29 +02:00
|
|
|
if (auto A = findEnumAttribute(Attribute::StackAlignment))
|
|
|
|
return A->getStackAlignment();
|
2019-10-22 09:51:06 +00:00
|
|
|
return None;
|
2013-01-29 03:20:31 +00:00
|
|
|
}
|
|
|
|
|
2021-07-14 21:09:06 +02:00
|
|
|
Type *AttributeSetNode::getAttributeType(Attribute::AttrKind Kind) const {
|
|
|
|
if (auto A = findEnumAttribute(Kind))
|
2021-03-29 08:42:23 -04:00
|
|
|
return A->getValueAsType();
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
2014-07-18 15:51:28 +00:00
|
|
|
uint64_t AttributeSetNode::getDereferenceableBytes() const {
|
2020-04-26 20:03:29 +02:00
|
|
|
if (auto A = findEnumAttribute(Attribute::Dereferenceable))
|
|
|
|
return A->getDereferenceableBytes();
|
2014-07-18 15:51:28 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-05-06 17:41:54 +00:00
|
|
|
uint64_t AttributeSetNode::getDereferenceableOrNullBytes() const {
|
2020-04-26 20:03:29 +02:00
|
|
|
if (auto A = findEnumAttribute(Attribute::DereferenceableOrNull))
|
|
|
|
return A->getDereferenceableOrNullBytes();
|
2015-05-06 17:41:54 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
std::pair<unsigned, Optional<unsigned>>
|
|
|
|
AttributeSetNode::getAllocSizeArgs() const {
|
2020-04-26 20:03:29 +02:00
|
|
|
if (auto A = findEnumAttribute(Attribute::AllocSize))
|
|
|
|
return A->getAllocSizeArgs();
|
2016-04-12 01:05:35 +00:00
|
|
|
return std::make_pair(0, 0);
|
|
|
|
}
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
unsigned AttributeSetNode::getVScaleRangeMin() const {
|
2021-03-03 13:53:30 +00:00
|
|
|
if (auto A = findEnumAttribute(Attribute::VScaleRange))
|
2021-12-07 09:53:16 +00:00
|
|
|
return A->getVScaleRangeMin();
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
Optional<unsigned> AttributeSetNode::getVScaleRangeMax() const {
|
|
|
|
if (auto A = findEnumAttribute(Attribute::VScaleRange))
|
|
|
|
return A->getVScaleRangeMax();
|
|
|
|
return None;
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
UWTableKind AttributeSetNode::getUWTableKind() const {
|
|
|
|
if (auto A = findEnumAttribute(Attribute::UWTable))
|
|
|
|
return A->getUWTableKind();
|
|
|
|
return UWTableKind::None;
|
|
|
|
}
|
|
|
|
|
2022-03-29 11:14:07 -04:00
|
|
|
AllocFnKind AttributeSetNode::getAllocKind() const {
|
|
|
|
if (auto A = findEnumAttribute(Attribute::AllocKind))
|
|
|
|
return A->getAllocKind();
|
|
|
|
return AllocFnKind::Unknown;
|
|
|
|
}
|
|
|
|
|
2013-05-01 13:07:03 +00:00
|
|
|
std::string AttributeSetNode::getAsString(bool InAttrGrp) const {
|
2013-04-19 11:43:21 +00:00
|
|
|
std::string Str;
|
2013-07-11 12:13:16 +00:00
|
|
|
for (iterator I = begin(), E = end(); I != E; ++I) {
|
|
|
|
if (I != begin())
|
2013-05-01 13:07:03 +00:00
|
|
|
Str += ' ';
|
|
|
|
Str += I->getAsString(InAttrGrp);
|
2013-01-29 03:20:31 +00:00
|
|
|
}
|
|
|
|
return Str;
|
|
|
|
}
|
|
|
|
|
2008-03-12 17:45:29 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// AttributeListImpl Definition
|
2008-03-12 17:45:29 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2017-10-11 01:40:38 +00:00
|
|
|
/// Map from AttributeList index to the internal array index. Adding one happens
|
2020-06-23 22:25:04 +02:00
|
|
|
/// to work, because -1 wraps around to 0.
|
2020-07-17 18:25:56 +02:00
|
|
|
static unsigned attrIdxToArrayIdx(unsigned Index) {
|
2020-06-23 22:25:04 +02:00
|
|
|
return Index + 1;
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
}
|
|
|
|
|
2020-05-01 14:18:29 +02:00
|
|
|
AttributeListImpl::AttributeListImpl(ArrayRef<AttributeSet> Sets)
|
|
|
|
: NumAttrSets(Sets.size()) {
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
assert(!Sets.empty() && "pointless AttributeListImpl");
|
2017-04-11 00:16:00 +00:00
|
|
|
|
|
|
|
// There's memory after the node where we can store the entries in.
|
2018-11-17 01:44:25 +00:00
|
|
|
llvm::copy(Sets, getTrailingObjects<AttributeSet>());
|
2017-04-11 00:16:00 +00:00
|
|
|
|
2020-06-14 22:49:57 +02:00
|
|
|
// Initialize AvailableFunctionAttrs and AvailableSomewhereAttrs
|
|
|
|
// summary bitsets.
|
2020-07-17 18:25:56 +02:00
|
|
|
for (const auto &I : Sets[attrIdxToArrayIdx(AttributeList::FunctionIndex)])
|
2020-06-14 23:46:18 +02:00
|
|
|
if (!I.isStringAttribute())
|
|
|
|
AvailableFunctionAttrs.addAttribute(I.getKindAsEnum());
|
2020-06-14 22:49:57 +02:00
|
|
|
|
|
|
|
for (const auto &Set : Sets)
|
|
|
|
for (const auto &I : Set)
|
|
|
|
if (!I.isStringAttribute())
|
|
|
|
AvailableSomewhereAttrs.addAttribute(I.getKindAsEnum());
|
2017-04-11 00:16:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void AttributeListImpl::Profile(FoldingSetNodeID &ID) const {
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
Profile(ID, makeArrayRef(begin(), end()));
|
2017-04-11 00:16:00 +00:00
|
|
|
}
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
void AttributeListImpl::Profile(FoldingSetNodeID &ID,
|
|
|
|
ArrayRef<AttributeSet> Sets) {
|
|
|
|
for (const auto &Set : Sets)
|
|
|
|
ID.AddPointer(Set.SetNode);
|
2017-04-11 00:16:00 +00:00
|
|
|
}
|
|
|
|
|
2020-06-14 22:49:57 +02:00
|
|
|
bool AttributeListImpl::hasAttrSomewhere(Attribute::AttrKind Kind,
|
|
|
|
unsigned *Index) const {
|
|
|
|
if (!AvailableSomewhereAttrs.hasAttribute(Kind))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (Index) {
|
|
|
|
for (unsigned I = 0, E = NumAttrSets; I != E; ++I) {
|
|
|
|
if (begin()[I].hasAttribute(Kind)) {
|
|
|
|
*Index = I - 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-10-15 14:32:27 +00:00
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
LLVM_DUMP_METHOD void AttributeListImpl::dump() const {
|
|
|
|
AttributeList(const_cast<AttributeListImpl *>(this)).dump();
|
2013-08-02 22:34:30 +00:00
|
|
|
}
|
2017-01-28 02:02:38 +00:00
|
|
|
#endif
|
2013-08-02 22:34:30 +00:00
|
|
|
|
2013-01-27 12:50:02 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// AttributeList Construction and Mutation Methods
|
2013-01-27 12:50:02 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
AttributeList AttributeList::getImpl(LLVMContext &C,
|
|
|
|
ArrayRef<AttributeSet> AttrSets) {
|
|
|
|
assert(!AttrSets.empty() && "pointless AttributeListImpl");
|
2017-04-10 23:31:05 +00:00
|
|
|
|
2013-01-29 00:34:06 +00:00
|
|
|
LLVMContextImpl *pImpl = C.pImpl;
|
|
|
|
FoldingSetNodeID ID;
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
AttributeListImpl::Profile(ID, AttrSets);
|
2012-10-16 06:01:44 +00:00
|
|
|
|
2012-11-20 05:09:20 +00:00
|
|
|
void *InsertPoint;
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeListImpl *PA =
|
|
|
|
pImpl->AttrsLists.FindNodeOrInsertPos(ID, InsertPoint);
|
2012-10-16 06:01:44 +00:00
|
|
|
|
2008-01-02 23:42:30 +00:00
|
|
|
// If we didn't find any existing attributes of the same shape then
|
|
|
|
// create a new one and insert it.
|
2012-11-20 05:09:20 +00:00
|
|
|
if (!PA) {
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// Coallocate entries after the AttributeListImpl itself.
|
2020-05-01 14:12:17 +02:00
|
|
|
void *Mem = pImpl->Alloc.Allocate(
|
|
|
|
AttributeListImpl::totalSizeToAlloc<AttributeSet>(AttrSets.size()),
|
|
|
|
alignof(AttributeListImpl));
|
2020-05-01 14:18:29 +02:00
|
|
|
PA = new (Mem) AttributeListImpl(AttrSets);
|
2012-11-20 05:09:20 +00:00
|
|
|
pImpl->AttrsLists.InsertNode(PA, InsertPoint);
|
2008-01-02 23:42:30 +00:00
|
|
|
}
|
2012-10-16 06:01:44 +00:00
|
|
|
|
2008-09-25 21:00:45 +00:00
|
|
|
// Return the AttributesList that we found or created.
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
return AttributeList(PA);
|
2008-01-02 23:42:30 +00:00
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList
|
|
|
|
AttributeList::get(LLVMContext &C,
|
|
|
|
ArrayRef<std::pair<unsigned, Attribute>> Attrs) {
|
2013-01-28 21:55:20 +00:00
|
|
|
// If there are no attributes then return a null AttributesList pointer.
|
|
|
|
if (Attrs.empty())
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
2013-01-28 21:55:20 +00:00
|
|
|
|
2022-06-04 21:23:18 -07:00
|
|
|
assert(llvm::is_sorted(Attrs, llvm::less_first()) &&
|
2020-04-13 14:46:41 +03:00
|
|
|
"Misordered Attributes list!");
|
2020-08-28 13:02:42 -07:00
|
|
|
assert(llvm::all_of(Attrs,
|
|
|
|
[](const std::pair<unsigned, Attribute> &Pair) {
|
|
|
|
return Pair.second.isValid();
|
|
|
|
}) &&
|
2016-08-11 21:15:00 +00:00
|
|
|
"Pointless attribute!");
|
2013-01-28 21:55:20 +00:00
|
|
|
|
2013-01-28 22:33:39 +00:00
|
|
|
// Create a vector if (unsigned, AttributeSetNode*) pairs from the attributes
|
2013-01-28 21:55:20 +00:00
|
|
|
// list.
|
2017-04-12 00:38:00 +00:00
|
|
|
SmallVector<std::pair<unsigned, AttributeSet>, 8> AttrPairVec;
|
2017-02-17 00:00:09 +00:00
|
|
|
for (ArrayRef<std::pair<unsigned, Attribute>>::iterator I = Attrs.begin(),
|
2013-01-28 21:55:20 +00:00
|
|
|
E = Attrs.end(); I != E; ) {
|
2013-01-28 22:33:39 +00:00
|
|
|
unsigned Index = I->first;
|
2013-01-28 21:55:20 +00:00
|
|
|
SmallVector<Attribute, 4> AttrVec;
|
2013-01-29 15:18:16 +00:00
|
|
|
while (I != E && I->first == Index) {
|
2013-01-28 21:55:20 +00:00
|
|
|
AttrVec.push_back(I->second);
|
|
|
|
++I;
|
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
AttrPairVec.emplace_back(Index, AttributeSet::get(C, AttrVec));
|
2013-01-28 21:55:20 +00:00
|
|
|
}
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
return get(C, AttrPairVec);
|
2013-01-28 21:55:20 +00:00
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList
|
|
|
|
AttributeList::get(LLVMContext &C,
|
2017-04-12 00:38:00 +00:00
|
|
|
ArrayRef<std::pair<unsigned, AttributeSet>> Attrs) {
|
2013-01-28 21:55:20 +00:00
|
|
|
// If there are no attributes then return a null AttributesList pointer.
|
|
|
|
if (Attrs.empty())
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
2013-01-28 21:55:20 +00:00
|
|
|
|
2022-06-04 21:23:18 -07:00
|
|
|
assert(llvm::is_sorted(Attrs, llvm::less_first()) &&
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
"Misordered Attributes list!");
|
2018-03-30 00:47:31 +00:00
|
|
|
assert(llvm::none_of(Attrs,
|
|
|
|
[](const std::pair<unsigned, AttributeSet> &Pair) {
|
|
|
|
return !Pair.second.hasAttributes();
|
|
|
|
}) &&
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
"Pointless attribute!");
|
|
|
|
|
|
|
|
unsigned MaxIndex = Attrs.back().first;
|
2018-04-16 17:05:01 +00:00
|
|
|
// If the MaxIndex is FunctionIndex and there are other indices in front
|
|
|
|
// of it, we need to use the largest of those to get the right size.
|
|
|
|
if (MaxIndex == FunctionIndex && Attrs.size() > 1)
|
|
|
|
MaxIndex = Attrs[Attrs.size() - 2].first;
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
|
|
|
|
SmallVector<AttributeSet, 4> AttrVec(attrIdxToArrayIdx(MaxIndex) + 1);
|
2019-12-17 21:57:58 +01:00
|
|
|
for (const auto &Pair : Attrs)
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
AttrVec[attrIdxToArrayIdx(Pair.first)] = Pair.second;
|
|
|
|
|
|
|
|
return getImpl(C, AttrVec);
|
2013-01-28 21:55:20 +00:00
|
|
|
}
|
|
|
|
|
2017-04-13 00:58:09 +00:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C, AttributeSet FnAttrs,
|
|
|
|
AttributeSet RetAttrs,
|
|
|
|
ArrayRef<AttributeSet> ArgAttrs) {
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
// Scan from the end to find the last argument with attributes. Most
|
|
|
|
// arguments don't have attributes, so it's nice if we can have fewer unique
|
|
|
|
// AttributeListImpls by dropping empty attribute sets at the end of the list.
|
|
|
|
unsigned NumSets = 0;
|
|
|
|
for (size_t I = ArgAttrs.size(); I != 0; --I) {
|
|
|
|
if (ArgAttrs[I - 1].hasAttributes()) {
|
|
|
|
NumSets = I + 2;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (NumSets == 0) {
|
|
|
|
// Check function and return attributes if we didn't have argument
|
|
|
|
// attributes.
|
|
|
|
if (RetAttrs.hasAttributes())
|
|
|
|
NumSets = 2;
|
|
|
|
else if (FnAttrs.hasAttributes())
|
|
|
|
NumSets = 1;
|
2017-04-10 23:31:05 +00:00
|
|
|
}
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
|
|
|
|
// If all attribute sets were empty, we can use the empty attribute list.
|
|
|
|
if (NumSets == 0)
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
|
|
|
|
SmallVector<AttributeSet, 8> AttrSets;
|
|
|
|
AttrSets.reserve(NumSets);
|
|
|
|
// If we have any attributes, we always have function attributes.
|
|
|
|
AttrSets.push_back(FnAttrs);
|
|
|
|
if (NumSets > 1)
|
|
|
|
AttrSets.push_back(RetAttrs);
|
|
|
|
if (NumSets > 2) {
|
|
|
|
// Drop the empty argument attribute sets at the end.
|
|
|
|
ArgAttrs = ArgAttrs.take_front(NumSets - 2);
|
2021-01-06 18:27:33 -08:00
|
|
|
llvm::append_range(AttrSets, ArgAttrs);
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return getImpl(C, AttrSets);
|
2017-04-10 23:31:05 +00:00
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
|
2021-12-18 14:56:12 +01:00
|
|
|
AttributeSet Attrs) {
|
|
|
|
if (!Attrs.hasAttributes())
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
Index = attrIdxToArrayIdx(Index);
|
|
|
|
SmallVector<AttributeSet, 8> AttrSets(Index + 1);
|
2021-12-18 14:56:12 +01:00
|
|
|
AttrSets[Index] = Attrs;
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
return getImpl(C, AttrSets);
|
2013-01-05 01:36:54 +00:00
|
|
|
}
|
|
|
|
|
2021-12-18 14:56:12 +01:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
|
|
|
|
const AttrBuilder &B) {
|
|
|
|
return get(C, Index, AttributeSet::get(C, B));
|
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
|
|
|
|
ArrayRef<Attribute::AttrKind> Kinds) {
|
2013-01-28 22:33:39 +00:00
|
|
|
SmallVector<std::pair<unsigned, Attribute>, 8> Attrs;
|
2018-03-30 00:47:31 +00:00
|
|
|
for (const auto K : Kinds)
|
2016-08-11 21:15:00 +00:00
|
|
|
Attrs.emplace_back(Index, Attribute::get(C, K));
|
2013-01-28 21:55:20 +00:00
|
|
|
return get(C, Attrs);
|
2013-01-23 06:14:59 +00:00
|
|
|
}
|
|
|
|
|
2020-05-27 16:02:15 -04:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
|
|
|
|
ArrayRef<Attribute::AttrKind> Kinds,
|
|
|
|
ArrayRef<uint64_t> Values) {
|
|
|
|
assert(Kinds.size() == Values.size() && "Mismatched attribute values.");
|
|
|
|
SmallVector<std::pair<unsigned, Attribute>, 8> Attrs;
|
|
|
|
auto VI = Values.begin();
|
|
|
|
for (const auto K : Kinds)
|
|
|
|
Attrs.emplace_back(Index, Attribute::get(C, K, *VI++));
|
|
|
|
return get(C, Attrs);
|
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C, unsigned Index,
|
|
|
|
ArrayRef<StringRef> Kinds) {
|
2016-06-15 17:50:39 +00:00
|
|
|
SmallVector<std::pair<unsigned, Attribute>, 8> Attrs;
|
2019-12-17 21:57:58 +01:00
|
|
|
for (const auto &K : Kinds)
|
2016-08-11 21:15:00 +00:00
|
|
|
Attrs.emplace_back(Index, Attribute::get(C, K));
|
2016-06-15 17:50:39 +00:00
|
|
|
return get(C, Attrs);
|
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList AttributeList::get(LLVMContext &C,
|
|
|
|
ArrayRef<AttributeList> Attrs) {
|
|
|
|
if (Attrs.empty())
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
if (Attrs.size() == 1)
|
|
|
|
return Attrs[0];
|
|
|
|
|
|
|
|
unsigned MaxSize = 0;
|
2019-12-17 21:57:58 +01:00
|
|
|
for (const auto &List : Attrs)
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
MaxSize = std::max(MaxSize, List.getNumAttrSets());
|
|
|
|
|
2017-05-31 14:24:06 +00:00
|
|
|
// If every list was empty, there is no point in merging the lists.
|
|
|
|
if (MaxSize == 0)
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
2017-05-31 14:24:06 +00:00
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
SmallVector<AttributeSet, 8> NewAttrSets(MaxSize);
|
|
|
|
for (unsigned I = 0; I < MaxSize; ++I) {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder CurBuilder(C);
|
2019-12-17 21:57:58 +01:00
|
|
|
for (const auto &List : Attrs)
|
2022-01-03 13:32:19 -05:00
|
|
|
CurBuilder.merge(AttrBuilder(C, List.getAttributes(I - 1)));
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
NewAttrSets[I] = AttributeSet::get(C, CurBuilder);
|
2013-01-25 23:09:36 +00:00
|
|
|
}
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
return getImpl(C, NewAttrSets);
|
2008-01-02 23:42:30 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList
|
|
|
|
AttributeList::addAttributeAtIndex(LLVMContext &C, unsigned Index,
|
|
|
|
Attribute::AttrKind Kind) const {
|
|
|
|
if (hasAttributeAtIndex(Index, Kind))
|
|
|
|
return *this;
|
2021-01-22 10:38:36 +01:00
|
|
|
AttributeSet Attrs = getAttributes(Index);
|
|
|
|
// TODO: Insert at correct position and avoid sort.
|
|
|
|
SmallVector<Attribute, 8> NewAttrs(Attrs.begin(), Attrs.end());
|
|
|
|
NewAttrs.push_back(Attribute::get(C, Kind));
|
2021-08-20 10:49:36 -07:00
|
|
|
return setAttributesAtIndex(C, Index, AttributeSet::get(C, NewAttrs));
|
2013-03-13 20:20:08 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList AttributeList::addAttributeAtIndex(LLVMContext &C, unsigned Index,
|
|
|
|
StringRef Kind,
|
|
|
|
StringRef Value) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2013-07-25 18:34:24 +00:00
|
|
|
B.addAttribute(Kind, Value);
|
2021-08-20 10:49:36 -07:00
|
|
|
return addAttributesAtIndex(C, Index, B);
|
2013-07-25 18:34:24 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList AttributeList::addAttributeAtIndex(LLVMContext &C, unsigned Index,
|
|
|
|
Attribute A) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2017-05-31 19:23:09 +00:00
|
|
|
B.addAttribute(A);
|
2021-08-20 10:49:36 -07:00
|
|
|
return addAttributesAtIndex(C, Index, B);
|
2015-12-02 06:58:49 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList AttributeList::setAttributesAtIndex(LLVMContext &C,
|
|
|
|
unsigned Index,
|
|
|
|
AttributeSet Attrs) const {
|
2021-01-22 10:38:36 +01:00
|
|
|
Index = attrIdxToArrayIdx(Index);
|
|
|
|
SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
|
|
|
|
if (Index >= AttrSets.size())
|
|
|
|
AttrSets.resize(Index + 1);
|
|
|
|
AttrSets[Index] = Attrs;
|
|
|
|
return AttributeList::getImpl(C, AttrSets);
|
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList AttributeList::addAttributesAtIndex(LLVMContext &C,
|
|
|
|
unsigned Index,
|
|
|
|
const AttrBuilder &B) const {
|
2017-04-19 01:51:13 +00:00
|
|
|
if (!B.hasAttributes())
|
2017-04-10 23:31:05 +00:00
|
|
|
return *this;
|
|
|
|
|
2017-04-18 22:10:18 +00:00
|
|
|
if (!pImpl)
|
2017-04-19 01:51:13 +00:00
|
|
|
return AttributeList::get(C, {{Index, AttributeSet::get(C, B)}});
|
2017-04-18 22:10:18 +00:00
|
|
|
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder Merged(C, getAttributes(Index));
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
Merged.merge(B);
|
2021-08-20 10:49:36 -07:00
|
|
|
return setAttributesAtIndex(C, Index, AttributeSet::get(C, Merged));
|
2017-04-10 23:31:05 +00:00
|
|
|
}
|
|
|
|
|
2017-05-31 19:23:09 +00:00
|
|
|
AttributeList AttributeList::addParamAttribute(LLVMContext &C,
|
|
|
|
ArrayRef<unsigned> ArgNos,
|
|
|
|
Attribute A) const {
|
2020-04-13 14:46:41 +03:00
|
|
|
assert(llvm::is_sorted(ArgNos));
|
2017-05-31 19:23:09 +00:00
|
|
|
|
|
|
|
SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
|
|
|
|
unsigned MaxIndex = attrIdxToArrayIdx(ArgNos.back() + FirstArgIndex);
|
|
|
|
if (MaxIndex >= AttrSets.size())
|
|
|
|
AttrSets.resize(MaxIndex + 1);
|
|
|
|
|
|
|
|
for (unsigned ArgNo : ArgNos) {
|
|
|
|
unsigned Index = attrIdxToArrayIdx(ArgNo + FirstArgIndex);
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C, AttrSets[Index]);
|
2017-05-31 19:23:09 +00:00
|
|
|
B.addAttribute(A);
|
|
|
|
AttrSets[Index] = AttributeSet::get(C, B);
|
|
|
|
}
|
|
|
|
|
|
|
|
return getImpl(C, AttrSets);
|
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList
|
|
|
|
AttributeList::removeAttributeAtIndex(LLVMContext &C, unsigned Index,
|
|
|
|
Attribute::AttrKind Kind) const {
|
|
|
|
if (!hasAttributeAtIndex(Index, Kind))
|
|
|
|
return *this;
|
2018-01-17 19:15:21 +00:00
|
|
|
|
|
|
|
Index = attrIdxToArrayIdx(Index);
|
|
|
|
SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
|
|
|
|
assert(Index < AttrSets.size());
|
|
|
|
|
|
|
|
AttrSets[Index] = AttrSets[Index].removeAttribute(C, Kind);
|
|
|
|
|
|
|
|
return getImpl(C, AttrSets);
|
2013-01-23 00:45:55 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList AttributeList::removeAttributeAtIndex(LLVMContext &C,
|
|
|
|
unsigned Index,
|
|
|
|
StringRef Kind) const {
|
|
|
|
if (!hasAttributeAtIndex(Index, Kind))
|
|
|
|
return *this;
|
2018-01-17 19:15:21 +00:00
|
|
|
|
|
|
|
Index = attrIdxToArrayIdx(Index);
|
|
|
|
SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
|
|
|
|
assert(Index < AttrSets.size());
|
|
|
|
|
|
|
|
AttrSets[Index] = AttrSets[Index].removeAttribute(C, Kind);
|
|
|
|
|
|
|
|
return getImpl(C, AttrSets);
|
2008-01-02 23:42:30 +00:00
|
|
|
}
|
|
|
|
|
2022-01-04 09:44:47 +01:00
|
|
|
AttributeList AttributeList::removeAttributesAtIndex(
|
|
|
|
LLVMContext &C, unsigned Index, const AttributeMask &AttrsToRemove) const {
|
2021-05-22 15:03:29 +02:00
|
|
|
AttributeSet Attrs = getAttributes(Index);
|
|
|
|
AttributeSet NewAttrs = Attrs.removeAttributes(C, AttrsToRemove);
|
|
|
|
// If nothing was removed, return the original list.
|
|
|
|
if (Attrs == NewAttrs)
|
|
|
|
return *this;
|
2021-08-20 10:49:36 -07:00
|
|
|
return setAttributesAtIndex(C, Index, NewAttrs);
|
2015-05-06 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
AttributeList
|
|
|
|
AttributeList::removeAttributesAtIndex(LLVMContext &C,
|
|
|
|
unsigned WithoutIndex) const {
|
2017-04-10 23:31:05 +00:00
|
|
|
if (!pImpl)
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
WithoutIndex = attrIdxToArrayIdx(WithoutIndex);
|
|
|
|
if (WithoutIndex >= getNumAttrSets())
|
|
|
|
return *this;
|
|
|
|
SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end());
|
|
|
|
AttrSets[WithoutIndex] = AttributeSet();
|
|
|
|
return getImpl(C, AttrSets);
|
2017-04-10 23:31:05 +00:00
|
|
|
}
|
|
|
|
|
2021-08-19 14:02:11 -07:00
|
|
|
AttributeList AttributeList::addDereferenceableRetAttr(LLVMContext &C,
|
|
|
|
uint64_t Bytes) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2015-02-14 19:37:54 +00:00
|
|
|
B.addDereferenceableAttr(Bytes);
|
2021-08-19 14:02:11 -07:00
|
|
|
return addRetAttributes(C, B);
|
|
|
|
}
|
|
|
|
|
|
|
|
AttributeList AttributeList::addDereferenceableParamAttr(LLVMContext &C,
|
|
|
|
unsigned Index,
|
|
|
|
uint64_t Bytes) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2021-08-19 14:02:11 -07:00
|
|
|
B.addDereferenceableAttr(Bytes);
|
|
|
|
return addParamAttributes(C, Index, B);
|
2015-02-14 19:37:54 +00:00
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList
|
2021-08-20 10:28:32 -07:00
|
|
|
AttributeList::addDereferenceableOrNullParamAttr(LLVMContext &C, unsigned Index,
|
|
|
|
uint64_t Bytes) const {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2015-04-16 20:29:50 +00:00
|
|
|
B.addDereferenceableOrNullAttr(Bytes);
|
2021-08-20 10:28:32 -07:00
|
|
|
return addParamAttributes(C, Index, B);
|
2015-04-16 20:29:50 +00:00
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
AttributeList
|
2021-08-20 10:28:32 -07:00
|
|
|
AttributeList::addAllocSizeParamAttr(LLVMContext &C, unsigned Index,
|
|
|
|
unsigned ElemSizeArg,
|
|
|
|
const Optional<unsigned> &NumElemsArg) {
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder B(C);
|
2016-04-12 01:05:35 +00:00
|
|
|
B.addAllocSizeAttr(ElemSizeArg, NumElemsArg);
|
2021-08-20 10:28:32 -07:00
|
|
|
return addParamAttributes(C, Index, B);
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
2013-01-29 00:34:06 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// AttributeList Accessor Methods
|
2013-01-29 00:34:06 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2021-08-13 11:16:52 -07:00
|
|
|
AttributeSet AttributeList::getParamAttrs(unsigned ArgNo) const {
|
2017-05-03 18:17:31 +00:00
|
|
|
return getAttributes(ArgNo + FirstArgIndex);
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-13 11:16:52 -07:00
|
|
|
AttributeSet AttributeList::getRetAttrs() const {
|
2017-04-10 23:31:05 +00:00
|
|
|
return getAttributes(ReturnIndex);
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-13 11:16:52 -07:00
|
|
|
AttributeSet AttributeList::getFnAttrs() const {
|
2017-04-10 23:31:05 +00:00
|
|
|
return getAttributes(FunctionIndex);
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
bool AttributeList::hasAttributeAtIndex(unsigned Index,
|
|
|
|
Attribute::AttrKind Kind) const {
|
2017-04-12 00:38:00 +00:00
|
|
|
return getAttributes(Index).hasAttribute(Kind);
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
bool AttributeList::hasAttributeAtIndex(unsigned Index, StringRef Kind) const {
|
2017-04-12 00:38:00 +00:00
|
|
|
return getAttributes(Index).hasAttribute(Kind);
|
2013-02-13 08:42:21 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
bool AttributeList::hasAttributesAtIndex(unsigned Index) const {
|
2017-04-12 00:38:00 +00:00
|
|
|
return getAttributes(Index).hasAttributes();
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-13 11:09:18 -07:00
|
|
|
bool AttributeList::hasFnAttr(Attribute::AttrKind Kind) const {
|
2016-01-29 22:25:19 +00:00
|
|
|
return pImpl && pImpl->hasFnAttribute(Kind);
|
|
|
|
}
|
|
|
|
|
2021-08-13 11:09:18 -07:00
|
|
|
bool AttributeList::hasFnAttr(StringRef Kind) const {
|
2021-08-20 10:49:36 -07:00
|
|
|
return hasAttributeAtIndex(AttributeList::FunctionIndex, Kind);
|
2016-09-09 04:50:38 +00:00
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
bool AttributeList::hasAttrSomewhere(Attribute::AttrKind Attr,
|
|
|
|
unsigned *Index) const {
|
2020-06-14 22:49:57 +02:00
|
|
|
return pImpl && pImpl->hasAttrSomewhere(Attr, Index);
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
Attribute AttributeList::getAttributeAtIndex(unsigned Index,
|
|
|
|
Attribute::AttrKind Kind) const {
|
2017-04-12 00:38:00 +00:00
|
|
|
return getAttributes(Index).getAttribute(Kind);
|
2013-02-13 08:42:21 +00:00
|
|
|
}
|
|
|
|
|
2021-08-20 10:49:36 -07:00
|
|
|
Attribute AttributeList::getAttributeAtIndex(unsigned Index,
|
|
|
|
StringRef Kind) const {
|
2017-04-12 00:38:00 +00:00
|
|
|
return getAttributes(Index).getAttribute(Kind);
|
2013-02-13 08:42:21 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign AttributeList::getRetAlignment() const {
|
2017-04-28 20:34:27 +00:00
|
|
|
return getAttributes(ReturnIndex).getAlignment();
|
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
MaybeAlign AttributeList::getParamAlignment(unsigned ArgNo) const {
|
2017-05-03 18:17:31 +00:00
|
|
|
return getAttributes(ArgNo + FirstArgIndex).getAlignment();
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
[clang][AArch64] Correctly align HFA arguments when passed on the stack
When we pass a AArch64 Homogeneous Floating-Point
Aggregate (HFA) argument with increased alignment
requirements, for example
struct S {
__attribute__ ((__aligned__(16))) double v[4];
};
Clang uses `[4 x double]` for the parameter, which is passed
on the stack at alignment 8, whereas it should be at
alignment 16, following Rule C.4 in
AAPCS (https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst#642parameter-passing-rules)
Currently we don't have a way to express in LLVM IR the
alignment requirements of the function arguments. The align
attribute is applicable to pointers only, and only for some
special ways of passing arguments (e..g byval). When
implementing AAPCS32/AAPCS64, clang resorts to dubious hacks
of coercing to types, which naturally have the needed
alignment. We don't have enough types to cover all the
cases, though.
This patch introduces a new use of the stackalign attribute
to control stack slot alignment, when and if an argument is
passed in memory.
The attribute align is left as an optimizer hint - it still
applies to pointer types only and pertains to the content of
the pointer, whereas the alignment of the pointer itself is
determined by the stackalign attribute.
For byval arguments, the stackalign attribute assumes the
role, previously perfomed by align, falling back to align if
stackalign` is absent.
On the clang side, when passing arguments using the "direct"
style (cf. `ABIArgInfo::Kind`), now we can optionally
specify an alignment, which is emitted as the new
`stackalign` attribute.
Patch by Momchil Velikov and Lucas Prates.
Differential Revision: https://reviews.llvm.org/D98794
2021-04-15 19:58:54 +01:00
|
|
|
MaybeAlign AttributeList::getParamStackAlignment(unsigned ArgNo) const {
|
|
|
|
return getAttributes(ArgNo + FirstArgIndex).getStackAlignment();
|
|
|
|
}
|
|
|
|
|
2019-05-30 18:48:23 +00:00
|
|
|
Type *AttributeList::getParamByValType(unsigned Index) const {
|
|
|
|
return getAttributes(Index+FirstArgIndex).getByValType();
|
|
|
|
}
|
|
|
|
|
2020-09-29 09:33:55 -04:00
|
|
|
Type *AttributeList::getParamStructRetType(unsigned Index) const {
|
|
|
|
return getAttributes(Index + FirstArgIndex).getStructRetType();
|
|
|
|
}
|
|
|
|
|
IR: Define byref parameter attribute
This allows tracking the in-memory type of a pointer argument to a
function for ABI purposes. This is essentially a stripped down version
of byval to remove some of the stack-copy implications in its
definition.
This includes the base IR changes, and some tests for places where it
should be treated similarly to byval. Codegen support will be in a
future patch.
My original attempt at solving some of these problems was to repurpose
byval with a different address space from the stack. However, it is
technically permitted for the callee to introduce a write to the
argument, although nothing does this in reality. There is also talk of
removing and replacing the byval attribute, so a new attribute would
need to take its place anyway.
This is intended avoid some optimization issues with the current
handling of aggregate arguments, as well as fixes inflexibilty in how
frontends can specify the kernel ABI. The most honest representation
of the amdgpu_kernel convention is to expose all kernel arguments as
loads from constant memory. Today, these are raw, SSA Argument values
and codegen is responsible for turning these into loads.
Background:
There currently isn't a satisfactory way to represent how arguments
for the amdgpu_kernel calling convention are passed. In reality,
arguments are passed in a single, flat, constant memory buffer
implicitly passed to the function. It is also illegal to call this
function in the IR, and this is only ever invoked by a driver of some
kind.
It does not make sense to have a stack passed parameter in this
context as is implied by byval. It is never valid to write to the
kernel arguments, as this would corrupt the inputs seen by other
dispatches of the kernel. These argumets are also not in the same
address space as the stack, so a copy is needed to an alloca. From a
source C-like language, the kernel parameters are invisible.
Semantically, a copy is always required from the constant argument
memory to a mutable variable.
The current clang calling convention lowering emits raw values,
including aggregates into the function argument list, since using
byval would not make sense. This has some unfortunate consequences for
the optimizer. In the aggregate case, we end up with an aggregate
store to alloca, which both SROA and instcombine turn into a store of
each aggregate field. The optimizer never pieces this back together to
see that this is really just a copy from constant memory, so we end up
stuck with expensive stack usage.
This also means the backend dictates the alignment of arguments, and
arbitrarily picks the LLVM IR ABI type alignment. By allowing an
explicit alignment, frontends can make better decisions. For example,
there's real no advantage to an aligment higher than 4, so a frontend
could choose to compact the argument layout. Similarly, there is a
high penalty to using an alignment lower than 4, so a frontend could
opt into more padding for small arguments.
Another design consideration is when it is appropriate to expose the
fact that these arguments are all really passed in adjacent
memory. Currently we have a late IR optimization pass in codegen to
rewrite the kernel argument values into explicit loads to enable
vectorization. In most programs, unrelated argument loads can be
merged together. However, exposing this property directly from the
frontend has some disadvantages. We still need a way to track the
original argument sizes and alignments to report to the driver. I find
using some side-channel, metadata mechanism to track this
unappealing. If the kernel arguments were exposed as a single buffer
to begin with, alias analysis would be unaware that the padding bits
betewen arguments are meaningless. Another family of problems is there
are still some gaps in replacing all of the available parameter
attributes with metadata equivalents once lowered to loads.
The immediate plan is to start using this new attribute to handle all
aggregate argumets for kernels. Long term, it makes sense to migrate
all kernel arguments, including scalars, to be passed indirectly in
the same manner.
Additional context is in D79744.
2020-06-05 16:58:47 -04:00
|
|
|
Type *AttributeList::getParamByRefType(unsigned Index) const {
|
|
|
|
return getAttributes(Index + FirstArgIndex).getByRefType();
|
|
|
|
}
|
|
|
|
|
Reland [X86] Codegen for preallocated
See https://reviews.llvm.org/D74651 for the preallocated IR constructs
and LangRef changes.
In X86TargetLowering::LowerCall(), if a call is preallocated, record
each argument's offset from the stack pointer and the total stack
adjustment. Associate the call Value with an integer index. Store the
info in X86MachineFunctionInfo with the integer index as the key.
This adds two new target independent ISDOpcodes and two new target
dependent Opcodes corresponding to @llvm.call.preallocated.{setup,arg}.
The setup ISelDAG node takes in a chain and outputs a chain and a
SrcValue of the preallocated call Value. It is lowered to a target
dependent node with the SrcValue replaced with the integer index key by
looking in X86MachineFunctionInfo. In
X86TargetLowering::EmitInstrWithCustomInserter() this is lowered to an
%esp adjustment, the exact amount determined by looking in
X86MachineFunctionInfo with the integer index key.
The arg ISelDAG node takes in a chain, a SrcValue of the preallocated
call Value, and the arg index int constant. It produces a chain and the
pointer fo the arg. It is lowered to a target dependent node with the
SrcValue replaced with the integer index key by looking in
X86MachineFunctionInfo. In
X86TargetLowering::EmitInstrWithCustomInserter() this is lowered to a
lea of the stack pointer plus an offset determined by looking in
X86MachineFunctionInfo with the integer index key.
Force any function containing a preallocated call to use the frame
pointer.
Does not yet handle a setup without a call, or a conditional call.
Does not yet handle musttail. That requires a LangRef change first.
Tried to look at all references to inalloca and see if they apply to
preallocated. I've made preallocated versions of tests testing inalloca
whenever possible and when they make sense (e.g. not alloca related,
inalloca edge cases).
Aside from the tests added here, I checked that this codegen produces
correct code for something like
```
struct A {
A();
A(A&&);
~A();
};
void bar() {
foo(foo(foo(foo(foo(A(), 4), 5), 6), 7), 8);
}
```
by replacing the inalloca version of the .ll file with the appropriate
preallocated code. Running the executable produces the same results as
using the current inalloca implementation.
Reverted due to unexpectedly passing tests, added REQUIRES: asserts for reland.
Subscribers: hiraditya, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D77689
2020-03-16 12:32:36 -07:00
|
|
|
Type *AttributeList::getParamPreallocatedType(unsigned Index) const {
|
|
|
|
return getAttributes(Index + FirstArgIndex).getPreallocatedType();
|
|
|
|
}
|
|
|
|
|
2021-03-29 08:42:23 -04:00
|
|
|
Type *AttributeList::getParamInAllocaType(unsigned Index) const {
|
|
|
|
return getAttributes(Index + FirstArgIndex).getInAllocaType();
|
|
|
|
}
|
|
|
|
|
2021-07-16 20:26:40 +02:00
|
|
|
Type *AttributeList::getParamElementType(unsigned Index) const {
|
|
|
|
return getAttributes(Index + FirstArgIndex).getElementType();
|
|
|
|
}
|
|
|
|
|
2021-08-19 14:20:59 -07:00
|
|
|
MaybeAlign AttributeList::getFnStackAlignment() const {
|
|
|
|
return getFnAttrs().getStackAlignment();
|
|
|
|
}
|
|
|
|
|
|
|
|
MaybeAlign AttributeList::getRetStackAlignment() const {
|
|
|
|
return getRetAttrs().getStackAlignment();
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-19 14:02:11 -07:00
|
|
|
uint64_t AttributeList::getRetDereferenceableBytes() const {
|
|
|
|
return getRetAttrs().getDereferenceableBytes();
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t AttributeList::getParamDereferenceableBytes(unsigned Index) const {
|
|
|
|
return getParamAttrs(Index).getDereferenceableBytes();
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t AttributeList::getRetDereferenceableOrNullBytes() const {
|
|
|
|
return getRetAttrs().getDereferenceableOrNullBytes();
|
2014-07-18 15:51:28 +00:00
|
|
|
}
|
|
|
|
|
2021-08-19 14:02:11 -07:00
|
|
|
uint64_t
|
|
|
|
AttributeList::getParamDereferenceableOrNullBytes(unsigned Index) const {
|
|
|
|
return getParamAttrs(Index).getDereferenceableOrNullBytes();
|
2015-05-06 17:41:54 +00:00
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
UWTableKind AttributeList::getUWTableKind() const {
|
|
|
|
return getFnAttrs().getUWTableKind();
|
|
|
|
}
|
|
|
|
|
2022-03-29 11:14:07 -04:00
|
|
|
AllocFnKind AttributeList::getAllocKind() const {
|
|
|
|
return getFnAttrs().getAllocKind();
|
|
|
|
}
|
|
|
|
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
std::string AttributeList::getAsString(unsigned Index, bool InAttrGrp) const {
|
2017-04-12 00:38:00 +00:00
|
|
|
return getAttributes(Index).getAsString(InAttrGrp);
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2017-04-12 00:38:00 +00:00
|
|
|
AttributeSet AttributeList::getAttributes(unsigned Index) const {
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
Index = attrIdxToArrayIdx(Index);
|
|
|
|
if (!pImpl || Index >= getNumAttrSets())
|
2018-03-30 00:47:31 +00:00
|
|
|
return {};
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
return pImpl->begin()[Index];
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-03-24 16:13:29 -07:00
|
|
|
bool AttributeList::hasParentContext(LLVMContext &C) const {
|
|
|
|
assert(!isEmpty() && "an empty attribute list has no parent context");
|
|
|
|
FoldingSetNodeID ID;
|
|
|
|
pImpl->Profile(ID);
|
|
|
|
void *Unused;
|
|
|
|
return C.pImpl->AttrsLists.FindNodeOrInsertPos(ID, Unused) == pImpl;
|
|
|
|
}
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
AttributeList::iterator AttributeList::begin() const {
|
|
|
|
return pImpl ? pImpl->begin() : nullptr;
|
2013-01-31 23:53:05 +00:00
|
|
|
}
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
AttributeList::iterator AttributeList::end() const {
|
|
|
|
return pImpl ? pImpl->end() : nullptr;
|
2013-01-31 23:53:05 +00:00
|
|
|
}
|
|
|
|
|
2013-01-29 00:34:06 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
Rename AttributeSet to AttributeList
Summary:
This class is a list of AttributeSetNodes corresponding the function
prototype of a call or function declaration. This class used to be
called ParamAttrListPtr, then AttrListPtr, then AttributeSet. It is
typically accessed by parameter and return value index, so
"AttributeList" seems like a more intuitive name.
Rename AttributeSetImpl to AttributeListImpl to follow suit.
It's useful to rename this class so that we can rename AttributeSetNode
to AttributeSet later. AttributeSet is the set of attributes that apply
to a single function, argument, or return value.
Reviewers: sanjoy, javed.absar, chandlerc, pete
Reviewed By: pete
Subscribers: pete, jholewinski, arsenm, dschuff, mehdi_amini, jfb, nhaehnle, sbc100, void, llvm-commits
Differential Revision: https://reviews.llvm.org/D31102
llvm-svn: 298393
2017-03-21 16:57:19 +00:00
|
|
|
// AttributeList Introspection Methods
|
2013-01-29 00:34:06 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
[IR] Switch AttributeList to use an array for O(1) access
Summary:
Before this change, AttributeLists stored a pair of index and
AttributeSet. This is memory efficient if most arguments do not have
attributes. However, it requires doing a search over the pairs to test
an argument or function attribute. Profiling shows that this loop was
0.76% of the time in 'opt -O2' of sqlite3.c, because LLVM constantly
tests values for nullability.
This was worth about 2.5% of mid-level optimization cycles on the
sqlite3 amalgamation. Here are the full perf results:
https://reviews.llvm.org/P7995
Here are just the before and after cycle counts:
```
$ perf stat -r 5 ./opt_before -O2 sqlite3.bc -o /dev/null
13,274,181,184 cycles # 3.047 GHz ( +- 0.28% )
$ perf stat -r 5 ./opt_after -O2 sqlite3.bc -o /dev/null
12,906,927,263 cycles # 3.043 GHz ( +- 0.51% )
```
This patch *does not* change the indices used to query attributes, as
requested by reviewers. Tracking whether an index is usable for array
indexing is a huge pain that affects many of the internal APIs, so it
would be good to come back later and do a cleanup to remove this
internal adjustment.
Reviewers: pete, chandlerc
Subscribers: javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D32819
llvm-svn: 303654
2017-05-23 17:01:48 +00:00
|
|
|
unsigned AttributeList::getNumAttrSets() const {
|
|
|
|
return pImpl ? pImpl->NumAttrSets : 0;
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2021-03-24 16:13:29 -07:00
|
|
|
void AttributeList::print(raw_ostream &O) const {
|
2021-04-28 13:15:39 -07:00
|
|
|
O << "AttributeList[\n";
|
2013-01-29 00:34:06 +00:00
|
|
|
|
2021-09-30 13:57:55 -07:00
|
|
|
for (unsigned i : indexes()) {
|
2021-04-28 13:15:39 -07:00
|
|
|
if (!getAttributes(i).hasAttributes())
|
|
|
|
continue;
|
|
|
|
O << " { ";
|
|
|
|
switch (i) {
|
|
|
|
case AttrIndex::ReturnIndex:
|
|
|
|
O << "return";
|
|
|
|
break;
|
|
|
|
case AttrIndex::FunctionIndex:
|
|
|
|
O << "function";
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
O << "arg(" << i - AttrIndex::FirstArgIndex << ")";
|
|
|
|
}
|
|
|
|
O << " => " << getAsString(i) << " }\n";
|
2008-03-12 17:45:29 +00:00
|
|
|
}
|
2012-10-16 06:01:44 +00:00
|
|
|
|
2021-03-24 16:13:29 -07:00
|
|
|
O << "]\n";
|
2008-01-06 18:27:01 +00:00
|
|
|
}
|
2021-03-24 16:13:29 -07:00
|
|
|
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
|
|
LLVM_DUMP_METHOD void AttributeList::dump() const { print(dbgs()); }
|
2017-01-28 02:02:38 +00:00
|
|
|
#endif
|
2013-01-25 23:09:36 +00:00
|
|
|
|
2013-01-29 00:34:06 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// AttrBuilder Method Implementations
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2022-01-03 13:32:19 -05:00
|
|
|
AttrBuilder::AttrBuilder(LLVMContext &Ctx, AttributeSet AS) : Ctx(Ctx) {
|
2022-01-15 10:05:22 +01:00
|
|
|
append_range(Attrs, AS);
|
|
|
|
assert(is_sorted(Attrs) && "AttributeSet should be sorted");
|
2017-04-10 23:31:05 +00:00
|
|
|
}
|
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
void AttrBuilder::clear() { Attrs.clear(); }
|
2013-01-29 00:34:06 +00:00
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
/// Attribute comparator that only compares attribute keys. Enum attributes are
|
|
|
|
/// sorted before string attributes.
|
|
|
|
struct AttributeComparator {
|
2022-01-03 13:32:19 -05:00
|
|
|
bool operator()(Attribute A0, Attribute A1) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
bool A0IsString = A0.isStringAttribute();
|
|
|
|
bool A1IsString = A1.isStringAttribute();
|
|
|
|
if (A0IsString) {
|
|
|
|
if (A1IsString)
|
|
|
|
return A0.getKindAsString() < A1.getKindAsString();
|
|
|
|
else
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (A1IsString)
|
|
|
|
return true;
|
|
|
|
return A0.getKindAsEnum() < A1.getKindAsEnum();
|
|
|
|
}
|
|
|
|
bool operator()(Attribute A0, Attribute::AttrKind Kind) const {
|
|
|
|
if (A0.isStringAttribute())
|
|
|
|
return false;
|
|
|
|
return A0.getKindAsEnum() < Kind;
|
2022-01-03 13:32:19 -05:00
|
|
|
}
|
|
|
|
bool operator()(Attribute A0, StringRef Kind) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
if (A0.isStringAttribute())
|
|
|
|
return A0.getKindAsString() < Kind;
|
|
|
|
return true;
|
2022-01-03 13:32:19 -05:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
template <typename K>
|
|
|
|
static void addAttributeImpl(SmallVectorImpl<Attribute> &Attrs, K Kind,
|
|
|
|
Attribute Attr) {
|
|
|
|
auto It = lower_bound(Attrs, Kind, AttributeComparator());
|
|
|
|
if (It != Attrs.end() && It->hasAttribute(Kind))
|
|
|
|
std::swap(*It, Attr);
|
|
|
|
else
|
|
|
|
Attrs.insert(It, Attr);
|
|
|
|
}
|
2013-01-29 00:34:06 +00:00
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) {
|
|
|
|
if (Attr.isStringAttribute())
|
|
|
|
addAttributeImpl(Attrs, Attr.getKindAsString(), Attr);
|
|
|
|
else
|
|
|
|
addAttributeImpl(Attrs, Attr.getKindAsEnum(), Attr);
|
|
|
|
return *this;
|
|
|
|
}
|
2021-03-29 08:42:23 -04:00
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
AttrBuilder &AttrBuilder::addAttribute(Attribute::AttrKind Kind) {
|
|
|
|
addAttributeImpl(Attrs, Kind, Attribute::get(Ctx, Kind));
|
2013-01-29 00:34:06 +00:00
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2013-02-05 08:09:32 +00:00
|
|
|
AttrBuilder &AttrBuilder::addAttribute(StringRef A, StringRef V) {
|
2022-01-15 10:05:22 +01:00
|
|
|
addAttributeImpl(Attrs, A, Attribute::get(Ctx, A, V));
|
|
|
|
return *this;
|
2013-02-05 08:09:32 +00:00
|
|
|
}
|
|
|
|
|
2013-01-31 23:38:01 +00:00
|
|
|
AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) {
|
2013-02-18 12:09:51 +00:00
|
|
|
assert((unsigned)Val < Attribute::EndAttrKinds && "Attribute out of range!");
|
2022-01-15 10:05:22 +01:00
|
|
|
auto It = lower_bound(Attrs, Val, AttributeComparator());
|
|
|
|
if (It != Attrs.end() && It->hasAttribute(Val))
|
|
|
|
Attrs.erase(It);
|
2013-01-31 23:38:01 +00:00
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2013-02-05 08:09:32 +00:00
|
|
|
AttrBuilder &AttrBuilder::removeAttribute(StringRef A) {
|
2022-01-15 10:05:22 +01:00
|
|
|
auto It = lower_bound(Attrs, A, AttributeComparator());
|
|
|
|
if (It != Attrs.end() && It->hasAttribute(A))
|
|
|
|
Attrs.erase(It);
|
2013-02-05 08:09:32 +00:00
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2021-10-03 22:23:05 +02:00
|
|
|
uint64_t AttrBuilder::getRawIntAttr(Attribute::AttrKind Kind) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
assert(Attribute::isIntAttrKind(Kind) && "Not an int attribute");
|
|
|
|
Attribute A = getAttribute(Kind);
|
|
|
|
return A.isValid() ? A.getValueAsInt() : 0;
|
2021-10-03 22:23:05 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
AttrBuilder &AttrBuilder::addRawIntAttr(Attribute::AttrKind Kind,
|
|
|
|
uint64_t Value) {
|
2022-01-15 10:05:22 +01:00
|
|
|
return addAttribute(Attribute::get(Ctx, Kind, Value));
|
2021-10-03 22:23:05 +02:00
|
|
|
}
|
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
std::pair<unsigned, Optional<unsigned>> AttrBuilder::getAllocSizeArgs() const {
|
2021-10-03 22:23:05 +02:00
|
|
|
return unpackAllocSizeArgs(getRawIntAttr(Attribute::AllocSize));
|
2016-04-12 01:05:35 +00:00
|
|
|
}
|
|
|
|
|
2021-12-07 09:53:16 +00:00
|
|
|
unsigned AttrBuilder::getVScaleRangeMin() const {
|
|
|
|
return unpackVScaleRangeArgs(getRawIntAttr(Attribute::VScaleRange)).first;
|
|
|
|
}
|
|
|
|
|
|
|
|
Optional<unsigned> AttrBuilder::getVScaleRangeMax() const {
|
|
|
|
return unpackVScaleRangeArgs(getRawIntAttr(Attribute::VScaleRange)).second;
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
AttrBuilder &AttrBuilder::addAlignmentAttr(MaybeAlign Align) {
|
2019-08-06 09:16:33 +00:00
|
|
|
if (!Align)
|
|
|
|
return *this;
|
2013-01-29 00:34:06 +00:00
|
|
|
|
[IR] Attribute/AttrBuilder: use Value::MaximumAlignment magic constant
Summary:
I initially encountered those assertions when trying to create
this IR `alignment` attribute from clang's `__attribute__((assume_aligned(imm)))`,
because until D72994 there is no sanity checking for the value of `imm`.
But even then, we have `llvm::Value::MaximumAlignment` constant (which is `536870912`),
which is enforced for clang attributes, and then there are some other magical constant
(`0x40000000` i.e. `1073741824` i.e. `2 * 536870912`) in
`Attribute::getWithAlignment()`/`AttrBuilder::addAlignmentAttr()`.
I strongly suspect that `0x40000000` is incorrect,
and that also should be `llvm::Value::MaximumAlignment`.
Reviewers: erichkeane, hfinkel, jdoerfert, gchatelet, courbet
Reviewed By: erichkeane
Subscribers: hiraditya, cfe-commits, llvm-commits
Tags: #llvm, #clang
Differential Revision: https://reviews.llvm.org/D72998
2020-01-23 22:50:06 +03:00
|
|
|
assert(*Align <= llvm::Value::MaximumAlignment && "Alignment too large.");
|
2021-10-03 22:23:05 +02:00
|
|
|
return addRawIntAttr(Attribute::Alignment, Align->value());
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2019-10-22 09:51:06 +00:00
|
|
|
AttrBuilder &AttrBuilder::addStackAlignmentAttr(MaybeAlign Align) {
|
2013-01-29 00:34:06 +00:00
|
|
|
// Default alignment, allow the target to define how to align it.
|
2019-08-06 09:16:33 +00:00
|
|
|
if (!Align)
|
|
|
|
return *this;
|
2013-01-29 00:34:06 +00:00
|
|
|
|
2019-08-06 09:16:33 +00:00
|
|
|
assert(*Align <= 0x100 && "Alignment too large.");
|
2021-10-03 22:23:05 +02:00
|
|
|
return addRawIntAttr(Attribute::StackAlignment, Align->value());
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2014-07-18 15:51:28 +00:00
|
|
|
AttrBuilder &AttrBuilder::addDereferenceableAttr(uint64_t Bytes) {
|
|
|
|
if (Bytes == 0) return *this;
|
|
|
|
|
2021-10-03 22:23:05 +02:00
|
|
|
return addRawIntAttr(Attribute::Dereferenceable, Bytes);
|
2014-07-18 15:51:28 +00:00
|
|
|
}
|
|
|
|
|
2015-04-16 20:29:50 +00:00
|
|
|
AttrBuilder &AttrBuilder::addDereferenceableOrNullAttr(uint64_t Bytes) {
|
|
|
|
if (Bytes == 0)
|
|
|
|
return *this;
|
|
|
|
|
2021-10-03 22:23:05 +02:00
|
|
|
return addRawIntAttr(Attribute::DereferenceableOrNull, Bytes);
|
2015-04-16 20:29:50 +00:00
|
|
|
}
|
|
|
|
|
2016-04-12 01:05:35 +00:00
|
|
|
AttrBuilder &AttrBuilder::addAllocSizeAttr(unsigned ElemSize,
|
|
|
|
const Optional<unsigned> &NumElems) {
|
|
|
|
return addAllocSizeAttrFromRawRepr(packAllocSizeArgs(ElemSize, NumElems));
|
|
|
|
}
|
|
|
|
|
|
|
|
AttrBuilder &AttrBuilder::addAllocSizeAttrFromRawRepr(uint64_t RawArgs) {
|
|
|
|
// (0, 0) is our "not present" value, so we need to check for it here.
|
|
|
|
assert(RawArgs && "Invalid allocsize arguments -- given allocsize(0, 0)");
|
2021-10-03 22:23:05 +02:00
|
|
|
return addRawIntAttr(Attribute::AllocSize, RawArgs);
|
2016-04-12 01:05:35 +00:00
|
|
|
}
|
|
|
|
|
2021-03-03 13:53:30 +00:00
|
|
|
AttrBuilder &AttrBuilder::addVScaleRangeAttr(unsigned MinValue,
|
2021-12-07 09:53:16 +00:00
|
|
|
Optional<unsigned> MaxValue) {
|
2021-03-03 13:53:30 +00:00
|
|
|
return addVScaleRangeAttrFromRawRepr(packVScaleRangeArgs(MinValue, MaxValue));
|
|
|
|
}
|
|
|
|
|
|
|
|
AttrBuilder &AttrBuilder::addVScaleRangeAttrFromRawRepr(uint64_t RawArgs) {
|
|
|
|
// (0, 0) is not present hence ignore this case
|
|
|
|
if (RawArgs == 0)
|
|
|
|
return *this;
|
|
|
|
|
2021-10-03 22:23:05 +02:00
|
|
|
return addRawIntAttr(Attribute::VScaleRange, RawArgs);
|
2021-03-03 13:53:30 +00:00
|
|
|
}
|
|
|
|
|
Extend the `uwtable` attribute with unwind table kind
We have the `clang -cc1` command-line option `-funwind-tables=1|2` and
the codegen option `VALUE_CODEGENOPT(UnwindTables, 2, 0) ///< Unwind
tables (1) or asynchronous unwind tables (2)`. However, this is
encoded in LLVM IR by the presence or the absence of the `uwtable`
attribute, i.e. we lose the information whether to generate want just
some unwind tables or asynchronous unwind tables.
Asynchronous unwind tables take more space in the runtime image, I'd
estimate something like 80-90% more, as the difference is adding
roughly the same number of CFI directives as for prologues, only a bit
simpler (e.g. `.cfi_offset reg, off` vs. `.cfi_restore reg`). Or even
more, if you consider tail duplication of epilogue blocks.
Asynchronous unwind tables could also restrict code generation to
having only a finite number of frame pointer adjustments (an example
of *not* having a finite number of `SP` adjustments is on AArch64 when
untagging the stack (MTE) in some cases the compiler can modify `SP`
in a loop).
Having the CFI precise up to an instruction generally also means one
cannot bundle together CFI instructions once the prologue is done,
they need to be interspersed with ordinary instructions, which means
extra `DW_CFA_advance_loc` commands, further increasing the unwind
tables size.
That is to say, async unwind tables impose a non-negligible overhead,
yet for the most common use cases (like C++ exceptions), they are not
even needed.
This patch extends the `uwtable` attribute with an optional
value:
- `uwtable` (default to `async`)
- `uwtable(sync)`, synchronous unwind tables
- `uwtable(async)`, asynchronous (instruction precise) unwind tables
Reviewed By: MaskRay
Differential Revision: https://reviews.llvm.org/D114543
2022-02-14 13:41:34 +00:00
|
|
|
AttrBuilder &AttrBuilder::addUWTableAttr(UWTableKind Kind) {
|
|
|
|
if (Kind == UWTableKind::None)
|
|
|
|
return *this;
|
|
|
|
return addRawIntAttr(Attribute::UWTable, uint64_t(Kind));
|
|
|
|
}
|
|
|
|
|
2022-03-29 11:14:07 -04:00
|
|
|
AttrBuilder &AttrBuilder::addAllocKindAttr(AllocFnKind Kind) {
|
|
|
|
return addRawIntAttr(Attribute::AllocKind, static_cast<uint64_t>(Kind));
|
|
|
|
}
|
|
|
|
|
2021-07-10 18:36:00 +02:00
|
|
|
Type *AttrBuilder::getTypeAttr(Attribute::AttrKind Kind) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
assert(Attribute::isTypeAttrKind(Kind) && "Not a type attribute");
|
|
|
|
Attribute A = getAttribute(Kind);
|
|
|
|
return A.isValid() ? A.getValueAsType() : nullptr;
|
2021-07-10 18:36:00 +02:00
|
|
|
}
|
|
|
|
|
2021-07-08 22:40:56 +02:00
|
|
|
AttrBuilder &AttrBuilder::addTypeAttr(Attribute::AttrKind Kind, Type *Ty) {
|
2022-01-15 10:05:22 +01:00
|
|
|
return addAttribute(Attribute::get(Ctx, Kind, Ty));
|
2019-05-30 18:48:23 +00:00
|
|
|
}
|
|
|
|
|
2021-07-08 22:40:56 +02:00
|
|
|
AttrBuilder &AttrBuilder::addByValAttr(Type *Ty) {
|
|
|
|
return addTypeAttr(Attribute::ByVal, Ty);
|
|
|
|
}
|
|
|
|
|
2020-09-29 09:33:55 -04:00
|
|
|
AttrBuilder &AttrBuilder::addStructRetAttr(Type *Ty) {
|
2021-07-08 22:40:56 +02:00
|
|
|
return addTypeAttr(Attribute::StructRet, Ty);
|
2020-09-29 09:33:55 -04:00
|
|
|
}
|
|
|
|
|
IR: Define byref parameter attribute
This allows tracking the in-memory type of a pointer argument to a
function for ABI purposes. This is essentially a stripped down version
of byval to remove some of the stack-copy implications in its
definition.
This includes the base IR changes, and some tests for places where it
should be treated similarly to byval. Codegen support will be in a
future patch.
My original attempt at solving some of these problems was to repurpose
byval with a different address space from the stack. However, it is
technically permitted for the callee to introduce a write to the
argument, although nothing does this in reality. There is also talk of
removing and replacing the byval attribute, so a new attribute would
need to take its place anyway.
This is intended avoid some optimization issues with the current
handling of aggregate arguments, as well as fixes inflexibilty in how
frontends can specify the kernel ABI. The most honest representation
of the amdgpu_kernel convention is to expose all kernel arguments as
loads from constant memory. Today, these are raw, SSA Argument values
and codegen is responsible for turning these into loads.
Background:
There currently isn't a satisfactory way to represent how arguments
for the amdgpu_kernel calling convention are passed. In reality,
arguments are passed in a single, flat, constant memory buffer
implicitly passed to the function. It is also illegal to call this
function in the IR, and this is only ever invoked by a driver of some
kind.
It does not make sense to have a stack passed parameter in this
context as is implied by byval. It is never valid to write to the
kernel arguments, as this would corrupt the inputs seen by other
dispatches of the kernel. These argumets are also not in the same
address space as the stack, so a copy is needed to an alloca. From a
source C-like language, the kernel parameters are invisible.
Semantically, a copy is always required from the constant argument
memory to a mutable variable.
The current clang calling convention lowering emits raw values,
including aggregates into the function argument list, since using
byval would not make sense. This has some unfortunate consequences for
the optimizer. In the aggregate case, we end up with an aggregate
store to alloca, which both SROA and instcombine turn into a store of
each aggregate field. The optimizer never pieces this back together to
see that this is really just a copy from constant memory, so we end up
stuck with expensive stack usage.
This also means the backend dictates the alignment of arguments, and
arbitrarily picks the LLVM IR ABI type alignment. By allowing an
explicit alignment, frontends can make better decisions. For example,
there's real no advantage to an aligment higher than 4, so a frontend
could choose to compact the argument layout. Similarly, there is a
high penalty to using an alignment lower than 4, so a frontend could
opt into more padding for small arguments.
Another design consideration is when it is appropriate to expose the
fact that these arguments are all really passed in adjacent
memory. Currently we have a late IR optimization pass in codegen to
rewrite the kernel argument values into explicit loads to enable
vectorization. In most programs, unrelated argument loads can be
merged together. However, exposing this property directly from the
frontend has some disadvantages. We still need a way to track the
original argument sizes and alignments to report to the driver. I find
using some side-channel, metadata mechanism to track this
unappealing. If the kernel arguments were exposed as a single buffer
to begin with, alias analysis would be unaware that the padding bits
betewen arguments are meaningless. Another family of problems is there
are still some gaps in replacing all of the available parameter
attributes with metadata equivalents once lowered to loads.
The immediate plan is to start using this new attribute to handle all
aggregate argumets for kernels. Long term, it makes sense to migrate
all kernel arguments, including scalars, to be passed indirectly in
the same manner.
Additional context is in D79744.
2020-06-05 16:58:47 -04:00
|
|
|
AttrBuilder &AttrBuilder::addByRefAttr(Type *Ty) {
|
2021-07-08 22:40:56 +02:00
|
|
|
return addTypeAttr(Attribute::ByRef, Ty);
|
IR: Define byref parameter attribute
This allows tracking the in-memory type of a pointer argument to a
function for ABI purposes. This is essentially a stripped down version
of byval to remove some of the stack-copy implications in its
definition.
This includes the base IR changes, and some tests for places where it
should be treated similarly to byval. Codegen support will be in a
future patch.
My original attempt at solving some of these problems was to repurpose
byval with a different address space from the stack. However, it is
technically permitted for the callee to introduce a write to the
argument, although nothing does this in reality. There is also talk of
removing and replacing the byval attribute, so a new attribute would
need to take its place anyway.
This is intended avoid some optimization issues with the current
handling of aggregate arguments, as well as fixes inflexibilty in how
frontends can specify the kernel ABI. The most honest representation
of the amdgpu_kernel convention is to expose all kernel arguments as
loads from constant memory. Today, these are raw, SSA Argument values
and codegen is responsible for turning these into loads.
Background:
There currently isn't a satisfactory way to represent how arguments
for the amdgpu_kernel calling convention are passed. In reality,
arguments are passed in a single, flat, constant memory buffer
implicitly passed to the function. It is also illegal to call this
function in the IR, and this is only ever invoked by a driver of some
kind.
It does not make sense to have a stack passed parameter in this
context as is implied by byval. It is never valid to write to the
kernel arguments, as this would corrupt the inputs seen by other
dispatches of the kernel. These argumets are also not in the same
address space as the stack, so a copy is needed to an alloca. From a
source C-like language, the kernel parameters are invisible.
Semantically, a copy is always required from the constant argument
memory to a mutable variable.
The current clang calling convention lowering emits raw values,
including aggregates into the function argument list, since using
byval would not make sense. This has some unfortunate consequences for
the optimizer. In the aggregate case, we end up with an aggregate
store to alloca, which both SROA and instcombine turn into a store of
each aggregate field. The optimizer never pieces this back together to
see that this is really just a copy from constant memory, so we end up
stuck with expensive stack usage.
This also means the backend dictates the alignment of arguments, and
arbitrarily picks the LLVM IR ABI type alignment. By allowing an
explicit alignment, frontends can make better decisions. For example,
there's real no advantage to an aligment higher than 4, so a frontend
could choose to compact the argument layout. Similarly, there is a
high penalty to using an alignment lower than 4, so a frontend could
opt into more padding for small arguments.
Another design consideration is when it is appropriate to expose the
fact that these arguments are all really passed in adjacent
memory. Currently we have a late IR optimization pass in codegen to
rewrite the kernel argument values into explicit loads to enable
vectorization. In most programs, unrelated argument loads can be
merged together. However, exposing this property directly from the
frontend has some disadvantages. We still need a way to track the
original argument sizes and alignments to report to the driver. I find
using some side-channel, metadata mechanism to track this
unappealing. If the kernel arguments were exposed as a single buffer
to begin with, alias analysis would be unaware that the padding bits
betewen arguments are meaningless. Another family of problems is there
are still some gaps in replacing all of the available parameter
attributes with metadata equivalents once lowered to loads.
The immediate plan is to start using this new attribute to handle all
aggregate argumets for kernels. Long term, it makes sense to migrate
all kernel arguments, including scalars, to be passed indirectly in
the same manner.
Additional context is in D79744.
2020-06-05 16:58:47 -04:00
|
|
|
}
|
|
|
|
|
2020-02-14 14:16:53 -08:00
|
|
|
AttrBuilder &AttrBuilder::addPreallocatedAttr(Type *Ty) {
|
2021-07-08 22:40:56 +02:00
|
|
|
return addTypeAttr(Attribute::Preallocated, Ty);
|
2020-02-14 14:16:53 -08:00
|
|
|
}
|
|
|
|
|
2021-03-29 08:42:23 -04:00
|
|
|
AttrBuilder &AttrBuilder::addInAllocaAttr(Type *Ty) {
|
2021-07-08 22:40:56 +02:00
|
|
|
return addTypeAttr(Attribute::InAlloca, Ty);
|
2021-03-29 08:42:23 -04:00
|
|
|
}
|
|
|
|
|
2013-02-06 01:16:00 +00:00
|
|
|
AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) {
|
2022-01-15 10:05:22 +01:00
|
|
|
// TODO: Could make this O(n) as we're merging two sorted lists.
|
|
|
|
for (const auto &I : B.attrs())
|
2022-01-03 13:32:19 -05:00
|
|
|
addAttribute(I);
|
2013-02-06 01:16:00 +00:00
|
|
|
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2022-01-04 09:44:47 +01:00
|
|
|
AttrBuilder &AttrBuilder::remove(const AttributeMask &AM) {
|
2022-01-15 10:05:22 +01:00
|
|
|
erase_if(Attrs, [&](Attribute A) { return AM.contains(A); });
|
2015-05-06 23:19:43 +00:00
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2022-01-04 09:44:47 +01:00
|
|
|
bool AttrBuilder::overlaps(const AttributeMask &AM) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
return any_of(Attrs, [&](Attribute A) { return AM.contains(A); });
|
|
|
|
}
|
2015-05-06 23:19:43 +00:00
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
Attribute AttrBuilder::getAttribute(Attribute::AttrKind A) const {
|
|
|
|
assert((unsigned)A < Attribute::EndAttrKinds && "Attribute out of range!");
|
|
|
|
auto It = lower_bound(Attrs, A, AttributeComparator());
|
|
|
|
if (It != Attrs.end() && It->hasAttribute(A))
|
|
|
|
return *It;
|
|
|
|
return {};
|
2015-05-06 23:19:43 +00:00
|
|
|
}
|
|
|
|
|
2022-01-18 12:12:22 +01:00
|
|
|
Attribute AttrBuilder::getAttribute(StringRef A) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
auto It = lower_bound(Attrs, A, AttributeComparator());
|
|
|
|
if (It != Attrs.end() && It->hasAttribute(A))
|
2022-01-18 12:12:22 +01:00
|
|
|
return *It;
|
|
|
|
return {};
|
|
|
|
}
|
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
bool AttrBuilder::contains(Attribute::AttrKind A) const {
|
2022-01-18 12:12:22 +01:00
|
|
|
return getAttribute(A).isValid();
|
2013-02-06 01:33:42 +00:00
|
|
|
}
|
|
|
|
|
2022-01-15 10:05:22 +01:00
|
|
|
bool AttrBuilder::contains(StringRef A) const {
|
|
|
|
return getAttribute(A).isValid();
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool AttrBuilder::hasAlignmentAttr() const {
|
2021-10-03 22:23:05 +02:00
|
|
|
return getRawIntAttr(Attribute::Alignment) != 0;
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2020-12-17 10:41:35 +00:00
|
|
|
bool AttrBuilder::operator==(const AttrBuilder &B) const {
|
2022-01-15 10:05:22 +01:00
|
|
|
return Attrs == B.Attrs;
|
2013-01-29 00:34:06 +00:00
|
|
|
}
|
|
|
|
|
2013-01-25 23:09:36 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
// AttributeFuncs Function Defintions
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2018-05-01 15:54:18 +00:00
|
|
|
/// Which attributes cannot be applied to a type.
|
2022-02-14 17:12:34 -06:00
|
|
|
AttributeMask AttributeFuncs::typeIncompatible(Type *Ty,
|
|
|
|
AttributeSafetyKind ASK) {
|
2022-01-04 09:44:47 +01:00
|
|
|
AttributeMask Incompatible;
|
2013-01-25 23:09:36 +00:00
|
|
|
|
2022-02-14 17:12:34 -06:00
|
|
|
if (!Ty->isIntegerTy()) {
|
2021-12-06 15:49:23 +00:00
|
|
|
// Attributes that only apply to integers.
|
2022-02-14 17:12:34 -06:00
|
|
|
if (ASK & ASK_SAFE_TO_DROP)
|
|
|
|
Incompatible.addAttribute(Attribute::AllocAlign);
|
|
|
|
if (ASK & ASK_UNSAFE_TO_DROP)
|
|
|
|
Incompatible.addAttribute(Attribute::SExt).addAttribute(Attribute::ZExt);
|
|
|
|
}
|
2013-01-25 23:09:36 +00:00
|
|
|
|
2022-02-14 17:12:34 -06:00
|
|
|
if (!Ty->isPointerTy()) {
|
2021-12-06 15:49:23 +00:00
|
|
|
// Attributes that only apply to pointers.
|
2022-02-14 17:12:34 -06:00
|
|
|
if (ASK & ASK_SAFE_TO_DROP)
|
|
|
|
Incompatible.addAttribute(Attribute::NoAlias)
|
|
|
|
.addAttribute(Attribute::NoCapture)
|
|
|
|
.addAttribute(Attribute::NonNull)
|
|
|
|
.addAttribute(Attribute::ReadNone)
|
|
|
|
.addAttribute(Attribute::ReadOnly)
|
|
|
|
.addAttribute(Attribute::Dereferenceable)
|
|
|
|
.addAttribute(Attribute::DereferenceableOrNull);
|
|
|
|
if (ASK & ASK_UNSAFE_TO_DROP)
|
|
|
|
Incompatible.addAttribute(Attribute::Nest)
|
|
|
|
.addAttribute(Attribute::SwiftError)
|
|
|
|
.addAttribute(Attribute::Preallocated)
|
|
|
|
.addAttribute(Attribute::InAlloca)
|
|
|
|
.addAttribute(Attribute::ByVal)
|
|
|
|
.addAttribute(Attribute::StructRet)
|
|
|
|
.addAttribute(Attribute::ByRef)
|
2022-02-24 10:40:20 -05:00
|
|
|
.addAttribute(Attribute::ElementType)
|
|
|
|
.addAttribute(Attribute::AllocatedPointer);
|
2022-02-14 17:12:34 -06:00
|
|
|
}
|
|
|
|
|
2021-12-06 15:49:23 +00:00
|
|
|
// Attributes that only apply to pointers or vectors of pointers.
|
2022-02-14 17:12:34 -06:00
|
|
|
if (!Ty->isPtrOrPtrVectorTy()) {
|
|
|
|
if (ASK & ASK_SAFE_TO_DROP)
|
|
|
|
Incompatible.addAttribute(Attribute::Alignment);
|
|
|
|
}
|
2021-12-06 15:49:23 +00:00
|
|
|
|
2020-09-08 10:13:11 -05:00
|
|
|
// Some attributes can apply to all "values" but there are no `void` values.
|
2022-02-14 17:12:34 -06:00
|
|
|
if (Ty->isVoidTy()) {
|
|
|
|
if (ASK & ASK_SAFE_TO_DROP)
|
|
|
|
Incompatible.addAttribute(Attribute::NoUndef);
|
|
|
|
}
|
2020-09-08 10:13:11 -05:00
|
|
|
|
2015-05-06 23:19:56 +00:00
|
|
|
return Incompatible;
|
2013-01-25 23:09:36 +00:00
|
|
|
}
|
2015-12-22 23:57:37 +00:00
|
|
|
|
2022-01-04 09:44:47 +01:00
|
|
|
AttributeMask AttributeFuncs::getUBImplyingAttributes() {
|
|
|
|
AttributeMask AM;
|
|
|
|
AM.addAttribute(Attribute::NoUndef);
|
|
|
|
AM.addAttribute(Attribute::Dereferenceable);
|
|
|
|
AM.addAttribute(Attribute::DereferenceableOrNull);
|
|
|
|
return AM;
|
2021-07-25 18:21:13 +02:00
|
|
|
}
|
|
|
|
|
2015-12-22 23:57:37 +00:00
|
|
|
template<typename AttrClass>
|
|
|
|
static bool isEqual(const Function &Caller, const Function &Callee) {
|
|
|
|
return Caller.getFnAttribute(AttrClass::getKind()) ==
|
|
|
|
Callee.getFnAttribute(AttrClass::getKind());
|
|
|
|
}
|
|
|
|
|
2018-05-01 15:54:18 +00:00
|
|
|
/// Compute the logical AND of the attributes of the caller and the
|
2015-12-22 23:57:37 +00:00
|
|
|
/// callee.
|
|
|
|
///
|
|
|
|
/// This function sets the caller's attribute to false if the callee's attribute
|
|
|
|
/// is false.
|
|
|
|
template<typename AttrClass>
|
|
|
|
static void setAND(Function &Caller, const Function &Callee) {
|
|
|
|
if (AttrClass::isSet(Caller, AttrClass::getKind()) &&
|
|
|
|
!AttrClass::isSet(Callee, AttrClass::getKind()))
|
|
|
|
AttrClass::set(Caller, AttrClass::getKind(), false);
|
|
|
|
}
|
|
|
|
|
2018-05-01 15:54:18 +00:00
|
|
|
/// Compute the logical OR of the attributes of the caller and the
|
2015-12-22 23:57:37 +00:00
|
|
|
/// callee.
|
|
|
|
///
|
|
|
|
/// This function sets the caller's attribute to true if the callee's attribute
|
|
|
|
/// is true.
|
|
|
|
template<typename AttrClass>
|
|
|
|
static void setOR(Function &Caller, const Function &Callee) {
|
|
|
|
if (!AttrClass::isSet(Caller, AttrClass::getKind()) &&
|
|
|
|
AttrClass::isSet(Callee, AttrClass::getKind()))
|
|
|
|
AttrClass::set(Caller, AttrClass::getKind(), true);
|
|
|
|
}
|
|
|
|
|
2018-05-01 15:54:18 +00:00
|
|
|
/// If the inlined function had a higher stack protection level than the
|
2015-12-22 23:57:37 +00:00
|
|
|
/// calling function, then bump up the caller's stack protection level.
|
|
|
|
static void adjustCallerSSPLevel(Function &Caller, const Function &Callee) {
|
Don't override __attribute__((no_stack_protector)) by inlining (PR52886)
Since 26c6a3e736d3, LLVM's inliner will "upgrade" the caller's stack protector
attribute based on the callee. This lead to surprising results with Clang's
no_stack_protector attribute added in 4fbf84c1732f (D46300). Consider the
following code compiled with clang -fstack-protector-strong -Os
(https://godbolt.org/z/7s3rW7a1q).
extern void h(int* p);
inline __attribute__((always_inline)) int g() {
return 0;
}
int __attribute__((__no_stack_protector__)) f() {
int a[1];
h(a);
return g();
}
LLVM will inline g() into f(), and f() would get a stack protector, against the
users explicit wishes, potentially breaking the program e.g. if h() changes the
value of the stack cookie. That's a miscompile.
More recently, bc044a88ee3c (D91816) addressed this problem by preventing
inlining when the stack protector is disabled in the caller and enabled in the
callee or vice versa. However, the problem remained if the callee is marked
always_inline as in the example above. This affected users, see e.g.
http://crbug.com/1274129 and http://llvm.org/pr52886.
One way to fix this would be to prevent inlining also in the always_inline
case. Despite the name, always_inline does not guarantee inlining, so this
would be legal but potentially surprising to users.
However, I think the better fix is to not enable the stack protector in a
caller based on the callee. The motivation for the old behaviour is unclear, it
seems counter-intuitive, and causes real problems as we've seen.
This commit implements that fix, which means in the example above, g() gets
inlined into f() (also without always_inline), and f() is emitted without stack
protector. I think that matches most developers' expectations, and that's also
what GCC does.
Another effect of this change is that a no_stack_protector function can now be
inlined into a stack protected function, e.g. (https://godbolt.org/z/hafP6W856):
extern void h(int* p);
inline int __attribute__((__no_stack_protector__)) __attribute__((always_inline)) g() {
return 0;
}
int f() {
int a[1];
h(a);
return g();
}
I think that's fine. Such code would be unusual since no_stack_protector is
normally applied to a program entry point which sets up the stack canary. And
even if such code exists, inlining doesn't change the semantics: there is still
no stack cookie setup/check around entry/exit of the g() code region, but there
may be in the surrounding context, as there was before inlining. This also
matches GCC.
See also the discussion at https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94722
Differential revision: https://reviews.llvm.org/D116589
2022-01-03 18:03:43 +01:00
|
|
|
// If the calling function has *no* stack protection level (e.g. it was built
|
|
|
|
// with Clang's -fno-stack-protector or no_stack_protector attribute), don't
|
|
|
|
// change it as that could change the program's semantics.
|
|
|
|
if (!Caller.hasStackProtectorFnAttr())
|
|
|
|
return;
|
|
|
|
|
2015-12-22 23:57:37 +00:00
|
|
|
// If upgrading the SSP attribute, clear out the old SSP Attributes first.
|
2020-11-17 17:17:44 -08:00
|
|
|
// Having multiple SSP attributes doesn't actually hurt, but it adds useless
|
|
|
|
// clutter to the IR.
|
2022-01-04 09:44:47 +01:00
|
|
|
AttributeMask OldSSPAttr;
|
2017-05-02 22:07:37 +00:00
|
|
|
OldSSPAttr.addAttribute(Attribute::StackProtect)
|
|
|
|
.addAttribute(Attribute::StackProtectStrong)
|
|
|
|
.addAttribute(Attribute::StackProtectReq);
|
2015-12-22 23:57:37 +00:00
|
|
|
|
2016-04-11 22:27:48 +00:00
|
|
|
if (Callee.hasFnAttribute(Attribute::StackProtectReq)) {
|
2021-08-16 17:12:49 -07:00
|
|
|
Caller.removeFnAttrs(OldSSPAttr);
|
2015-12-22 23:57:37 +00:00
|
|
|
Caller.addFnAttr(Attribute::StackProtectReq);
|
|
|
|
} else if (Callee.hasFnAttribute(Attribute::StackProtectStrong) &&
|
|
|
|
!Caller.hasFnAttribute(Attribute::StackProtectReq)) {
|
2021-08-16 17:12:49 -07:00
|
|
|
Caller.removeFnAttrs(OldSSPAttr);
|
2015-12-22 23:57:37 +00:00
|
|
|
Caller.addFnAttr(Attribute::StackProtectStrong);
|
|
|
|
} else if (Callee.hasFnAttribute(Attribute::StackProtect) &&
|
|
|
|
!Caller.hasFnAttribute(Attribute::StackProtectReq) &&
|
|
|
|
!Caller.hasFnAttribute(Attribute::StackProtectStrong))
|
|
|
|
Caller.addFnAttr(Attribute::StackProtect);
|
|
|
|
}
|
|
|
|
|
2018-05-01 15:54:18 +00:00
|
|
|
/// If the inlined function required stack probes, then ensure that
|
2017-06-21 18:46:50 +00:00
|
|
|
/// the calling function has those too.
|
|
|
|
static void adjustCallerStackProbes(Function &Caller, const Function &Callee) {
|
2017-06-22 23:22:36 +00:00
|
|
|
if (!Caller.hasFnAttribute("probe-stack") &&
|
|
|
|
Callee.hasFnAttribute("probe-stack")) {
|
|
|
|
Caller.addFnAttr(Callee.getFnAttribute("probe-stack"));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-01 15:54:18 +00:00
|
|
|
/// If the inlined function defines the size of guard region
|
2017-06-22 23:22:36 +00:00
|
|
|
/// on the stack, then ensure that the calling function defines a guard region
|
|
|
|
/// that is no larger.
|
|
|
|
static void
|
|
|
|
adjustCallerStackProbeSize(Function &Caller, const Function &Callee) {
|
2020-08-28 23:15:34 -07:00
|
|
|
Attribute CalleeAttr = Callee.getFnAttribute("stack-probe-size");
|
|
|
|
if (CalleeAttr.isValid()) {
|
|
|
|
Attribute CallerAttr = Caller.getFnAttribute("stack-probe-size");
|
|
|
|
if (CallerAttr.isValid()) {
|
|
|
|
uint64_t CallerStackProbeSize, CalleeStackProbeSize;
|
|
|
|
CallerAttr.getValueAsString().getAsInteger(0, CallerStackProbeSize);
|
|
|
|
CalleeAttr.getValueAsString().getAsInteger(0, CalleeStackProbeSize);
|
|
|
|
|
2017-06-22 23:22:36 +00:00
|
|
|
if (CallerStackProbeSize > CalleeStackProbeSize) {
|
2020-08-28 23:15:34 -07:00
|
|
|
Caller.addFnAttr(CalleeAttr);
|
2017-06-22 23:22:36 +00:00
|
|
|
}
|
|
|
|
} else {
|
2020-08-28 23:15:34 -07:00
|
|
|
Caller.addFnAttr(CalleeAttr);
|
2017-06-22 23:22:36 +00:00
|
|
|
}
|
|
|
|
}
|
2017-06-21 18:46:50 +00:00
|
|
|
}
|
|
|
|
|
2018-07-24 18:49:00 +00:00
|
|
|
/// If the inlined function defines a min legal vector width, then ensure
|
2018-11-29 07:27:38 +00:00
|
|
|
/// the calling function has the same or larger min legal vector width. If the
|
|
|
|
/// caller has the attribute, but the callee doesn't, we need to remove the
|
|
|
|
/// attribute from the caller since we can't make any guarantees about the
|
|
|
|
/// caller's requirements.
|
|
|
|
/// This function is called after the inlining decision has been made so we have
|
|
|
|
/// to merge the attribute this way. Heuristics that would use
|
2018-07-24 18:49:00 +00:00
|
|
|
/// min-legal-vector-width to determine inline compatibility would need to be
|
|
|
|
/// handled as part of inline cost analysis.
|
|
|
|
static void
|
|
|
|
adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) {
|
2020-08-28 23:15:34 -07:00
|
|
|
Attribute CallerAttr = Caller.getFnAttribute("min-legal-vector-width");
|
|
|
|
if (CallerAttr.isValid()) {
|
|
|
|
Attribute CalleeAttr = Callee.getFnAttribute("min-legal-vector-width");
|
|
|
|
if (CalleeAttr.isValid()) {
|
|
|
|
uint64_t CallerVectorWidth, CalleeVectorWidth;
|
|
|
|
CallerAttr.getValueAsString().getAsInteger(0, CallerVectorWidth);
|
|
|
|
CalleeAttr.getValueAsString().getAsInteger(0, CalleeVectorWidth);
|
2018-11-29 07:27:38 +00:00
|
|
|
if (CallerVectorWidth < CalleeVectorWidth)
|
2020-08-28 23:15:34 -07:00
|
|
|
Caller.addFnAttr(CalleeAttr);
|
2018-07-24 18:49:00 +00:00
|
|
|
} else {
|
2018-11-29 07:27:38 +00:00
|
|
|
// If the callee doesn't have the attribute then we don't know anything
|
|
|
|
// and must drop the attribute from the caller.
|
|
|
|
Caller.removeFnAttr("min-legal-vector-width");
|
2018-07-24 18:49:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-25 12:57:07 +02:00
|
|
|
/// If the inlined function has null_pointer_is_valid attribute,
|
2018-07-30 19:33:53 +00:00
|
|
|
/// set this attribute in the caller post inlining.
|
|
|
|
static void
|
|
|
|
adjustNullPointerValidAttr(Function &Caller, const Function &Callee) {
|
|
|
|
if (Callee.nullPointerIsDefined() && !Caller.nullPointerIsDefined()) {
|
2020-04-25 12:57:07 +02:00
|
|
|
Caller.addFnAttr(Attribute::NullPointerIsValid);
|
2018-07-30 19:33:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-02 14:46:50 +01:00
|
|
|
struct EnumAttr {
|
|
|
|
static bool isSet(const Function &Fn,
|
|
|
|
Attribute::AttrKind Kind) {
|
|
|
|
return Fn.hasFnAttribute(Kind);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void set(Function &Fn,
|
|
|
|
Attribute::AttrKind Kind, bool Val) {
|
|
|
|
if (Val)
|
|
|
|
Fn.addFnAttr(Kind);
|
|
|
|
else
|
|
|
|
Fn.removeFnAttr(Kind);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
struct StrBoolAttr {
|
|
|
|
static bool isSet(const Function &Fn,
|
|
|
|
StringRef Kind) {
|
|
|
|
auto A = Fn.getFnAttribute(Kind);
|
|
|
|
return A.getValueAsString().equals("true");
|
|
|
|
}
|
|
|
|
|
|
|
|
static void set(Function &Fn,
|
|
|
|
StringRef Kind, bool Val) {
|
|
|
|
Fn.addFnAttr(Kind, Val ? "true" : "false");
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
#define GET_ATTR_NAMES
|
|
|
|
#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \
|
|
|
|
struct ENUM_NAME##Attr : EnumAttr { \
|
|
|
|
static enum Attribute::AttrKind getKind() { \
|
|
|
|
return llvm::Attribute::ENUM_NAME; \
|
|
|
|
} \
|
|
|
|
};
|
|
|
|
#define ATTRIBUTE_STRBOOL(ENUM_NAME, DISPLAY_NAME) \
|
|
|
|
struct ENUM_NAME##Attr : StrBoolAttr { \
|
|
|
|
static StringRef getKind() { return #DISPLAY_NAME; } \
|
|
|
|
};
|
|
|
|
#include "llvm/IR/Attributes.inc"
|
|
|
|
|
2015-12-22 23:57:37 +00:00
|
|
|
#define GET_ATTR_COMPAT_FUNC
|
2020-02-02 14:46:50 +01:00
|
|
|
#include "llvm/IR/Attributes.inc"
|
2015-12-22 23:57:37 +00:00
|
|
|
|
|
|
|
bool AttributeFuncs::areInlineCompatible(const Function &Caller,
|
|
|
|
const Function &Callee) {
|
|
|
|
return hasCompatibleFnAttrs(Caller, Callee);
|
|
|
|
}
|
|
|
|
|
2020-08-24 03:24:59 -05:00
|
|
|
bool AttributeFuncs::areOutlineCompatible(const Function &A,
|
|
|
|
const Function &B) {
|
|
|
|
return hasCompatibleFnAttrs(A, B);
|
|
|
|
}
|
|
|
|
|
2015-12-22 23:57:37 +00:00
|
|
|
void AttributeFuncs::mergeAttributesForInlining(Function &Caller,
|
|
|
|
const Function &Callee) {
|
|
|
|
mergeFnAttrs(Caller, Callee);
|
|
|
|
}
|
2020-08-24 03:24:59 -05:00
|
|
|
|
|
|
|
void AttributeFuncs::mergeAttributesForOutlining(Function &Base,
|
|
|
|
const Function &ToMerge) {
|
|
|
|
|
|
|
|
// We merge functions so that they meet the most general case.
|
|
|
|
// For example, if the NoNansFPMathAttr is set in one function, but not in
|
|
|
|
// the other, in the merged function we can say that the NoNansFPMathAttr
|
|
|
|
// is not set.
|
|
|
|
// However if we have the SpeculativeLoadHardeningAttr set true in one
|
|
|
|
// function, but not the other, we make sure that the function retains
|
|
|
|
// that aspect in the merged function.
|
|
|
|
mergeFnAttrs(Base, ToMerge);
|
|
|
|
}
|
2022-05-02 13:29:34 +08:00
|
|
|
|
|
|
|
void AttributeFuncs::updateMinLegalVectorWidthAttr(Function &Fn,
|
|
|
|
uint64_t Width) {
|
|
|
|
Attribute Attr = Fn.getFnAttribute("min-legal-vector-width");
|
|
|
|
if (Attr.isValid()) {
|
|
|
|
uint64_t OldWidth;
|
|
|
|
Attr.getValueAsString().getAsInteger(0, OldWidth);
|
|
|
|
if (Width > OldWidth)
|
|
|
|
Fn.addFnAttr("min-legal-vector-width", llvm::utostr(Width));
|
|
|
|
}
|
|
|
|
}
|