mirrors/llvm-project
0
0
Fork 0
mirror of https://github.com/llvm/llvm-project.git synced 2025-05-02 19:46:05 +00:00
Code Issues Projects Releases Wiki Activity
llvm-project/clang/lib/CodeGen/CGAtomic.cpp
Ulrich Weigand 03ce2a16bf Respect alignment of nested bitfields 
tools/clang/test/CodeGen/packed-nest-unpacked.c contains this test:

struct XBitfield {
  unsigned b1 : 10;
  unsigned b2 : 12;
  unsigned b3 : 10;
};
struct YBitfield {
  char x;
  struct XBitfield y;
} __attribute((packed));
struct YBitfield gbitfield;

unsigned test7() {
  // CHECK: @test7
  // CHECK: load i32, i32* getelementptr inbounds (%struct.YBitfield, %struct.YBitfield* @gbitfield, i32 0, i32 1, i32 0), align 4
  return gbitfield.y.b2;
}

The "align 4" is actually wrong.  Accessing all of "gbitfield.y" as a single
i32 is of course possible, but that still doesn't make it 4-byte aligned as
it remains packed at offset 1 in the surrounding gbitfield object.

This alignment was changed by commit r169489, which also introduced changes
to bitfield access code in CGExpr.cpp.  Code before that change used to take
into account *both* the alignment of the field to be accessed within the
current struct, *and* the alignment of that outer struct itself; this logic
was removed by the above commit.

Neglecting to consider both values can cause incorrect code to be generated
(I've seen an unaligned access crash on SystemZ due to this bug).

In order to always use the best known alignment value, this patch removes
the CGBitFieldInfo::StorageAlignment member and replaces it with a
StorageOffset member specifying the offset from the start of the surrounding
struct to the bitfield's underlying storage.  This offset can then be combined
with the best-known alignment for a bitfield access lvalue to determine the
alignment to use when accessing the bitfield's storage.

Differential Revision: http://reviews.llvm.org/D11034

llvm-svn: 241916
2015-07-10 17:30:00 +00:00

1789 lines
72 KiB
C++
Raw Blame History

//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file contains the code for emitting atomic operations.
//
//===----------------------------------------------------------------------===//
#include "CodeGenFunction.h"
#include "CGCall.h"
#include "CGRecordLayout.h"
#include "CodeGenModule.h"
#include "clang/AST/ASTContext.h"
#include "clang/CodeGen/CGFunctionInfo.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Operator.h"
using namespace clang;
using namespace CodeGen;
namespace {
class AtomicInfo {
CodeGenFunction &CGF;
QualType AtomicTy;
QualType ValueTy;
uint64_t AtomicSizeInBits;
uint64_t ValueSizeInBits;
CharUnits AtomicAlign;
CharUnits ValueAlign;
CharUnits LValueAlign;
TypeEvaluationKind EvaluationKind;
bool UseLibcall;
LValue LVal;
CGBitFieldInfo BFI;
public:
AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
: CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
EvaluationKind(TEK_Scalar), UseLibcall(true) {
assert(!lvalue.isGlobalReg());
ASTContext &C = CGF.getContext();
if (lvalue.isSimple()) {
AtomicTy = lvalue.getType();
if (auto *ATy = AtomicTy->getAs<AtomicType>())
ValueTy = ATy->getValueType();
else
ValueTy = AtomicTy;
EvaluationKind = CGF.getEvaluationKind(ValueTy);
uint64_t ValueAlignInBits;
uint64_t AtomicAlignInBits;
TypeInfo ValueTI = C.getTypeInfo(ValueTy);
ValueSizeInBits = ValueTI.Width;
ValueAlignInBits = ValueTI.Align;
TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
AtomicSizeInBits = AtomicTI.Width;
AtomicAlignInBits = AtomicTI.Align;
assert(ValueSizeInBits <= AtomicSizeInBits);
assert(ValueAlignInBits <= AtomicAlignInBits);
AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
if (lvalue.getAlignment().isZero())
lvalue.setAlignment(AtomicAlign);
LVal = lvalue;
} else if (lvalue.isBitField()) {
ValueTy = lvalue.getType();
ValueSizeInBits = C.getTypeSize(ValueTy);
auto &OrigBFI = lvalue.getBitFieldInfo();
auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
AtomicSizeInBits = C.toBits(
C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
.RoundUpToAlignment(lvalue.getAlignment()));
auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldAddr());
auto OffsetInChars =
(C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
lvalue.getAlignment();
VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
VoidPtrAddr, OffsetInChars.getQuantity());
auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
VoidPtrAddr,
CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
"atomic_bitfield_base");
BFI = OrigBFI;
BFI.Offset = Offset;
BFI.StorageSize = AtomicSizeInBits;
BFI.StorageOffset += OffsetInChars;
LVal = LValue::MakeBitfield(Addr, BFI, lvalue.getType(),
lvalue.getAlignment());
LVal.setTBAAInfo(lvalue.getTBAAInfo());
AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
if (AtomicTy.isNull()) {
llvm::APInt Size(
/*numBits=*/32,
C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
/*IndexTypeQuals=*/0);
}
AtomicAlign = ValueAlign = lvalue.getAlignment();
} else if (lvalue.isVectorElt()) {
ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
ValueSizeInBits = C.getTypeSize(ValueTy);
AtomicTy = lvalue.getType();
AtomicSizeInBits = C.getTypeSize(AtomicTy);
AtomicAlign = ValueAlign = lvalue.getAlignment();
LVal = lvalue;
} else {
assert(lvalue.isExtVectorElt());
ValueTy = lvalue.getType();
ValueSizeInBits = C.getTypeSize(ValueTy);
AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
lvalue.getType(), lvalue.getExtVectorAddr()
->getType()
->getPointerElementType()
->getVectorNumElements());
AtomicSizeInBits = C.getTypeSize(AtomicTy);
AtomicAlign = ValueAlign = lvalue.getAlignment();
LVal = lvalue;
}
UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
}
QualType getAtomicType() const { return AtomicTy; }
QualType getValueType() const { return ValueTy; }
CharUnits getAtomicAlignment() const { return AtomicAlign; }
CharUnits getValueAlignment() const { return ValueAlign; }
uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
bool shouldUseLibcall() const { return UseLibcall; }
const LValue &getAtomicLValue() const { return LVal; }
llvm::Value *getAtomicAddress() const {
if (LVal.isSimple())
return LVal.getAddress();
else if (LVal.isBitField())
return LVal.getBitFieldAddr();
else if (LVal.isVectorElt())
return LVal.getVectorAddr();
assert(LVal.isExtVectorElt());
return LVal.getExtVectorAddr();
}
/// Is the atomic size larger than the underlying value type?
///
/// Note that the absence of padding does not mean that atomic
/// objects are completely interchangeable with non-atomic
/// objects: we might have promoted the alignment of a type
/// without making it bigger.
bool hasPadding() const {
return (ValueSizeInBits != AtomicSizeInBits);
}
bool emitMemSetZeroIfNecessary() const;
llvm::Value *getAtomicSizeValue() const {
CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
return CGF.CGM.getSize(size);
}
/// Cast the given pointer to an integer pointer suitable for
/// atomic operations.
llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const;
/// Turn an atomic-layout object into an r-value.
RValue convertTempToRValue(llvm::Value *addr, AggValueSlot resultSlot,
SourceLocation loc, bool AsValue) const;
/// \brief Converts a rvalue to integer value.
llvm::Value *convertRValueToInt(RValue RVal) const;
RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
AggValueSlot ResultSlot,
SourceLocation Loc, bool AsValue) const;
/// Copy an atomic r-value into atomic-layout memory.
void emitCopyIntoMemory(RValue rvalue) const;
/// Project an l-value down to the value field.
LValue projectValue() const {
assert(LVal.isSimple());
llvm::Value *addr = getAtomicAddress();
if (hasPadding())
addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
return LValue::MakeAddr(addr, getValueType(), LVal.getAlignment(),
CGF.getContext(), LVal.getTBAAInfo());
}
/// \brief Emits atomic load.
/// \returns Loaded value.
RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
bool AsValue, llvm::AtomicOrdering AO,
bool IsVolatile);
/// \brief Emits atomic compare-and-exchange sequence.
/// \param Expected Expected value.
/// \param Desired Desired value.
/// \param Success Atomic ordering for success operation.
/// \param Failure Atomic ordering for failed operation.
/// \param IsWeak true if atomic operation is weak, false otherwise.
/// \returns Pair of values: previous value from storage (value type) and
/// boolean flag (i1 type) with true if success and false otherwise.
std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
RValue Expected, RValue Desired,
llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
bool IsWeak = false);
/// \brief Emits atomic update.
/// \param AO Atomic ordering.
/// \param UpdateOp Update operation for the current lvalue.
void EmitAtomicUpdate(llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile);
/// \brief Emits atomic update.
/// \param AO Atomic ordering.
void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
bool IsVolatile);
/// Materialize an atomic r-value in atomic-layout memory.
llvm::Value *materializeRValue(RValue rvalue) const;
/// \brief Translates LLVM atomic ordering to GNU atomic ordering for
/// libcalls.
static AtomicExpr::AtomicOrderingKind
translateAtomicOrdering(const llvm::AtomicOrdering AO);
private:
bool requiresMemSetZero(llvm::Type *type) const;
/// \brief Creates temp alloca for intermediate operations on atomic value.
llvm::Value *CreateTempAlloca() const;
/// \brief Emits atomic load as a libcall.
void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
llvm::AtomicOrdering AO, bool IsVolatile);
/// \brief Emits atomic load as LLVM instruction.
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
/// \brief Emits atomic compare-and-exchange op as a libcall.
llvm::Value *EmitAtomicCompareExchangeLibcall(
llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
/// \brief Emits atomic compare-and-exchange op as LLVM instruction.
std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
bool IsWeak = false);
/// \brief Emit atomic update as libcalls.
void
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile);
/// \brief Emit atomic update as LLVM instructions.
void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile);
/// \brief Emit atomic update as libcalls.
void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
bool IsVolatile);
/// \brief Emit atomic update as LLVM instructions.
void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
bool IsVolatile);
};
}
AtomicExpr::AtomicOrderingKind
AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
switch (AO) {
case llvm::Unordered:
case llvm::NotAtomic:
case llvm::Monotonic:
return AtomicExpr::AO_ABI_memory_order_relaxed;
case llvm::Acquire:
return AtomicExpr::AO_ABI_memory_order_acquire;
case llvm::Release:
return AtomicExpr::AO_ABI_memory_order_release;
case llvm::AcquireRelease:
return AtomicExpr::AO_ABI_memory_order_acq_rel;
case llvm::SequentiallyConsistent:
return AtomicExpr::AO_ABI_memory_order_seq_cst;
}
llvm_unreachable("Unhandled AtomicOrdering");
}
llvm::Value *AtomicInfo::CreateTempAlloca() const {
auto *TempAlloca = CGF.CreateMemTemp(
(LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
: AtomicTy,
"atomic-temp");
TempAlloca->setAlignment(getAtomicAlignment().getQuantity());
// Cast to pointer to value type for bitfields.
if (LVal.isBitField())
return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
TempAlloca, getAtomicAddress()->getType());
return TempAlloca;
}
static RValue emitAtomicLibcall(CodeGenFunction &CGF,
StringRef fnName,
QualType resultType,
CallArgList &args) {
const CGFunctionInfo &fnInfo =
CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
FunctionType::ExtInfo(), RequiredArgs::All);
llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
}
/// Does a store of the given IR type modify the full expected width?
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
uint64_t expectedSize) {
return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
}
/// Does the atomic type require memsetting to zero before initialization?
///
/// The IR type is provided as a way of making certain queries faster.
bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
// If the atomic type has size padding, we definitely need a memset.
if (hasPadding()) return true;
// Otherwise, do some simple heuristics to try to avoid it:
switch (getEvaluationKind()) {
// For scalars and complexes, check whether the store size of the
// type uses the full size.
case TEK_Scalar:
return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
case TEK_Complex:
return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
AtomicSizeInBits / 2);
// Padding in structs has an undefined bit pattern. User beware.
case TEK_Aggregate:
return false;
}
llvm_unreachable("bad evaluation kind");
}
bool AtomicInfo::emitMemSetZeroIfNecessary() const {
assert(LVal.isSimple());
llvm::Value *addr = LVal.getAddress();
if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
return false;
CGF.Builder.CreateMemSet(
addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
LVal.getAlignment().getQuantity());
return true;
}
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
llvm::Value *Dest, llvm::Value *Ptr,
llvm::Value *Val1, llvm::Value *Val2,
uint64_t Size, unsigned Align,
llvm::AtomicOrdering SuccessOrder,
llvm::AtomicOrdering FailureOrder) {
// Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
llvm::LoadInst *Expected = CGF.Builder.CreateLoad(Val1);
Expected->setAlignment(Align);
llvm::LoadInst *Desired = CGF.Builder.CreateLoad(Val2);
Desired->setAlignment(Align);
llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
Ptr, Expected, Desired, SuccessOrder, FailureOrder);
Pair->setVolatile(E->isVolatile());
Pair->setWeak(IsWeak);
// Cmp holds the result of the compare-exchange operation: true on success,
// false on failure.
llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
// This basic block is used to hold the store instruction if the operation
// failed.
llvm::BasicBlock *StoreExpectedBB =
CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
// This basic block is the exit point of the operation, we should end up
// here regardless of whether or not the operation succeeded.
llvm::BasicBlock *ContinueBB =
CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
// Update Expected if Expected isn't equal to Old, otherwise branch to the
// exit point.
CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
CGF.Builder.SetInsertPoint(StoreExpectedBB);
// Update the memory at Expected with Old's value.
llvm::StoreInst *StoreExpected = CGF.Builder.CreateStore(Old, Val1);
StoreExpected->setAlignment(Align);
// Finally, branch to the exit point.
CGF.Builder.CreateBr(ContinueBB);
CGF.Builder.SetInsertPoint(ContinueBB);
// Update the memory at Dest with Cmp's value.
CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
return;
}
/// Given an ordering required on success, emit all possible cmpxchg
/// instructions to cope with the provided (but possibly only dynamically known)
/// FailureOrder.
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
bool IsWeak, llvm::Value *Dest,
llvm::Value *Ptr, llvm::Value *Val1,
llvm::Value *Val2,
llvm::Value *FailureOrderVal,
uint64_t Size, unsigned Align,
llvm::AtomicOrdering SuccessOrder) {
llvm::AtomicOrdering FailureOrder;
if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
switch (FO->getSExtValue()) {
default:
FailureOrder = llvm::Monotonic;
break;
case AtomicExpr::AO_ABI_memory_order_consume:
case AtomicExpr::AO_ABI_memory_order_acquire:
FailureOrder = llvm::Acquire;
break;
case AtomicExpr::AO_ABI_memory_order_seq_cst:
FailureOrder = llvm::SequentiallyConsistent;
break;
}
if (FailureOrder >= SuccessOrder) {
// Don't assert on undefined behaviour.
FailureOrder =
llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
}
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, Align,
SuccessOrder, FailureOrder);
return;
}
// Create all the relevant BB's
llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
*SeqCstBB = nullptr;
MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
if (SuccessOrder == llvm::SequentiallyConsistent)
SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
// Emit all the different atomics
// MonotonicBB is arbitrarily chosen as the default case; in practice, this
// doesn't matter unless someone is crazy enough to use something that
// doesn't fold to a constant for the ordering.
CGF.Builder.SetInsertPoint(MonotonicBB);
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
Size, Align, SuccessOrder, llvm::Monotonic);
CGF.Builder.CreateBr(ContBB);
if (AcquireBB) {
CGF.Builder.SetInsertPoint(AcquireBB);
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
Size, Align, SuccessOrder, llvm::Acquire);
CGF.Builder.CreateBr(ContBB);
SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
AcquireBB);
SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
AcquireBB);
}
if (SeqCstBB) {
CGF.Builder.SetInsertPoint(SeqCstBB);
emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
Size, Align, SuccessOrder, llvm::SequentiallyConsistent);
CGF.Builder.CreateBr(ContBB);
SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
SeqCstBB);
}
CGF.Builder.SetInsertPoint(ContBB);
}
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
llvm::Value *IsWeak, llvm::Value *FailureOrder,
uint64_t Size, unsigned Align,
llvm::AtomicOrdering Order) {
llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
switch (E->getOp()) {
case AtomicExpr::AO__c11_atomic_init:
llvm_unreachable("Already handled!");
case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
FailureOrder, Size, Align, Order);
return;
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
FailureOrder, Size, Align, Order);
return;
case AtomicExpr::AO__atomic_compare_exchange:
case AtomicExpr::AO__atomic_compare_exchange_n: {
if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
Val1, Val2, FailureOrder, Size, Align, Order);
} else {
// Create all the relevant BB's
llvm::BasicBlock *StrongBB =
CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
llvm::BasicBlock *ContBB =
CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
SI->addCase(CGF.Builder.getInt1(false), StrongBB);
CGF.Builder.SetInsertPoint(StrongBB);
emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
FailureOrder, Size, Align, Order);
CGF.Builder.CreateBr(ContBB);
CGF.Builder.SetInsertPoint(WeakBB);
emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
FailureOrder, Size, Align, Order);
CGF.Builder.CreateBr(ContBB);
CGF.Builder.SetInsertPoint(ContBB);
}
return;
}
case AtomicExpr::AO__c11_atomic_load:
case AtomicExpr::AO__atomic_load_n:
case AtomicExpr::AO__atomic_load: {
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
Load->setAtomic(Order);
Load->setAlignment(Size);
Load->setVolatile(E->isVolatile());
llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
StoreDest->setAlignment(Align);
return;
}
case AtomicExpr::AO__c11_atomic_store:
case AtomicExpr::AO__atomic_store:
case AtomicExpr::AO__atomic_store_n: {
assert(!Dest && "Store does not return a value");
llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
LoadVal1->setAlignment(Align);
llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
Store->setAtomic(Order);
Store->setAlignment(Size);
Store->setVolatile(E->isVolatile());
return;
}
case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_exchange:
Op = llvm::AtomicRMWInst::Xchg;
break;
case AtomicExpr::AO__atomic_add_fetch:
PostOp = llvm::Instruction::Add;
// Fall through.
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
Op = llvm::AtomicRMWInst::Add;
break;
case AtomicExpr::AO__atomic_sub_fetch:
PostOp = llvm::Instruction::Sub;
// Fall through.
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
Op = llvm::AtomicRMWInst::Sub;
break;
case AtomicExpr::AO__atomic_and_fetch:
PostOp = llvm::Instruction::And;
// Fall through.
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
Op = llvm::AtomicRMWInst::And;
break;
case AtomicExpr::AO__atomic_or_fetch:
PostOp = llvm::Instruction::Or;
// Fall through.
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
Op = llvm::AtomicRMWInst::Or;
break;
case AtomicExpr::AO__atomic_xor_fetch:
PostOp = llvm::Instruction::Xor;
// Fall through.
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
Op = llvm::AtomicRMWInst::Xor;
break;
case AtomicExpr::AO__atomic_nand_fetch:
PostOp = llvm::Instruction::And;
// Fall through.
case AtomicExpr::AO__atomic_fetch_nand:
Op = llvm::AtomicRMWInst::Nand;
break;
}
llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
LoadVal1->setAlignment(Align);
llvm::AtomicRMWInst *RMWI =
CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
RMWI->setVolatile(E->isVolatile());
// For __atomic_*_fetch operations, perform the operation again to
// determine the value which was written.
llvm::Value *Result = RMWI;
if (PostOp)
Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
Result = CGF.Builder.CreateNot(Result);
llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
StoreDest->setAlignment(Align);
}
// This function emits any expression (scalar, complex, or aggregate)
// into a temporary alloca.
static llvm::Value *
EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
/*Init*/ true);
return DeclPtr;
}
static void
AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
SourceLocation Loc, CharUnits SizeInChars) {
if (UseOptimizedLibcall) {
// Load value and pass it to the function directly.
unsigned Align = CGF.getContext().getTypeAlignInChars(ValTy).getQuantity();
int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
ValTy =
CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
SizeInBits)->getPointerTo();
Val = CGF.EmitLoadOfScalar(CGF.Builder.CreateBitCast(Val, IPtrTy), false,
Align, CGF.getContext().getPointerType(ValTy),
Loc);
// Coerce the value into an appropriately sized integer type.
Args.add(RValue::get(Val), ValTy);
} else {
// Non-optimized functions always take a reference.
Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
CGF.getContext().VoidPtrTy);
}
}
RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
QualType MemTy = AtomicTy;
if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
MemTy = AT->getValueType();
CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
uint64_t Size = sizeChars.getQuantity();
CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
unsigned Align = alignChars.getQuantity();
unsigned MaxInlineWidthInBits =
getTarget().getMaxAtomicInlineWidth();
bool UseLibcall = (Size != Align ||
getContext().toBits(sizeChars) > MaxInlineWidthInBits);
llvm::Value *IsWeak = nullptr, *OrderFail = nullptr, *Val1 = nullptr,
*Val2 = nullptr;
llvm::Value *Ptr = EmitScalarExpr(E->getPtr());
if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
assert(!Dest && "Init does not return a value");
LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext());
EmitAtomicInit(E->getVal1(), lvalue);
return RValue::get(nullptr);
}
llvm::Value *Order = EmitScalarExpr(E->getOrder());
switch (E->getOp()) {
case AtomicExpr::AO__c11_atomic_init:
llvm_unreachable("Already handled!");
case AtomicExpr::AO__c11_atomic_load:
case AtomicExpr::AO__atomic_load_n:
break;
case AtomicExpr::AO__atomic_load:
Dest = EmitScalarExpr(E->getVal1());
break;
case AtomicExpr::AO__atomic_store:
Val1 = EmitScalarExpr(E->getVal1());
break;
case AtomicExpr::AO__atomic_exchange:
Val1 = EmitScalarExpr(E->getVal1());
Dest = EmitScalarExpr(E->getVal2());
break;
case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
case AtomicExpr::AO__atomic_compare_exchange_n:
case AtomicExpr::AO__atomic_compare_exchange:
Val1 = EmitScalarExpr(E->getVal1());
if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
Val2 = EmitScalarExpr(E->getVal2());
else
Val2 = EmitValToTemp(*this, E->getVal2());
OrderFail = EmitScalarExpr(E->getOrderFail());
if (E->getNumSubExprs() == 6)
IsWeak = EmitScalarExpr(E->getWeak());
break;
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__c11_atomic_fetch_sub:
if (MemTy->isPointerType()) {
// For pointer arithmetic, we're required to do a bit of math:
// adding 1 to an int* is not the same as adding 1 to a uintptr_t.
// ... but only for the C11 builtins. The GNU builtins expect the
// user to multiply by sizeof(T).
QualType Val1Ty = E->getVal1()->getType();
llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
CharUnits PointeeIncAmt =
getContext().getTypeSizeInChars(MemTy->getPointeeType());
Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
break;
}
// Fall through.
case AtomicExpr::AO__atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_sub:
case AtomicExpr::AO__atomic_add_fetch:
case AtomicExpr::AO__atomic_sub_fetch:
case AtomicExpr::AO__c11_atomic_store:
case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__atomic_store_n:
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_nand:
case AtomicExpr::AO__atomic_and_fetch:
case AtomicExpr::AO__atomic_or_fetch:
case AtomicExpr::AO__atomic_xor_fetch:
case AtomicExpr::AO__atomic_nand_fetch:
Val1 = EmitValToTemp(*this, E->getVal1());
break;
}
QualType RValTy = E->getType().getUnqualifiedType();
auto GetDest = [&] {
if (!RValTy->isVoidType() && !Dest) {
Dest = CreateMemTemp(RValTy, ".atomicdst");
}
return Dest;
};
// Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
if (UseLibcall) {
bool UseOptimizedLibcall = false;
switch (E->getOp()) {
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
// For these, only library calls for certain sizes exist.
UseOptimizedLibcall = true;
break;
default:
// Only use optimized library calls for sizes for which they exist.
if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
UseOptimizedLibcall = true;
break;
}
CallArgList Args;
if (!UseOptimizedLibcall) {
// For non-optimized library calls, the size is the first parameter
Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
getContext().getSizeType());
}
// Atomic address is the first or second parameter
Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), getContext().VoidPtrTy);
std::string LibCallName;
QualType LoweredMemTy =
MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
QualType RetTy;
bool HaveRetTy = false;
switch (E->getOp()) {
// There is only one libcall for compare an exchange, because there is no
// optimisation benefit possible from a libcall version of a weak compare
// and exchange.
// bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
// void *desired, int success, int failure)
// bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
// int success, int failure)
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
case AtomicExpr::AO__atomic_compare_exchange:
case AtomicExpr::AO__atomic_compare_exchange_n:
LibCallName = "__atomic_compare_exchange";
RetTy = getContext().BoolTy;
HaveRetTy = true;
Args.add(RValue::get(EmitCastToVoidPtr(Val1)), getContext().VoidPtrTy);
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2, MemTy,
E->getExprLoc(), sizeChars);
Args.add(RValue::get(Order), getContext().IntTy);
Order = OrderFail;
break;
// void __atomic_exchange(size_t size, void *mem, void *val, void *return,
// int order)
// T __atomic_exchange_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_exchange:
LibCallName = "__atomic_exchange";
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
E->getExprLoc(), sizeChars);
break;
// void __atomic_store(size_t size, void *mem, void *val, int order)
// void __atomic_store_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_store:
case AtomicExpr::AO__atomic_store:
case AtomicExpr::AO__atomic_store_n:
LibCallName = "__atomic_store";
RetTy = getContext().VoidTy;
HaveRetTy = true;
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
E->getExprLoc(), sizeChars);
break;
// void __atomic_load(size_t size, void *mem, void *return, int order)
// T __atomic_load_N(T *mem, int order)
case AtomicExpr::AO__c11_atomic_load:
case AtomicExpr::AO__atomic_load:
case AtomicExpr::AO__atomic_load_n:
LibCallName = "__atomic_load";
break;
// T __atomic_fetch_add_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
LibCallName = "__atomic_fetch_add";
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
E->getExprLoc(), sizeChars);
break;
// T __atomic_fetch_and_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
LibCallName = "__atomic_fetch_and";
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
E->getExprLoc(), sizeChars);
break;
// T __atomic_fetch_or_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
LibCallName = "__atomic_fetch_or";
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
E->getExprLoc(), sizeChars);
break;
// T __atomic_fetch_sub_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
LibCallName = "__atomic_fetch_sub";
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, LoweredMemTy,
E->getExprLoc(), sizeChars);
break;
// T __atomic_fetch_xor_N(T *mem, T val, int order)
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
LibCallName = "__atomic_fetch_xor";
AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1, MemTy,
E->getExprLoc(), sizeChars);
break;
default: return EmitUnsupportedRValue(E, "atomic library call");
}
// Optimized functions have the size in their name.
if (UseOptimizedLibcall)
LibCallName += "_" + llvm::utostr(Size);
// By default, assume we return a value of the atomic type.
if (!HaveRetTy) {
if (UseOptimizedLibcall) {
// Value is returned directly.
// The function returns an appropriately sized integer type.
RetTy = getContext().getIntTypeForBitwidth(
getContext().toBits(sizeChars), /*Signed=*/false);
} else {
// Value is returned through parameter before the order.
RetTy = getContext().VoidTy;
Args.add(RValue::get(EmitCastToVoidPtr(Dest)), getContext().VoidPtrTy);
}
}
// order is always the last parameter
Args.add(RValue::get(Order),
getContext().IntTy);
RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
// The value is returned directly from the libcall.
if (HaveRetTy && !RetTy->isVoidType())
return Res;
// The value is returned via an explicit out param.
if (RetTy->isVoidType())
return RValue::get(nullptr);
// The value is returned directly for optimized libcalls but the caller is
// expected an out-param.
if (UseOptimizedLibcall) {
llvm::Value *ResVal = Res.getScalarVal();
llvm::StoreInst *StoreDest = Builder.CreateStore(
ResVal,
Builder.CreateBitCast(GetDest(), ResVal->getType()->getPointerTo()));
StoreDest->setAlignment(Align);
}
return convertTempToRValue(Dest, RValTy, E->getExprLoc());
}
bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
E->getOp() == AtomicExpr::AO__atomic_store ||
E->getOp() == AtomicExpr::AO__atomic_store_n;
bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
E->getOp() == AtomicExpr::AO__atomic_load ||
E->getOp() == AtomicExpr::AO__atomic_load_n;
llvm::Type *ITy =
llvm::IntegerType::get(getLLVMContext(), Size * 8);
llvm::Value *OrigDest = GetDest();
Ptr = Builder.CreateBitCast(
Ptr, ITy->getPointerTo(Ptr->getType()->getPointerAddressSpace()));
if (Val1) Val1 = Builder.CreateBitCast(Val1, ITy->getPointerTo());
if (Val2) Val2 = Builder.CreateBitCast(Val2, ITy->getPointerTo());
if (Dest && !E->isCmpXChg())
Dest = Builder.CreateBitCast(Dest, ITy->getPointerTo());
if (isa<llvm::ConstantInt>(Order)) {
int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
switch (ord) {
case AtomicExpr::AO_ABI_memory_order_relaxed:
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::Monotonic);
break;
case AtomicExpr::AO_ABI_memory_order_consume:
case AtomicExpr::AO_ABI_memory_order_acquire:
if (IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::Acquire);
break;
case AtomicExpr::AO_ABI_memory_order_release:
if (IsLoad)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::Release);
break;
case AtomicExpr::AO_ABI_memory_order_acq_rel:
if (IsLoad || IsStore)
break; // Avoid crashing on code with undefined behavior
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::AcquireRelease);
break;
case AtomicExpr::AO_ABI_memory_order_seq_cst:
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::SequentiallyConsistent);
break;
default: // invalid order
// We should not ever get here normally, but it's hard to
// enforce that in general.
break;
}
if (RValTy->isVoidType())
return RValue::get(nullptr);
return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
}
// Long case, when Order isn't obviously constant.
// Create all the relevant BB's
llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
*ReleaseBB = nullptr, *AcqRelBB = nullptr,
*SeqCstBB = nullptr;
MonotonicBB = createBasicBlock("monotonic", CurFn);
if (!IsStore)
AcquireBB = createBasicBlock("acquire", CurFn);
if (!IsLoad)
ReleaseBB = createBasicBlock("release", CurFn);
if (!IsLoad && !IsStore)
AcqRelBB = createBasicBlock("acqrel", CurFn);
SeqCstBB = createBasicBlock("seqcst", CurFn);
llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
// Create the switch for the split
// MonotonicBB is arbitrarily chosen as the default case; in practice, this
// doesn't matter unless someone is crazy enough to use something that
// doesn't fold to a constant for the ordering.
Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
// Emit all the different atomics
Builder.SetInsertPoint(MonotonicBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::Monotonic);
Builder.CreateBr(ContBB);
if (!IsStore) {
Builder.SetInsertPoint(AcquireBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::Acquire);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
AcquireBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
AcquireBB);
}
if (!IsLoad) {
Builder.SetInsertPoint(ReleaseBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::Release);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
ReleaseBB);
}
if (!IsLoad && !IsStore) {
Builder.SetInsertPoint(AcqRelBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::AcquireRelease);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
AcqRelBB);
}
Builder.SetInsertPoint(SeqCstBB);
EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
Size, Align, llvm::SequentiallyConsistent);
Builder.CreateBr(ContBB);
SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
SeqCstBB);
// Cleanup and return
Builder.SetInsertPoint(ContBB);
if (RValTy->isVoidType())
return RValue::get(nullptr);
return convertTempToRValue(OrigDest, RValTy, E->getExprLoc());
}
llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const {
unsigned addrspace =
cast<llvm::PointerType>(addr->getType())->getAddressSpace();
llvm::IntegerType *ty =
llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
}
RValue AtomicInfo::convertTempToRValue(llvm::Value *addr,
AggValueSlot resultSlot,
SourceLocation loc, bool AsValue) const {
if (LVal.isSimple()) {
if (EvaluationKind == TEK_Aggregate)
return resultSlot.asRValue();
// Drill into the padding structure if we have one.
if (hasPadding())
addr = CGF.Builder.CreateStructGEP(nullptr, addr, 0);
// Otherwise, just convert the temporary to an r-value using the
// normal conversion routine.
return CGF.convertTempToRValue(addr, getValueType(), loc);
}
if (!AsValue)
// Get RValue from temp memory as atomic for non-simple lvalues
return RValue::get(
CGF.Builder.CreateAlignedLoad(addr, AtomicAlign.getQuantity()));
if (LVal.isBitField())
return CGF.EmitLoadOfBitfieldLValue(LValue::MakeBitfield(
addr, LVal.getBitFieldInfo(), LVal.getType(), LVal.getAlignment()));
if (LVal.isVectorElt())
return CGF.EmitLoadOfLValue(LValue::MakeVectorElt(addr, LVal.getVectorIdx(),
LVal.getType(),
LVal.getAlignment()),
loc);
assert(LVal.isExtVectorElt());
return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
addr, LVal.getExtVectorElts(), LVal.getType(), LVal.getAlignment()));
}
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
AggValueSlot ResultSlot,
SourceLocation Loc,
bool AsValue) const {
// Try not to in some easy cases.
assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
if (getEvaluationKind() == TEK_Scalar &&
(((!LVal.isBitField() ||
LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
!hasPadding()) ||
!AsValue)) {
auto *ValTy = AsValue
? CGF.ConvertTypeForMem(ValueTy)
: getAtomicAddress()->getType()->getPointerElementType();
if (ValTy->isIntegerTy()) {
assert(IntVal->getType() == ValTy && "Different integer types.");
return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
} else if (ValTy->isPointerTy())
return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
}
// Create a temporary. This needs to be big enough to hold the
// atomic integer.
llvm::Value *Temp;
bool TempIsVolatile = false;
CharUnits TempAlignment;
if (AsValue && getEvaluationKind() == TEK_Aggregate) {
assert(!ResultSlot.isIgnored());
Temp = ResultSlot.getAddr();
TempAlignment = getValueAlignment();
TempIsVolatile = ResultSlot.isVolatile();
} else {
Temp = CreateTempAlloca();
TempAlignment = getAtomicAlignment();
}
// Slam the integer into the temporary.
llvm::Value *CastTemp = emitCastToAtomicIntPointer(Temp);
CGF.Builder.CreateAlignedStore(IntVal, CastTemp, TempAlignment.getQuantity())
->setVolatile(TempIsVolatile);
return convertTempToRValue(Temp, ResultSlot, Loc, AsValue);
}
void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
llvm::AtomicOrdering AO, bool) {
// void __atomic_load(size_t size, void *mem, void *return, int order);
CallArgList Args;
Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(
llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
CGF.getContext().IntTy);
emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
}
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
bool IsVolatile) {
// Okay, we're doing this natively.
llvm::Value *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
Load->setAtomic(AO);
// Other decoration.
Load->setAlignment(getAtomicAlignment().getQuantity());
if (IsVolatile)
Load->setVolatile(true);
if (LVal.getTBAAInfo())
CGF.CGM.DecorateInstruction(Load, LVal.getTBAAInfo());
return Load;
}
/// An LValue is a candidate for having its loads and stores be made atomic if
/// we are operating under /volatile:ms *and* the LValue itself is volatile and
/// performing such an operation can be performed without a libcall.
bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
AtomicInfo AI(*this, LV);
bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
// An atomic is inline if we don't need to use a libcall.
bool AtomicIsInline = !AI.shouldUseLibcall();
return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
}
/// An type is a candidate for having its loads and stores be made atomic if
/// we are operating under /volatile:ms *and* we know the access is volatile and
/// performing such an operation can be performed without a libcall.
bool CodeGenFunction::typeIsSuitableForInlineAtomic(QualType Ty,
bool IsVolatile) const {
// An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
}
RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
AggValueSlot Slot) {
llvm::AtomicOrdering AO;
bool IsVolatile = LV.isVolatileQualified();
if (LV.getType()->isAtomicType()) {
AO = llvm::SequentiallyConsistent;
} else {
AO = llvm::Acquire;
IsVolatile = true;
}
return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
}
RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
bool AsValue, llvm::AtomicOrdering AO,
bool IsVolatile) {
// Check whether we should use a library call.
if (shouldUseLibcall()) {
llvm::Value *TempAddr;
if (LVal.isSimple() && !ResultSlot.isIgnored()) {
assert(getEvaluationKind() == TEK_Aggregate);
TempAddr = ResultSlot.getAddr();
} else
TempAddr = CreateTempAlloca();
EmitAtomicLoadLibcall(TempAddr, AO, IsVolatile);
// Okay, turn that back into the original value or whole atomic (for
// non-simple lvalues) type.
return convertTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
}
// Okay, we're doing this natively.
auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
// If we're ignoring an aggregate return, don't do anything.
if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
return RValue::getAggregate(nullptr, false);
// Okay, turn that back into the original value or atomic (for non-simple
// lvalues) type.
return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
}
/// Emit a load from an l-value of atomic type. Note that the r-value
/// we produce is an r-value of the atomic *value* type.
RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
llvm::AtomicOrdering AO, bool IsVolatile,
AggValueSlot resultSlot) {
AtomicInfo Atomics(*this, src);
return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
IsVolatile);
}
/// Copy an r-value into memory as part of storing to an atomic type.
/// This needs to create a bit-pattern suitable for atomic operations.
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
assert(LVal.isSimple());
// If we have an r-value, the rvalue should be of the atomic type,
// which means that the caller is responsible for having zeroed
// any padding. Just do an aggregate copy of that type.
if (rvalue.isAggregate()) {
CGF.EmitAggregateCopy(getAtomicAddress(),
rvalue.getAggregateAddr(),
getAtomicType(),
(rvalue.isVolatileQualified()
|| LVal.isVolatileQualified()),
LVal.getAlignment());
return;
}
// Okay, otherwise we're copying stuff.
// Zero out the buffer if necessary.
emitMemSetZeroIfNecessary();
// Drill past the padding if present.
LValue TempLVal = projectValue();
// Okay, store the rvalue in.
if (rvalue.isScalar()) {
CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
} else {
CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
}
}
/// Materialize an r-value into memory for the purposes of storing it
/// to an atomic type.
llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const {
// Aggregate r-values are already in memory, and EmitAtomicStore
// requires them to be values of the atomic type.
if (rvalue.isAggregate())
return rvalue.getAggregateAddr();
// Otherwise, make a temporary and materialize into it.
LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType(),
getAtomicAlignment());
AtomicInfo Atomics(CGF, TempLV);
Atomics.emitCopyIntoMemory(rvalue);
return TempLV.getAddress();
}
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
// If we've got a scalar value of the right size, try to avoid going
// through memory.
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
llvm::Value *Value = RVal.getScalarVal();
if (isa<llvm::IntegerType>(Value->getType()))
return CGF.EmitToMemory(Value, ValueTy);
else {
llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
CGF.getLLVMContext(),
LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
if (isa<llvm::PointerType>(Value->getType()))
return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
return CGF.Builder.CreateBitCast(Value, InputIntTy);
}
}
// Otherwise, we need to go through memory.
// Put the r-value in memory.
llvm::Value *Addr = materializeRValue(RVal);
// Cast the temporary to the atomic int type and pull a value out.
Addr = emitCastToAtomicIntPointer(Addr);
return CGF.Builder.CreateAlignedLoad(Addr,
getAtomicAlignment().getQuantity());
}
std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
// Do the atomic store.
auto *Addr = emitCastToAtomicIntPointer(getAtomicAddress());
auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
Success, Failure);
// Other decoration.
Inst->setVolatile(LVal.isVolatileQualified());
Inst->setWeak(IsWeak);
// Okay, turn that back into the original value type.
auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
return std::make_pair(PreviousVal, SuccessFailureVal);
}
llvm::Value *
AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
llvm::Value *DesiredAddr,
llvm::AtomicOrdering Success,
llvm::AtomicOrdering Failure) {
// bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
// void *desired, int success, int failure);
CallArgList Args;
Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicAddress())),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
CGF.getContext().VoidPtrTy);
Args.add(RValue::get(llvm::ConstantInt::get(
CGF.IntTy, translateAtomicOrdering(Success))),
CGF.getContext().IntTy);
Args.add(RValue::get(llvm::ConstantInt::get(
CGF.IntTy, translateAtomicOrdering(Failure))),
CGF.getContext().IntTy);
auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
CGF.getContext().BoolTy, Args);
return SuccessFailureRVal.getScalarVal();
}
std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
llvm::AtomicOrdering Failure, bool IsWeak) {
if (Failure >= Success)
// Don't assert on undefined behavior.
Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
// Check whether we should use a library call.
if (shouldUseLibcall()) {
// Produce a source address.
auto *ExpectedAddr = materializeRValue(Expected);
auto *DesiredAddr = materializeRValue(Desired);
auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr,
Success, Failure);
return std::make_pair(
convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false),
Res);
}
// If we've got a scalar value of the right size, try to avoid going
// through memory.
auto *ExpectedVal = convertRValueToInt(Expected);
auto *DesiredVal = convertRValueToInt(Desired);
auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
Failure, IsWeak);
return std::make_pair(
ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false),
Res.second);
}
static void
EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
const llvm::function_ref<RValue(RValue)> &UpdateOp,
llvm::Value *DesiredAddr) {
llvm::Value *Ptr = nullptr;
LValue UpdateLVal;
RValue UpRVal;
LValue AtomicLVal = Atomics.getAtomicLValue();
LValue DesiredLVal;
if (AtomicLVal.isSimple()) {
UpRVal = OldRVal;
DesiredLVal =
LValue::MakeAddr(DesiredAddr, AtomicLVal.getType(),
AtomicLVal.getAlignment(), CGF.CGM.getContext());
} else {
// Build new lvalue for temp address
Ptr = Atomics.materializeRValue(OldRVal);
if (AtomicLVal.isBitField()) {
UpdateLVal =
LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
AtomicLVal.getType(), AtomicLVal.getAlignment());
DesiredLVal =
LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
AtomicLVal.getType(), AtomicLVal.getAlignment());
} else if (AtomicLVal.isVectorElt()) {
UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
AtomicLVal.getType(),
AtomicLVal.getAlignment());
DesiredLVal = LValue::MakeVectorElt(
DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
AtomicLVal.getAlignment());
} else {
assert(AtomicLVal.isExtVectorElt());
UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
AtomicLVal.getType(),
AtomicLVal.getAlignment());
DesiredLVal = LValue::MakeExtVectorElt(
DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
AtomicLVal.getAlignment());
}
UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
}
// Store new value in the corresponding memory area
RValue NewRVal = UpdateOp(UpRVal);
if (NewRVal.isScalar()) {
CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
} else {
assert(NewRVal.isComplex());
CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
/*isInit=*/false);
}
}
void AtomicInfo::EmitAtomicUpdateLibcall(
llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile) {
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
llvm::Value *ExpectedAddr = CreateTempAlloca();
EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
auto *ContBB = CGF.createBasicBlock("atomic_cont");
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
CGF.EmitBlock(ContBB);
auto *DesiredAddr = CreateTempAlloca();
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
requiresMemSetZero(
getAtomicAddress()->getType()->getPointerElementType())) {
auto *OldVal = CGF.Builder.CreateAlignedLoad(
ExpectedAddr, getAtomicAlignment().getQuantity());
CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
getAtomicAlignment().getQuantity());
}
auto OldRVal = convertTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false);
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
auto *Res =
EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
void AtomicInfo::EmitAtomicUpdateOp(
llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile) {
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
// Do the atomic load.
auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
// For non-simple lvalues perform compare-and-swap procedure.
auto *ContBB = CGF.createBasicBlock("atomic_cont");
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
auto *CurBB = CGF.Builder.GetInsertBlock();
CGF.EmitBlock(ContBB);
llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
/*NumReservedValues=*/2);
PHI->addIncoming(OldVal, CurBB);
auto *NewAtomicAddr = CreateTempAlloca();
auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
requiresMemSetZero(
getAtomicAddress()->getType()->getPointerElementType())) {
CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
getAtomicAlignment().getQuantity());
}
auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false);
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
NewAtomicIntAddr, getAtomicAlignment().getQuantity());
// Try to write new value using cmpxchg operation
auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
RValue UpdateRVal, llvm::Value *DesiredAddr) {
LValue AtomicLVal = Atomics.getAtomicLValue();
LValue DesiredLVal;
// Build new lvalue for temp address
if (AtomicLVal.isBitField()) {
DesiredLVal =
LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
AtomicLVal.getType(), AtomicLVal.getAlignment());
} else if (AtomicLVal.isVectorElt()) {
DesiredLVal =
LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
AtomicLVal.getType(), AtomicLVal.getAlignment());
} else {
assert(AtomicLVal.isExtVectorElt());
DesiredLVal = LValue::MakeExtVectorElt(
DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
AtomicLVal.getAlignment());
}
DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
// Store new value in the corresponding memory area
assert(UpdateRVal.isScalar());
CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
}
void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
RValue UpdateRVal, bool IsVolatile) {
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
llvm::Value *ExpectedAddr = CreateTempAlloca();
EmitAtomicLoadLibcall(ExpectedAddr, AO, IsVolatile);
auto *ContBB = CGF.createBasicBlock("atomic_cont");
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
CGF.EmitBlock(ContBB);
auto *DesiredAddr = CreateTempAlloca();
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
requiresMemSetZero(
getAtomicAddress()->getType()->getPointerElementType())) {
auto *OldVal = CGF.Builder.CreateAlignedLoad(
ExpectedAddr, getAtomicAlignment().getQuantity());
CGF.Builder.CreateAlignedStore(OldVal, DesiredAddr,
getAtomicAlignment().getQuantity());
}
EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
auto *Res =
EmitAtomicCompareExchangeLibcall(ExpectedAddr, DesiredAddr, AO, Failure);
CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
bool IsVolatile) {
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
// Do the atomic load.
auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
// For non-simple lvalues perform compare-and-swap procedure.
auto *ContBB = CGF.createBasicBlock("atomic_cont");
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
auto *CurBB = CGF.Builder.GetInsertBlock();
CGF.EmitBlock(ContBB);
llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
/*NumReservedValues=*/2);
PHI->addIncoming(OldVal, CurBB);
auto *NewAtomicAddr = CreateTempAlloca();
auto *NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
requiresMemSetZero(
getAtomicAddress()->getType()->getPointerElementType())) {
CGF.Builder.CreateAlignedStore(PHI, NewAtomicIntAddr,
getAtomicAlignment().getQuantity());
}
EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
auto *DesiredVal = CGF.Builder.CreateAlignedLoad(
NewAtomicIntAddr, getAtomicAlignment().getQuantity());
// Try to write new value using cmpxchg operation
auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
}
void AtomicInfo::EmitAtomicUpdate(
llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
bool IsVolatile) {
if (shouldUseLibcall()) {
EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
} else {
EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
}
}
void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
bool IsVolatile) {
if (shouldUseLibcall()) {
EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
} else {
EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
}
}
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
bool isInit) {
bool IsVolatile = lvalue.isVolatileQualified();
llvm::AtomicOrdering AO;
if (lvalue.getType()->isAtomicType()) {
AO = llvm::SequentiallyConsistent;
} else {
AO = llvm::Release;
IsVolatile = true;
}
return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
}
/// Emit a store to an l-value of atomic type.
///
/// Note that the r-value is expected to be an r-value *of the atomic
/// type*; this means that for aggregate r-values, it should include
/// storage for any padding that was necessary.
void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
llvm::AtomicOrdering AO, bool IsVolatile,
bool isInit) {
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!rvalue.isAggregate() ||
rvalue.getAggregateAddr()->getType()->getPointerElementType()
== dest.getAddress()->getType()->getPointerElementType());
AtomicInfo atomics(*this, dest);
LValue LVal = atomics.getAtomicLValue();
// If this is an initialization, just put the value there normally.
if (LVal.isSimple()) {
if (isInit) {
atomics.emitCopyIntoMemory(rvalue);
return;
}
// Check whether we should use a library call.
if (atomics.shouldUseLibcall()) {
// Produce a source address.
llvm::Value *srcAddr = atomics.materializeRValue(rvalue);
// void __atomic_store(size_t size, void *mem, void *val, int order)
CallArgList args;
args.add(RValue::get(atomics.getAtomicSizeValue()),
getContext().getSizeType());
args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicAddress())),
getContext().VoidPtrTy);
args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), getContext().VoidPtrTy);
args.add(RValue::get(llvm::ConstantInt::get(
IntTy, AtomicInfo::translateAtomicOrdering(AO))),
getContext().IntTy);
emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
return;
}
// Okay, we're doing this natively.
llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
// Do the atomic store.
llvm::Value *addr =
atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
intValue = Builder.CreateIntCast(
intValue, addr->getType()->getPointerElementType(), /*isSigned=*/false);
llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
// Initializations don't need to be atomic.
if (!isInit)
store->setAtomic(AO);
// Other decoration.
store->setAlignment(dest.getAlignment().getQuantity());
if (IsVolatile)
store->setVolatile(true);
if (dest.getTBAAInfo())
CGM.DecorateInstruction(store, dest.getTBAAInfo());
return;
}
// Emit simple atomic update operation.
atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
}
/// Emit a compare-and-exchange op for atomic type.
///
std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
AggValueSlot Slot) {
// If this is an aggregate r-value, it should agree in type except
// maybe for address-space qualification.
assert(!Expected.isAggregate() ||
Expected.getAggregateAddr()->getType()->getPointerElementType() ==
Obj.getAddress()->getType()->getPointerElementType());
assert(!Desired.isAggregate() ||
Desired.getAggregateAddr()->getType()->getPointerElementType() ==
Obj.getAddress()->getType()->getPointerElementType());
AtomicInfo Atomics(*this, Obj);
return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
IsWeak);
}
void CodeGenFunction::EmitAtomicUpdate(
LValue LVal, llvm::AtomicOrdering AO,
const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
AtomicInfo Atomics(*this, LVal);
Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
}
void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
AtomicInfo atomics(*this, dest);
switch (atomics.getEvaluationKind()) {
case TEK_Scalar: {
llvm::Value *value = EmitScalarExpr(init);
atomics.emitCopyIntoMemory(RValue::get(value));
return;
}
case TEK_Complex: {
ComplexPairTy value = EmitComplexExpr(init);
atomics.emitCopyIntoMemory(RValue::getComplex(value));
return;
}
case TEK_Aggregate: {
// Fix up the destination if the initializer isn't an expression
// of atomic type.
bool Zeroed = false;
if (!init->getType()->isAtomicType()) {
Zeroed = atomics.emitMemSetZeroIfNecessary();
dest = atomics.projectValue();
}
// Evaluate the expression directly into the destination.
AggValueSlot slot = AggValueSlot::forLValue(dest,
AggValueSlot::IsNotDestructed,
AggValueSlot::DoesNotNeedGCBarriers,
AggValueSlot::IsNotAliased,
Zeroed ? AggValueSlot::IsZeroed :
AggValueSlot::IsNotZeroed);
EmitAggExpr(init, slot);
return;
}
}
llvm_unreachable("bad evaluation kind");
}
Reference in New Issue View Git Blame Copy Permalink