bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder);
+ bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder);
+
// Stubs to keep the compiler happy while we implement the rest of the
// translation.
bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) {
bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) {
return false;
}
- bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder) {
- return false;
- }
- bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) {
- return false;
- }
bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) {
return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder);
}
MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val,
unsigned Idx);
- /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
+ /// Build and insert `OldValRes<def>, SuccessRes<def> =
+ /// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with \p NewVal if it is currently
+ /// \p CmpVal otherwise leaves it unchanged. Puts the original value from \p
+ /// Addr in \p Res, along with an s1 indicating whether it was replaced.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register of scalar type.
+ /// \pre \p SuccessRes must be a generic virtual register of scalar type. It
+ /// will be assigned 0 on failure and 1 on success.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, \p CmpVal, and \p NewVal must be generic virtual
+ /// registers of the same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder
+ buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes,
+ unsigned Addr, unsigned CmpVal, unsigned NewVal,
+ MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal,
/// MMO`.
///
/// Atomically replace the value at \p Addr with \p NewVal if it is currently
MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
unsigned CmpVal, unsigned NewVal,
MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO`.
+ ///
+ /// Atomically read-modify-update the value at \p Addr with \p Val. Puts the
+ /// original value from \p Addr in \p OldValRes. The modification is
+ /// determined by the opcode.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
+ unsigned Addr, unsigned Val,
+ MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with \p Val. Puts the original
+ /// value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_ADD Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the addition of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_SUB Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the subtraction of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_AND Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise and of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_NAND Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise nand of \p Val
+ /// and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_OR Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise or of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_XOR Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the bitwise xor of \p Val and
+ /// the original value. Puts the original value from \p Addr in \p OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_MAX Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the signed maximum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_MIN Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the signed minimum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMAX Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the unsigned maximum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
+
+ /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMIN Addr, Val, MMO`.
+ ///
+ /// Atomically replace the value at \p Addr with the unsigned minimum of \p
+ /// Val and the original value. Puts the original value from \p Addr in \p
+ /// OldValRes.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p OldValRes must be a generic virtual register.
+ /// \pre \p Addr must be a generic virtual register with pointer type.
+ /// \pre \p OldValRes, and \p Val must be generic virtual registers of the
+ /// same type.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO);
};
/// A CRTP class that contains methods for building instructions that can
} else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
Alignment = LI->getAlignment();
ValTy = LI->getType();
+ } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
+ // TODO(PR27168): This instruction has no alignment attribute, but unlike
+ // the default alignment for load/store, the default here is to assume
+ // it has NATURAL alignment, not DataLayout-specified alignment.
+ const DataLayout &DL = AI->getModule()->getDataLayout();
+ Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
+ ValTy = AI->getCompareOperand()->getType();
+ } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
+ // TODO(PR27168): This instruction has no alignment attribute, but unlike
+ // the default alignment for load/store, the default here is to assume
+ // it has NATURAL alignment, not DataLayout-specified alignment.
+ const DataLayout &DL = AI->getModule()->getDataLayout();
+ Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
+ ValTy = AI->getType();
} else {
OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
R << "unable to translate memop: " << ore::NV("Opcode", &I);
return true;
}
+bool IRTranslator::translateAtomicCmpXchg(const User &U,
+ MachineIRBuilder &MIRBuilder) {
+ const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
+
+ if (I.isWeak())
+ return false;
+
+ auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
+ : MachineMemOperand::MONone;
+ Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+ Type *ResType = I.getType();
+ Type *ValType = ResType->Type::getStructElementType(0);
+
+ auto Res = getOrCreateVRegs(I);
+ unsigned OldValRes = Res[0];
+ unsigned SuccessRes = Res[1];
+ unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
+ unsigned Cmp = getOrCreateVReg(*I.getCompareOperand());
+ unsigned NewVal = getOrCreateVReg(*I.getNewValOperand());
+
+ MIRBuilder.buildAtomicCmpXchgWithSuccess(
+ OldValRes, SuccessRes, Addr, Cmp, NewVal,
+ *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
+ Flags, DL->getTypeStoreSize(ValType),
+ getMemOpAlignment(I), AAMDNodes(), nullptr,
+ I.getSyncScopeID(), I.getSuccessOrdering(),
+ I.getFailureOrdering()));
+ return true;
+}
+
+bool IRTranslator::translateAtomicRMW(const User &U,
+ MachineIRBuilder &MIRBuilder) {
+ const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
+
+ auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
+ : MachineMemOperand::MONone;
+ Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+ Type *ResType = I.getType();
+
+ unsigned Res = getOrCreateVReg(I);
+ unsigned Addr = getOrCreateVReg(*I.getPointerOperand());
+ unsigned Val = getOrCreateVReg(*I.getValOperand());
+
+ unsigned Opcode = 0;
+ switch (I.getOperation()) {
+ default:
+ llvm_unreachable("Unknown atomicrmw op");
+ return false;
+ case AtomicRMWInst::Xchg:
+ Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
+ break;
+ case AtomicRMWInst::Add:
+ Opcode = TargetOpcode::G_ATOMICRMW_ADD;
+ break;
+ case AtomicRMWInst::Sub:
+ Opcode = TargetOpcode::G_ATOMICRMW_SUB;
+ break;
+ case AtomicRMWInst::And:
+ Opcode = TargetOpcode::G_ATOMICRMW_AND;
+ break;
+ case AtomicRMWInst::Nand:
+ Opcode = TargetOpcode::G_ATOMICRMW_NAND;
+ break;
+ case AtomicRMWInst::Or:
+ Opcode = TargetOpcode::G_ATOMICRMW_OR;
+ break;
+ case AtomicRMWInst::Xor:
+ Opcode = TargetOpcode::G_ATOMICRMW_XOR;
+ break;
+ case AtomicRMWInst::Max:
+ Opcode = TargetOpcode::G_ATOMICRMW_MAX;
+ break;
+ case AtomicRMWInst::Min:
+ Opcode = TargetOpcode::G_ATOMICRMW_MIN;
+ break;
+ case AtomicRMWInst::UMax:
+ Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
+ break;
+ case AtomicRMWInst::UMin:
+ Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
+ break;
+ }
+
+ MIRBuilder.buildAtomicRMW(
+ Opcode, Res, Addr, Val,
+ *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
+ Flags, DL->getTypeStoreSize(ResType),
+ getMemOpAlignment(I), AAMDNodes(), nullptr,
+ I.getSyncScopeID(), I.getOrdering()));
+ return true;
+}
+
void IRTranslator::finishPendingPhis() {
for (auto &Phi : PendingPHIs) {
const PHINode *PI = Phi.first;
.addUse(Idx);
}
+MachineInstrBuilder MachineIRBuilderBase::buildAtomicCmpXchgWithSuccess(
+ unsigned OldValRes, unsigned SuccessRes, unsigned Addr, unsigned CmpVal,
+ unsigned NewVal, MachineMemOperand &MMO) {
+#ifndef NDEBUG
+ LLT OldValResTy = getMRI()->getType(OldValRes);
+ LLT SuccessResTy = getMRI()->getType(SuccessRes);
+ LLT AddrTy = getMRI()->getType(Addr);
+ LLT CmpValTy = getMRI()->getType(CmpVal);
+ LLT NewValTy = getMRI()->getType(NewVal);
+ assert(OldValResTy.isScalar() && "invalid operand type");
+ assert(SuccessResTy.isScalar() && "invalid operand type");
+ assert(AddrTy.isPointer() && "invalid operand type");
+ assert(CmpValTy.isValid() && "invalid operand type");
+ assert(NewValTy.isValid() && "invalid operand type");
+ assert(OldValResTy == CmpValTy && "type mismatch");
+ assert(OldValResTy == NewValTy && "type mismatch");
+#endif
+
+ return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
+ .addDef(OldValRes)
+ .addDef(SuccessRes)
+ .addUse(Addr)
+ .addUse(CmpVal)
+ .addUse(NewVal)
+ .addMemOperand(&MMO);
+}
+
MachineInstrBuilder
MachineIRBuilderBase::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr,
unsigned CmpVal, unsigned NewVal,
.addMemOperand(&MMO);
}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMW(unsigned Opcode, unsigned OldValRes,
+ unsigned Addr, unsigned Val,
+ MachineMemOperand &MMO) {
+#ifndef NDEBUG
+ LLT OldValResTy = getMRI()->getType(OldValRes);
+ LLT AddrTy = getMRI()->getType(Addr);
+ LLT ValTy = getMRI()->getType(Val);
+ assert(OldValResTy.isScalar() && "invalid operand type");
+ assert(AddrTy.isPointer() && "invalid operand type");
+ assert(ValTy.isValid() && "invalid operand type");
+ assert(OldValResTy == ValTy && "type mismatch");
+#endif
+
+ return buildInstr(Opcode)
+ .addDef(OldValRes)
+ .addUse(Addr)
+ .addUse(Val)
+ .addMemOperand(&MMO);
+}
+
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWSub(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWNand(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWOr(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWXor(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWMax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWMin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
+ MMO);
+}
+MachineInstrBuilder
+MachineIRBuilderBase::buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr,
+ unsigned Val, MachineMemOperand &MMO) {
+ return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
+ MMO);
+}
+
void MachineIRBuilderBase::validateTruncExt(unsigned Dst, unsigned Src,
bool IsExtend) {
#ifndef NDEBUG
LLT DstTy = getMRI()->getType(Dst);
if (DstTy.isVector()) {
- assert(SrcTy.isVector() && "mismatched cast between vecot and non-vector");
+ assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
"different number of elements in a trunc/ext");
} else
; CHECK: RET_ReallyLR implicit $w0
ret i1 true
}
+
+; Try one cmpxchg
+define i32 @test_atomic_cmpxchg_1(i32* %addr) {
+; CHECK-LABEL: name: test_atomic_cmpxchg_1
+; CHECK: bb.1.entry:
+; CHECK-NEXT: successors: %bb.{{[^)]+}}
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+; CHECK: bb.2.repeat:
+; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK: [[OLDVALRES:%[0-9]+]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store monotonic monotonic 4 on %ir.addr)
+; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT: G_BR %bb.2
+; CHECK: bb.3.done:
+entry:
+ br label %repeat
+repeat:
+ %val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic
+ %value_loaded = extractvalue { i32, i1 } %val_success, 0
+ %success = extractvalue { i32, i1 } %val_success, 1
+ br i1 %success, label %done, label %repeat
+done:
+ ret i32 %value_loaded
+}
+
+; Try one cmpxchg with a small type and high atomic ordering.
+define i16 @test_atomic_cmpxchg_2(i16* %addr) {
+; CHECK-LABEL: name: test_atomic_cmpxchg_2
+; CHECK: bb.1.entry:
+; CHECK-NEXT: successors: %bb.2({{[^)]+}})
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 0
+; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+; CHECK: bb.2.repeat:
+; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK: [[OLDVALRES:%[0-9]+]]:_(s16), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst seq_cst 2 on %ir.addr)
+; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT: G_BR %bb.2
+; CHECK: bb.3.done:
+entry:
+ br label %repeat
+repeat:
+ %val_success = cmpxchg i16* %addr, i16 0, i16 1 seq_cst seq_cst
+ %value_loaded = extractvalue { i16, i1 } %val_success, 0
+ %success = extractvalue { i16, i1 } %val_success, 1
+ br i1 %success, label %done, label %repeat
+done:
+ ret i16 %value_loaded
+}
+
+; Try one cmpxchg where the success order and failure order differ.
+define i64 @test_atomic_cmpxchg_3(i64* %addr) {
+; CHECK-LABEL: name: test_atomic_cmpxchg_3
+; CHECK: bb.1.entry:
+; CHECK-NEXT: successors: %bb.2({{[^)]+}})
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[OLDVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+; CHECK: bb.2.repeat:
+; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}})
+; CHECK: [[OLDVALRES:%[0-9]+]]:_(s64), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst acquire 8 on %ir.addr)
+; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3
+; CHECK-NEXT: G_BR %bb.2
+; CHECK: bb.3.done:
+entry:
+ br label %repeat
+repeat:
+ %val_success = cmpxchg i64* %addr, i64 0, i64 1 seq_cst acquire
+ %value_loaded = extractvalue { i64, i1 } %val_success, 0
+ %success = extractvalue { i64, i1 } %val_success, 1
+ br i1 %success, label %done, label %repeat
+done:
+ ret i64 %value_loaded
+}
+
+; Try a monotonic atomicrmw xchg
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_xchg(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_xchg
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw xchg i256* %addr, i256 1 monotonic
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an acquire atomicrmw add
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_add(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_add
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw add i256* %addr, i256 1 acquire
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try a release atomicrmw sub
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_sub(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_sub
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw sub i256* %addr, i256 1 release
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an acq_rel atomicrmw and
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_and(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_and
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw and i256* %addr, i256 1 acq_rel
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw nand
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_nand(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_nand
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_NAND [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw nand i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw or
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_or(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_or
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw or i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw xor
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_xor(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_xor
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw xor i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw min
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_min(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_min
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw min i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw max
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_max(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_max
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw max i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw unsigned min
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_umin(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_umin
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw umin i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}
+
+; Try an seq_cst atomicrmw unsigned max
+; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this.
+define i32 @test_atomicrmw_umax(i256* %addr) {
+; CHECK-LABEL: name: test_atomicrmw_umax
+; CHECK: bb.1 (%ir-block.{{[0-9]+}}):
+; CHECK-NEXT: liveins: $x0
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1
+; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst 32 on %ir.addr)
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]]
+ %oldval = atomicrmw umax i256* %addr, i256 1 seq_cst
+ ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this
+ ; test so work around it by truncating to i32 for now.
+ %oldval.trunc = trunc i256 %oldval to i32
+ ret i32 %oldval.trunc
+}