/**
* @brief Memory barrier types (see "The JSR-133 Cookbook for Compiler Writers").
- * @details Without context sensitive analysis, the most conservative set of barriers
- * must be issued to ensure the Java Memory Model. Thus the recipe is as follows:
- * -# Use StoreStore barrier before volatile store.
- * -# Use StoreLoad barrier after volatile store.
- * -# Use LoadLoad and LoadStore barrier after each volatile load.
+ * @details We define the combined barrier types that are actually required
+ * by the Java Memory Model, rather than using exactly the terminology from
+ * the JSR-133 cookbook. These should, in many cases, be replaced by acquire/release
+ * primitives. Note that the JSR-133 cookbook generally does not deal with
+ * store atomicity issues, and the recipes there are not always entirely sufficient.
+ * The current recipe is as follows:
+ * -# Use AnyStore ~= (LoadStore | StoreStore) ~= release barrier before volatile store.
+ * -# Use AnyAny barrier after volatile store. (StoreLoad is as expensive.)
+ * -# Use LoadAny barrier ~= (LoadLoad | LoadStore) ~= acquire barrierafter each volatile load.
* -# Use StoreStore barrier after all stores but before return from any constructor whose
- * class has final fields.
+ * class has final fields.
*/
enum MemBarrierKind {
- kLoadStore,
- kLoadLoad,
+ kAnyStore,
+ kLoadAny,
kStoreStore,
- kStoreLoad
+ kAnyAny
};
std::ostream& operator<<(std::ostream& os, const MemBarrierKind& kind);
LIR* success_target = NewLIR0(kPseudoTargetLabel);
lock_success_branch->target = success_target;
- GenMemBarrier(kLoadLoad);
+ GenMemBarrier(kLoadAny);
} else {
// Explicit null-check as slow-path is entered using an IT.
GenNullCheck(rs_r0, opt_flags);
LIR* call_inst = OpReg(kOpBlx/*ne*/, rs_rARM_LR);
OpEndIT(it);
MarkSafepointPC(call_inst);
- GenMemBarrier(kLoadLoad);
+ GenMemBarrier(kLoadAny);
}
}
MarkPossibleNullPointerException(opt_flags);
LoadConstantNoClobber(rs_r3, 0);
LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_r1, rs_r2, NULL);
- GenMemBarrier(kStoreLoad);
+ GenMemBarrier(kAnyStore);
Store32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r3);
LIR* unlock_success_branch = OpUnconditionalBranch(NULL);
OpRegReg(kOpCmp, rs_r1, rs_r2);
LIR* it = OpIT(kCondEq, "EE");
- if (GenMemBarrier(kStoreLoad)) {
+ if (GenMemBarrier(kAnyStore)) {
UpdateIT(it, "TEE");
}
Store32Disp/*eq*/(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r3);
UNIMPLEMENTED(FATAL) << "Should not be called.";
}
+// Generate a CAS with memory_order_seq_cst semantics.
bool ArmMir2Lir::GenInlinedCas(CallInfo* info, bool is_long, bool is_object) {
DCHECK_EQ(cu_->instruction_set, kThumb2);
// Unused - RegLocation rl_src_unsafe = info->args[0];
}
}
- // Release store semantics, get the barrier out of the way. TODO: revisit
- GenMemBarrier(kStoreLoad);
+ // Prevent reordering with prior memory operations.
+ GenMemBarrier(kAnyStore);
RegLocation rl_object = LoadValue(rl_src_obj, kRefReg);
RegLocation rl_new_value;
FreeTemp(rl_expected.reg); // Now unneeded.
}
+ // Prevent reordering with subsequent memory operations.
+ GenMemBarrier(kLoadAny);
+
// result := (tmp1 != 0) ? 0 : 1;
RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
OpRegRegImm(kOpRsub, rl_result.reg, r_tmp, 1);
int dmb_flavor;
// TODO: revisit Arm barrier kinds
switch (barrier_kind) {
- case kLoadStore: dmb_flavor = kISH; break;
- case kLoadLoad: dmb_flavor = kISH; break;
+ case kAnyStore: dmb_flavor = kISH; break;
+ case kLoadAny: dmb_flavor = kISH; break;
case kStoreStore: dmb_flavor = kISHST; break;
- case kStoreLoad: dmb_flavor = kISH; break;
+ case kAnyAny: dmb_flavor = kISH; break;
default:
LOG(FATAL) << "Unexpected MemBarrierKind: " << barrier_kind;
dmb_flavor = kSY; // quiet gcc.
}
if (UNLIKELY(is_volatile == kVolatile)) {
- // Without context sensitive analysis, we must issue the most conservative barriers.
- // In this case, either a load or store may follow so we issue both barriers.
- GenMemBarrier(kLoadLoad);
- GenMemBarrier(kLoadStore);
+ GenMemBarrier(kLoadAny);
}
return load;
LIR* ArmMir2Lir::StoreBaseDisp(RegStorage r_base, int displacement, RegStorage r_src,
OpSize size, VolatileKind is_volatile) {
if (UNLIKELY(is_volatile == kVolatile)) {
- // There might have been a store before this volatile one so insert StoreStore barrier.
- GenMemBarrier(kStoreStore);
+ // Ensure that prior accesses become visible to other threads first.
+ GenMemBarrier(kAnyStore);
}
LIR* store;
}
if (UNLIKELY(is_volatile == kVolatile)) {
- // A load might follow the volatile store so insert a StoreLoad barrier.
- GenMemBarrier(kStoreLoad);
+ // Preserve order with respect to any subsequent volatile loads.
+ // We need StoreLoad, but that generally requires the most expensive barrier.
+ GenMemBarrier(kAnyAny);
}
return store;
kST = 0xe,
kISH = 0xb,
kISHST = 0xa,
+ kISHLD = 0x9,
kNSH = 0x7,
kNSHST = 0x6
};
LIR* success_target = NewLIR0(kPseudoTargetLabel);
lock_success_branch->target = success_target;
- GenMemBarrier(kLoadLoad);
+ GenMemBarrier(kLoadAny);
}
/*
Load32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_w2);
MarkPossibleNullPointerException(opt_flags);
LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_w1, rs_w2, NULL);
- GenMemBarrier(kStoreLoad);
+ GenMemBarrier(kAnyStore);
Store32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_wzr);
LIR* unlock_success_branch = OpUnconditionalBranch(NULL);
int dmb_flavor;
// TODO: revisit Arm barrier kinds
switch (barrier_kind) {
- case kLoadStore: dmb_flavor = kISH; break;
- case kLoadLoad: dmb_flavor = kISH; break;
+ case kAnyStore: dmb_flavor = kISH; break;
+ case kLoadAny: dmb_flavor = kISH; break;
+ // We conjecture that kISHLD is insufficient. It is documented
+ // to provide LoadLoad | StoreStore ordering. But if this were used
+ // to implement volatile loads, we suspect that the lack of store
+ // atomicity on ARM would cause us to allow incorrect results for
+ // the canonical IRIW example. But we're not sure.
+ // We should be using acquire loads instead.
case kStoreStore: dmb_flavor = kISHST; break;
- case kStoreLoad: dmb_flavor = kISH; break;
+ case kAnyAny: dmb_flavor = kISH; break;
default:
LOG(FATAL) << "Unexpected MemBarrierKind: " << barrier_kind;
dmb_flavor = kSY; // quiet gcc.
LIR* load = LoadBaseDispBody(r_base, displacement, r_dest, size);
if (UNLIKELY(is_volatile == kVolatile)) {
- // Without context sensitive analysis, we must issue the most conservative barriers.
- // In this case, either a load or store may follow so we issue both barriers.
- GenMemBarrier(kLoadLoad);
- GenMemBarrier(kLoadStore);
+ // TODO: This should generate an acquire load instead of the barrier.
+ GenMemBarrier(kLoadAny);
}
return load;
LIR* Arm64Mir2Lir::StoreBaseDisp(RegStorage r_base, int displacement, RegStorage r_src,
OpSize size, VolatileKind is_volatile) {
+ // TODO: This should generate a release store and no barriers.
if (UNLIKELY(is_volatile == kVolatile)) {
- // There might have been a store before this volatile one so insert StoreStore barrier.
- GenMemBarrier(kStoreStore);
+ // Ensure that prior accesses become visible to other threads first.
+ GenMemBarrier(kAnyStore);
}
// StoreBaseDisp() will emit correct insn for atomic store on arm64
LIR* store = StoreBaseDispBody(r_base, displacement, r_src, size);
if (UNLIKELY(is_volatile == kVolatile)) {
- // A load might follow the volatile store so insert a StoreLoad barrier.
- GenMemBarrier(kStoreLoad);
+ // Preserve order with respect to any subsequent volatile loads.
+ // We need StoreLoad, but that generally requires the most expensive barrier.
+ GenMemBarrier(kAnyAny);
}
return store;
field_info.StorageIndex(), r_base));
FreeTemp(r_tmp);
- // Ensure load of status and load of value don't re-order.
- GenMemBarrier(kLoadLoad);
+ // Ensure load of status and store of value don't re-order.
+ // TODO: Presumably the actual value store is control-dependent on the status load,
+ // and will thus not be reordered in any case, since stores are never speculated.
+ // Does later code "know" that the class is now initialized? If so, we still
+ // need the barrier to guard later static loads.
+ GenMemBarrier(kLoadAny);
}
FreeTemp(r_method);
}
FreeTemp(r_tmp);
// Ensure load of status and load of value don't re-order.
- GenMemBarrier(kLoadLoad);
+ GenMemBarrier(kLoadAny);
}
FreeTemp(r_method);
}
}
if (is_volatile) {
- // Without context sensitive analysis, we must issue the most conservative barriers.
- // In this case, either a load or store may follow so we issue both barriers.
- GenMemBarrier(kLoadLoad);
- GenMemBarrier(kLoadStore);
+ GenMemBarrier(kLoadAny);
}
if (is_long) {
rl_src_offset = NarrowRegLoc(rl_src_offset); // ignore high half in info->args[3]
RegLocation rl_src_value = info->args[4]; // value to store
if (is_volatile || is_ordered) {
- // There might have been a store before this volatile one so insert StoreStore barrier.
- GenMemBarrier(kStoreStore);
+ GenMemBarrier(kAnyStore);
}
RegLocation rl_object = LoadValue(rl_src_obj, kRefReg);
RegLocation rl_offset = LoadValue(rl_src_offset, kCoreReg);
FreeTemp(rl_offset.reg);
if (is_volatile) {
- // A load might follow the volatile store so insert a StoreLoad barrier.
- GenMemBarrier(kStoreLoad);
+ // Prevent reordering with a subsequent volatile load.
+ // May also be needed to address store atomicity issues.
+ GenMemBarrier(kAnyAny);
}
if (is_object) {
MarkGCCard(rl_value.reg, rl_object.reg);
load = LoadBaseDispBody(r_base, displacement, r_dest, size);
if (UNLIKELY(is_volatile == kVolatile)) {
- // Without context sensitive analysis, we must issue the most conservative barriers.
- // In this case, either a load or store may follow so we issue both barriers.
- GenMemBarrier(kLoadLoad);
- GenMemBarrier(kLoadStore);
+ GenMemBarrier(kLoadAny);
}
return load;
OpSize size, VolatileKind is_volatile) {
if (is_volatile == kVolatile) {
DCHECK(size != k64 && size != kDouble);
- // There might have been a store before this volatile one so insert StoreStore barrier.
- GenMemBarrier(kStoreStore);
+ // Ensure that prior accesses become visible to other threads first.
+ GenMemBarrier(kAnyStore);
}
// TODO: base this on target.
store = StoreBaseDispBody(r_base, displacement, r_src, size);
if (UNLIKELY(is_volatile == kVolatile)) {
- // A load might follow the volatile store so insert a StoreLoad barrier.
- GenMemBarrier(kStoreLoad);
+ // Preserve order with respect to any subsequent volatile loads.
+ // We need StoreLoad, but that generally requires the most expensive barrier.
+ GenMemBarrier(kAnyAny);
}
return store;
// After a store we need to insert barrier in case of potential load. Since the
// locked cmpxchg has full barrier semantics, only a scheduling barrier will be generated.
- GenMemBarrier(kStoreLoad);
+ GenMemBarrier(kAnyAny);
FreeTemp(rs_r0q);
} else if (is_long) {
}
NewLIR4(kX86LockCmpxchg64A, rs_obj.GetReg(), rs_off.GetReg(), 0, 0);
- // After a store we need to insert barrier in case of potential load. Since the
- // locked cmpxchg has full barrier semantics, only a scheduling barrier will be generated.
- GenMemBarrier(kStoreLoad);
-
+ // After a store we need to insert barrier to prevent reordering with either
+ // earlier or later memory accesses. Since
+ // locked cmpxchg has full barrier semantics, only a scheduling barrier will be generated,
+ // and it will be associated with the cmpxchg instruction, preventing both.
+ GenMemBarrier(kAnyAny);
if (push_si) {
FreeTemp(rs_rSI);
LoadValueDirect(rl_src_expected, rs_r0);
NewLIR5(kX86LockCmpxchgAR, rl_object.reg.GetReg(), rl_offset.reg.GetReg(), 0, 0, rl_new_value.reg.GetReg());
- // After a store we need to insert barrier in case of potential load. Since the
- // locked cmpxchg has full barrier semantics, only a scheduling barrier will be generated.
- GenMemBarrier(kStoreLoad);
+ // After a store we need to insert barrier to prevent reordering with either
+ // earlier or later memory accesses. Since
+ // locked cmpxchg has full barrier semantics, only a scheduling barrier will be generated,
+ // and it will be associated with the cmpxchg instruction, preventing both.
+ GenMemBarrier(kAnyAny);
FreeTemp(rs_r0);
}
bool ret = false;
/*
- * According to the JSR-133 Cookbook, for x86 only StoreLoad barriers need memory fence. All other barriers
- * (LoadLoad, LoadStore, StoreStore) are nops due to the x86 memory model. For those cases, all we need
- * to ensure is that there is a scheduling barrier in place.
+ * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
+ * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
+ * For those cases, all we need to ensure is that there is a scheduling barrier in place.
*/
- if (barrier_kind == kStoreLoad) {
+ if (barrier_kind == kAnyAny) {
// If no LIR exists already that can be used a barrier, then generate an mfence.
if (mem_barrier == nullptr) {
mem_barrier = NewLIR0(kX86Mfence);
size);
if (UNLIKELY(is_volatile == kVolatile)) {
- // Without context sensitive analysis, we must issue the most conservative barriers.
- // In this case, either a load or store may follow so we issue both barriers.
- GenMemBarrier(kLoadLoad);
- GenMemBarrier(kLoadStore);
+ GenMemBarrier(kLoadAny); // Only a scheduling barrier.
}
return load;
LIR* X86Mir2Lir::StoreBaseDisp(RegStorage r_base, int displacement, RegStorage r_src, OpSize size,
VolatileKind is_volatile) {
if (UNLIKELY(is_volatile == kVolatile)) {
- // There might have been a store before this volatile one so insert StoreStore barrier.
- GenMemBarrier(kStoreStore);
+ GenMemBarrier(kAnyStore); // Only a scheduling barrier.
}
// StoreBaseDisp() will emit correct insn for atomic store on x86
LIR* store = StoreBaseIndexedDisp(r_base, RegStorage::InvalidReg(), 0, displacement, r_src, size);
if (UNLIKELY(is_volatile == kVolatile)) {
- // A load might follow the volatile store so insert a StoreLoad barrier.
- GenMemBarrier(kStoreLoad);
+ // A volatile load might follow the volatile store so insert a StoreLoad barrier.
+ // This does require a fence, even on x86.
+ GenMemBarrier(kAnyAny);
}
return store;
field_value = SignOrZeroExtendCat1Types(field_value, field_jty);
if (is_volatile) {
- irb_.CreateMemoryBarrier(art::kLoadLoad);
+ irb_.CreateMemoryBarrier(art::kLoadAny);
}
}
DCHECK_GE(field_offset.Int32Value(), 0);
if (is_volatile) {
- irb_.CreateMemoryBarrier(art::kStoreStore);
+ irb_.CreateMemoryBarrier(art::kAnyStore);
}
llvm::PointerType* field_type =
irb_.CreateStore(new_value, field_addr, kTBAAHeapInstance, field_jty);
if (is_volatile) {
- irb_.CreateMemoryBarrier(art::kLoadLoad);
+ irb_.CreateMemoryBarrier(art::kAnyAny);
}
if (field_jty == kObject) { // If put an object, mark the GC card table.
phi->addIncoming(loaded_storage_object_addr, block_after_load_static);
// Ensure load of status and load of value don't re-order.
- irb_.CreateMemoryBarrier(art::kLoadLoad);
+ irb_.CreateMemoryBarrier(art::kLoadAny);
return phi;
}
static_field_value = SignOrZeroExtendCat1Types(static_field_value, field_jty);
if (is_volatile) {
- irb_.CreateMemoryBarrier(art::kLoadLoad);
+ irb_.CreateMemoryBarrier(art::kLoadAny);
}
}
}
if (is_volatile) {
- irb_.CreateMemoryBarrier(art::kStoreStore);
+ irb_.CreateMemoryBarrier(art::kAnyStore);
}
llvm::Value* static_field_offset_value = irb_.getPtrEquivInt(field_offset.Int32Value());
irb_.CreateStore(new_value, static_field_addr, kTBAAHeapStatic, field_jty);
if (is_volatile) {
- irb_.CreateMemoryBarrier(art::kStoreLoad);
+ irb_.CreateMemoryBarrier(art::kAnyAny);
}
if (field_jty == kObject) { // If put an object, mark the GC card table.
OSDN Git Service
