Skip to content

Commit a9a05df

Browse files
LancernLancern
authored
[CIR] Scoped atomic store (#171627)
This patch adds support for `__scoped_atomic_store` and `__scoped_atomic_store_n`.
1 parent 6b7b0ab commit a9a05df

File tree

8 files changed

+63
-20
lines changed

8 files changed

+63
-20
lines changed

‎clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,8 +330,10 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
330330
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
331331
bool isVolatile = false,
332332
mlir::IntegerAttr align = {},
333+
cir::SyncScopeKindAttr scope = {},
333334
cir::MemOrderAttr order = {}) {
334-
return cir::StoreOp::create(*this, loc, val, dst, isVolatile, align, order);
335+
return cir::StoreOp::create(*this, loc, val, dst, isVolatile, align, scope,
336+
order);
335337
}
336338

337339
/// Emit a load from an boolean flag variable.

‎clang/include/clang/CIR/Dialect/IR/CIROps.td‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,11 +671,13 @@ def CIR_StoreOp : CIR_Op<"store", [
671671
[MemWrite]>:$addr,
672672
UnitAttr:$is_volatile,
673673
OptionalAttr<I64Attr>:$alignment,
674+
OptionalAttr<CIR_SyncScopeKind>:$sync_scope,
674675
OptionalAttr<CIR_MemOrder>:$mem_order);
675676

676677
let assemblyFormat = [{
677678
(`volatile` $is_volatile^)?
678679
(`align` `(` $alignment^ `)`)?
680+
(`syncscope` `(` $sync_scope^ `)`)?
679681
(`atomic` `(` $mem_order^ `)`)?
680682
$value `,` $addr attr-dict `:` type($value) `,` qualified(type($addr))
681683
}];

‎clang/lib/CIR/CodeGen/CIRGenAtomic.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -455,13 +455,15 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
455455

456456
case AtomicExpr::AO__c11_atomic_store:
457457
case AtomicExpr::AO__atomic_store_n:
458-
case AtomicExpr::AO__atomic_store: {
458+
case AtomicExpr::AO__atomic_store:
459+
case AtomicExpr::AO__scoped_atomic_store:
460+
case AtomicExpr::AO__scoped_atomic_store_n: {
459461
cir::LoadOp loadVal1 = builder.createLoad(loc, val1);
460462

461463
assert(!cir::MissingFeatures::atomicSyncScopeID());
462464

463465
builder.createStore(loc, loadVal1, ptr, expr->isVolatile(),
464-
/*align=*/mlir::IntegerAttr{}, orderAttr);
466+
/*align=*/mlir::IntegerAttr{}, scopeAttr, orderAttr);
465467
return;
466468
}
467469

@@ -584,8 +586,6 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
584586

585587
case AtomicExpr::AO__opencl_atomic_store:
586588
case AtomicExpr::AO__hip_atomic_store:
587-
case AtomicExpr::AO__scoped_atomic_store:
588-
case AtomicExpr::AO__scoped_atomic_store_n:
589589

590590
case AtomicExpr::AO__hip_atomic_exchange:
591591
case AtomicExpr::AO__opencl_atomic_exchange:
@@ -849,6 +849,7 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
849849
break;
850850

851851
case AtomicExpr::AO__atomic_store:
852+
case AtomicExpr::AO__scoped_atomic_store:
852853
val1 = emitPointerWithAlignment(e->getVal1());
853854
break;
854855

@@ -912,6 +913,7 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
912913
case AtomicExpr::AO__c11_atomic_fetch_xor:
913914
case AtomicExpr::AO__c11_atomic_exchange:
914915
case AtomicExpr::AO__c11_atomic_store:
916+
case AtomicExpr::AO__scoped_atomic_store_n:
915917
val1 = emitValToTemp(*this, e->getVal1());
916918
break;
917919
}

‎clang/lib/CIR/CodeGen/CIRGenBuilder.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,11 +492,12 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
492492
cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst,
493493
bool isVolatile = false,
494494
mlir::IntegerAttr align = {},
495+
cir::SyncScopeKindAttr scope = {},
495496
cir::MemOrderAttr order = {}) {
496497
if (!align)
497498
align = getAlignmentAttr(dst.getAlignment());
498499
return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), isVolatile,
499-
align, order);
500+
align, scope, order);
500501
}
501502

502503
/// Create a cir.complex.real_ptr operation that derives a pointer to the real

‎clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ DeletionKind cir::CopyOp::removeBlockingUses(
143143
cir::StoreOp::create(builder, getLoc(), reachingDefinition, getDst(),
144144
/*isVolatile=*/false,
145145
/*alignment=*/mlir::IntegerAttr{},
146+
/*sync_scope=*/cir::SyncScopeKindAttr(),
146147
/*mem-order=*/cir::MemOrderAttr());
147148
return DeletionKind::Delete;
148149
}

‎clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1684,13 +1684,16 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
16841684
// Convert adapted value to its memory type if needed.
16851685
mlir::Value value = emitToMemory(rewriter, dataLayout,
16861686
op.getValue().getType(), adaptor.getValue());
1687-
// TODO: nontemporal, syncscope.
1687+
// TODO: nontemporal.
16881688
assert(!cir::MissingFeatures::opLoadStoreNontemporal());
16891689
assert(!cir::MissingFeatures::opLoadStoreTbaa());
1690+
std::optional<llvm::StringRef> syncScope =
1691+
getLLVMSyncScope(op.getSyncScope());
16901692
mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
16911693
rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
16921694
op.getIsVolatile(),
1693-
/*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder);
1695+
/*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder,
1696+
syncScope.value_or(llvm::StringRef()));
16941697
rewriter.replaceOp(op, storeOp);
16951698
assert(!cir::MissingFeatures::opLoadStoreTbaa());
16961699
return mlir::LogicalResult::success();

‎clang/test/CIR/CodeGen/atomic-scoped.c‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,35 @@ void scoped_atomic_load_n(int *ptr) {
3838
// LLVM: %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
3939
// OGCG: %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
4040
}
41+
42+
void scoped_atomic_store(int *ptr, int value) {
43+
// CIR-LABEL: @scoped_atomic_store
44+
// LLVM-LABEL: @scoped_atomic_store
45+
// OGCG-LABEL: @scoped_atomic_store
46+
47+
__scoped_atomic_store(ptr, &value, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE);
48+
// CIR: cir.store align(4) syncscope(single_thread) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
49+
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} syncscope("singlethread") monotonic, align 4
50+
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
51+
52+
__scoped_atomic_store(ptr, &value, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
53+
// CIR: cir.store align(4) syncscope(system) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
54+
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
55+
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
56+
}
57+
58+
void scoped_atomic_store_n(int *ptr, int value) {
59+
// CIR-LABEL: @scoped_atomic_store_n
60+
// LLVM-LABEL: @scoped_atomic_store_n
61+
// OGCG-LABEL: @scoped_atomic_store_n
62+
63+
__scoped_atomic_store_n(ptr, value, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE);
64+
// CIR: cir.store align(4) syncscope(single_thread) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
65+
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} syncscope("singlethread") monotonic, align 4
66+
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
67+
68+
__scoped_atomic_store_n(ptr, value, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
69+
// CIR: cir.store align(4) syncscope(system) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
70+
// LLVM: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
71+
// OGCG: store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
72+
}

‎clang/test/CIR/CodeGen/atomic.c‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,9 @@ void store(int *ptr, int x) {
165165
}
166166

167167
// CIR-LABEL: @store
168-
// CIR: cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
169-
// CIR: cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
170-
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
168+
// CIR: cir.store align(4) syncscope(system) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
169+
// CIR: cir.store align(4) syncscope(system) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
170+
// CIR: cir.store align(4) syncscope(system) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
171171
// CIR: }
172172

173173
// LLVM-LABEL: @store
@@ -189,9 +189,9 @@ void store_n(int *ptr, int x) {
189189
}
190190

191191
// CIR-LABEL: @store_n
192-
// CIR: cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
193-
// CIR: cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
194-
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
192+
// CIR: cir.store align(4) syncscope(system) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
193+
// CIR: cir.store align(4) syncscope(system) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
194+
// CIR: cir.store align(4) syncscope(system) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
195195
// CIR: }
196196

197197
// LLVM-LABEL: @store_n
@@ -213,9 +213,9 @@ void c11_store(_Atomic(int) *ptr, int x) {
213213
}
214214

215215
// CIR-LABEL: @c11_store
216-
// CIR: cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
217-
// CIR: cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
218-
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
216+
// CIR: cir.store align(4) syncscope(system) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
217+
// CIR: cir.store align(4) syncscope(system) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
218+
// CIR: cir.store align(4) syncscope(system) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
219219
// CIR: }
220220

221221
// LLVM-LABEL: @c11_store
@@ -1222,17 +1222,17 @@ void atomic_store_dynamic_order(int *ptr, int order) {
12221222
// CIR: cir.switch(%[[ORDER]] : !s32i) {
12231223
// CIR-NEXT: cir.case(default, []) {
12241224
// CIR-NEXT: %[[VALUE:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
1225-
// CIR-NEXT: cir.store align(4) atomic(relaxed) %[[VALUE]], %[[PTR]] : !s32i, !cir.ptr<!s32i>
1225+
// CIR-NEXT: cir.store align(4) syncscope(system) atomic(relaxed) %[[VALUE]], %[[PTR]] : !s32i, !cir.ptr<!s32i>
12261226
// CIR-NEXT: cir.break
12271227
// CIR-NEXT: }
12281228
// CIR-NEXT: cir.case(anyof, [#cir.int<3> : !s32i]) {
12291229
// CIR-NEXT: %[[VALUE:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
1230-
// CIR-NEXT: cir.store align(4) atomic(release) %[[VALUE]], %[[PTR]] : !s32i, !cir.ptr<!s32i>
1230+
// CIR-NEXT: cir.store align(4) syncscope(system) atomic(release) %[[VALUE]], %[[PTR]] : !s32i, !cir.ptr<!s32i>
12311231
// CIR-NEXT: cir.break
12321232
// CIR-NEXT: }
12331233
// CIR-NEXT: cir.case(anyof, [#cir.int<5> : !s32i]) {
12341234
// CIR-NEXT: %[[VALUE:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
1235-
// CIR-NEXT: cir.store align(4) atomic(seq_cst) %[[VALUE]], %[[PTR]] : !s32i, !cir.ptr<!s32i>
1235+
// CIR-NEXT: cir.store align(4) syncscope(system) atomic(seq_cst) %[[VALUE]], %[[PTR]] : !s32i, !cir.ptr<!s32i>
12361236
// CIR-NEXT: cir.break
12371237
// CIR-NEXT: }
12381238
// CIR-NEXT: cir.yield

0 commit comments

Comments
 (0)