1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers atomic intrinsics to non-atomic form for use in a known
10 // non-preemptible environment.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Scalar/LowerAtomic.h"
15 #include "llvm/IR/Function.h"
16 #include "llvm/IR/IRBuilder.h"
17 #include "llvm/InitializePasses.h"
18 #include "llvm/Pass.h"
19 #include "llvm/Transforms/Scalar.h"
20 using namespace llvm;
21
22 #define DEBUG_TYPE "loweratomic"
23
LowerAtomicCmpXchgInst(AtomicCmpXchgInst * CXI)24 static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
25 IRBuilder<> Builder(CXI);
26 Value *Ptr = CXI->getPointerOperand();
27 Value *Cmp = CXI->getCompareOperand();
28 Value *Val = CXI->getNewValOperand();
29
30 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
31 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
32 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
33 Builder.CreateStore(Res, Ptr);
34
35 Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
36 Res = Builder.CreateInsertValue(Res, Equal, 1);
37
38 CXI->replaceAllUsesWith(Res);
39 CXI->eraseFromParent();
40 return true;
41 }
42
LowerAtomicRMWInst(AtomicRMWInst * RMWI)43 static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
44 IRBuilder<> Builder(RMWI);
45 Value *Ptr = RMWI->getPointerOperand();
46 Value *Val = RMWI->getValOperand();
47
48 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
49 Value *Res = nullptr;
50
51 switch (RMWI->getOperation()) {
52 default: llvm_unreachable("Unexpected RMW operation");
53 case AtomicRMWInst::Xchg:
54 Res = Val;
55 break;
56 case AtomicRMWInst::Add:
57 Res = Builder.CreateAdd(Orig, Val);
58 break;
59 case AtomicRMWInst::Sub:
60 Res = Builder.CreateSub(Orig, Val);
61 break;
62 case AtomicRMWInst::And:
63 Res = Builder.CreateAnd(Orig, Val);
64 break;
65 case AtomicRMWInst::Nand:
66 Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
67 break;
68 case AtomicRMWInst::Or:
69 Res = Builder.CreateOr(Orig, Val);
70 break;
71 case AtomicRMWInst::Xor:
72 Res = Builder.CreateXor(Orig, Val);
73 break;
74 case AtomicRMWInst::Max:
75 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
76 Val, Orig);
77 break;
78 case AtomicRMWInst::Min:
79 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
80 Orig, Val);
81 break;
82 case AtomicRMWInst::UMax:
83 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
84 Val, Orig);
85 break;
86 case AtomicRMWInst::UMin:
87 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
88 Orig, Val);
89 break;
90 case AtomicRMWInst::FAdd:
91 Res = Builder.CreateFAdd(Orig, Val);
92 break;
93 case AtomicRMWInst::FSub:
94 Res = Builder.CreateFSub(Orig, Val);
95 break;
96 }
97 Builder.CreateStore(Res, Ptr);
98 RMWI->replaceAllUsesWith(Orig);
99 RMWI->eraseFromParent();
100 return true;
101 }
102
LowerFenceInst(FenceInst * FI)103 static bool LowerFenceInst(FenceInst *FI) {
104 FI->eraseFromParent();
105 return true;
106 }
107
LowerLoadInst(LoadInst * LI)108 static bool LowerLoadInst(LoadInst *LI) {
109 LI->setAtomic(AtomicOrdering::NotAtomic);
110 return true;
111 }
112
LowerStoreInst(StoreInst * SI)113 static bool LowerStoreInst(StoreInst *SI) {
114 SI->setAtomic(AtomicOrdering::NotAtomic);
115 return true;
116 }
117
runOnBasicBlock(BasicBlock & BB)118 static bool runOnBasicBlock(BasicBlock &BB) {
119 bool Changed = false;
120 for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE;) {
121 Instruction *Inst = &*DI++;
122 if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
123 Changed |= LowerFenceInst(FI);
124 else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
125 Changed |= LowerAtomicCmpXchgInst(CXI);
126 else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
127 Changed |= LowerAtomicRMWInst(RMWI);
128 else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
129 if (LI->isAtomic())
130 LowerLoadInst(LI);
131 } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
132 if (SI->isAtomic())
133 LowerStoreInst(SI);
134 }
135 }
136 return Changed;
137 }
138
lowerAtomics(Function & F)139 static bool lowerAtomics(Function &F) {
140 bool Changed = false;
141 for (BasicBlock &BB : F) {
142 Changed |= runOnBasicBlock(BB);
143 }
144 return Changed;
145 }
146
run(Function & F,FunctionAnalysisManager &)147 PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) {
148 if (lowerAtomics(F))
149 return PreservedAnalyses::none();
150 return PreservedAnalyses::all();
151 }
152
153 namespace {
154 class LowerAtomicLegacyPass : public FunctionPass {
155 public:
156 static char ID;
157
LowerAtomicLegacyPass()158 LowerAtomicLegacyPass() : FunctionPass(ID) {
159 initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry());
160 }
161
runOnFunction(Function & F)162 bool runOnFunction(Function &F) override {
163 // Don't skip optnone functions; atomics still need to be lowered.
164 FunctionAnalysisManager DummyFAM;
165 auto PA = Impl.run(F, DummyFAM);
166 return !PA.areAllPreserved();
167 }
168
169 private:
170 LowerAtomicPass Impl;
171 };
172 }
173
174 char LowerAtomicLegacyPass::ID = 0;
175 INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic",
176 "Lower atomic intrinsics to non-atomic form", false, false)
177
createLowerAtomicPass()178 Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); }
179