80     bool tryExpandAtomicRMW(AtomicRMWInst *AI);
89         AtomicRMWInst *I,
91     AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI);
93     void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);
105     bool isIdempotentRMW(AtomicRMWInst *RMWI);
106     bool simplifyIdempotentRMW(AtomicRMWInst *RMWI);
115     void expandAtomicRMWToLibcall(AtomicRMWInst *I);
119     llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
145 static unsigned getAtomicOpSize(AtomicRMWInst *RMWI) {  in getAtomicOpSize()
190     auto RMWI = dyn_cast<AtomicRMWInst>(I);  in runOnFunction()
285         AtomicRMWInst::BinOp Op = RMWI->getOperation();  in runOnFunction()
287             (Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||  in runOnFunction()
288              Op == AtomicRMWInst::And)) {  in runOnFunction()
457   AtomicRMWInst *AI =  in expandAtomicStore()
458       Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(),  in expandAtomicStore()
494 static Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder,  in performAtomicOp()
498   case AtomicRMWInst::Xchg:  in performAtomicOp()
500   case AtomicRMWInst::Add:  in performAtomicOp()
502   case AtomicRMWInst::Sub:  in performAtomicOp()
504   case AtomicRMWInst::And:  in performAtomicOp()
506   case AtomicRMWInst::Nand:  in performAtomicOp()
508   case AtomicRMWInst::Or:  in performAtomicOp()
510   case AtomicRMWInst::Xor:  in performAtomicOp()
512   case AtomicRMWInst::Max:  in performAtomicOp()
515   case AtomicRMWInst::Min:  in performAtomicOp()
518   case AtomicRMWInst::UMax:  in performAtomicOp()
521   case AtomicRMWInst::UMin:  in performAtomicOp()
524   case AtomicRMWInst::FAdd:  in performAtomicOp()
526   case AtomicRMWInst::FSub:  in performAtomicOp()
533 bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {  in tryExpandAtomicRMW()
709 static Value *performMaskedAtomicOp(AtomicRMWInst::BinOp Op,  in performMaskedAtomicOp()
717   case AtomicRMWInst::Xchg: {  in performMaskedAtomicOp()
722   case AtomicRMWInst::Or:  in performMaskedAtomicOp()
723   case AtomicRMWInst::Xor:  in performMaskedAtomicOp()
724   case AtomicRMWInst::And:  in performMaskedAtomicOp()
726   case AtomicRMWInst::Add:  in performMaskedAtomicOp()
727   case AtomicRMWInst::Sub:  in performMaskedAtomicOp()
728   case AtomicRMWInst::Nand: {  in performMaskedAtomicOp()
736   case AtomicRMWInst::Max:  in performMaskedAtomicOp()
737   case AtomicRMWInst::Min:  in performMaskedAtomicOp()
738   case AtomicRMWInst::UMax:  in performMaskedAtomicOp()
739   case AtomicRMWInst::UMin: {  in performMaskedAtomicOp()
761     AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {  in expandPartwordAtomicRMW()
796 AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {  in widenPartwordAtomicRMW()
798   AtomicRMWInst::BinOp Op = AI->getOperation();  in widenPartwordAtomicRMW()
800   assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||  in widenPartwordAtomicRMW()
801           Op == AtomicRMWInst::And) &&  in widenPartwordAtomicRMW()
814   if (Op == AtomicRMWInst::And)  in widenPartwordAtomicRMW()
820   AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(Op, PMV.AlignedAddr,  in widenPartwordAtomicRMW()
966 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {  in expandAtomicRMWToMaskedIntrinsic()
977   AtomicRMWInst::BinOp RMWOp = AI->getOperation();  in expandAtomicRMWToMaskedIntrinsic()
978   if (RMWOp == AtomicRMWInst::Max || RMWOp == AtomicRMWInst::Min)  in expandAtomicRMWToMaskedIntrinsic()
1353 bool AtomicExpand::isIdempotentRMW(AtomicRMWInst* RMWI) {  in isIdempotentRMW()
1358   AtomicRMWInst::BinOp Op = RMWI->getOperation();  in isIdempotentRMW()
1360     case AtomicRMWInst::Add:  in isIdempotentRMW()
1361     case AtomicRMWInst::Sub:  in isIdempotentRMW()
1362     case AtomicRMWInst::Or:  in isIdempotentRMW()
1363     case AtomicRMWInst::Xor:  in isIdempotentRMW()
1365     case AtomicRMWInst::And:  in isIdempotentRMW()
1373 bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) {  in simplifyIdempotentRMW()
1465 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,  in expandAtomicRMWToCmpXchg()
1542 static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {  in GetRMWLibcall()
1573   case AtomicRMWInst::BAD_BINOP:  in GetRMWLibcall()
1575   case AtomicRMWInst::Xchg:  in GetRMWLibcall()
1577   case AtomicRMWInst::Add:  in GetRMWLibcall()
1579   case AtomicRMWInst::Sub:  in GetRMWLibcall()
1581   case AtomicRMWInst::And:  in GetRMWLibcall()
1583   case AtomicRMWInst::Or:  in GetRMWLibcall()
1585   case AtomicRMWInst::Xor:  in GetRMWLibcall()
1587   case AtomicRMWInst::Nand:  in GetRMWLibcall()
1589   case AtomicRMWInst::Max:  in GetRMWLibcall()
1590   case AtomicRMWInst::Min:  in GetRMWLibcall()
1591   case AtomicRMWInst::UMax:  in GetRMWLibcall()
1592   case AtomicRMWInst::UMin:  in GetRMWLibcall()
1593   case AtomicRMWInst::FAdd:  in GetRMWLibcall()
1594   case AtomicRMWInst::FSub:  in GetRMWLibcall()
1601 void AtomicExpand::expandAtomicRMWToLibcall(AtomicRMWInst *I) {  in expandAtomicRMWToLibcall()