Lines Matching refs:PMV
591 raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { in operator <<() argument
601 PrintObj(PMV.WordType); in operator <<()
603 PrintObj(PMV.ValueType); in operator <<()
605 PrintObj(PMV.AlignedAddr); in operator <<()
607 PrintObj(PMV.ShiftAmt); in operator <<()
609 PrintObj(PMV.Mask); in operator <<()
611 PrintObj(PMV.Inv_Mask); in operator <<()
637 PartwordMaskValues PMV; in createMaskInstrs() local
644 PMV.ValueType = ValueType; in createMaskInstrs()
645 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8) in createMaskInstrs()
647 if (PMV.ValueType == PMV.WordType) { in createMaskInstrs()
648 PMV.AlignedAddr = Addr; in createMaskInstrs()
649 return PMV; in createMaskInstrs()
655 PMV.WordType->getPointerTo(Addr->getType()->getPointerAddressSpace()); in createMaskInstrs()
658 PMV.AlignedAddr = Builder.CreateIntToPtr( in createMaskInstrs()
665 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3); in createMaskInstrs()
668 PMV.ShiftAmt = Builder.CreateShl( in createMaskInstrs()
672 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt"); in createMaskInstrs()
673 PMV.Mask = Builder.CreateShl( in createMaskInstrs()
674 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt, in createMaskInstrs()
676 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask"); in createMaskInstrs()
677 return PMV; in createMaskInstrs()
681 const PartwordMaskValues &PMV) { in extractMaskedValue() argument
682 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in extractMaskedValue()
683 if (PMV.WordType == PMV.ValueType) in extractMaskedValue()
686 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted"); in extractMaskedValue()
687 Value *Trunc = Builder.CreateTrunc(Shift, PMV.ValueType, "extracted"); in extractMaskedValue()
692 Value *Updated, const PartwordMaskValues &PMV) { in insertMaskedValue() argument
693 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch"); in insertMaskedValue()
694 assert(Updated->getType() == PMV.ValueType && "Value type mismatch"); in insertMaskedValue()
695 if (PMV.WordType == PMV.ValueType) in insertMaskedValue()
698 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended"); in insertMaskedValue()
700 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true); in insertMaskedValue()
701 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked"); in insertMaskedValue()
712 const PartwordMaskValues &PMV) { in performMaskedAtomicOp() argument
718 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
731 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask); in performMaskedAtomicOp()
732 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask); in performMaskedAtomicOp()
743 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV); in performMaskedAtomicOp()
745 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV); in performMaskedAtomicOp()
766 PartwordMaskValues PMV = in expandPartwordAtomicRMW() local
771 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in expandPartwordAtomicRMW()
772 PMV.ShiftAmt, "ValOperand_Shifted"); in expandPartwordAtomicRMW()
776 ValOperand_Shifted, AI->getValOperand(), PMV); in expandPartwordAtomicRMW()
782 insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder, in expandPartwordAtomicRMW()
786 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, in expandPartwordAtomicRMW()
790 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandPartwordAtomicRMW()
804 PartwordMaskValues PMV = in widenPartwordAtomicRMW() local
809 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType), in widenPartwordAtomicRMW()
810 PMV.ShiftAmt, "ValOperand_Shifted"); in widenPartwordAtomicRMW()
816 Builder.CreateOr(PMV.Inv_Mask, ValOperand_Shifted, "AndOperand"); in widenPartwordAtomicRMW()
820 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(Op, PMV.AlignedAddr, in widenPartwordAtomicRMW()
823 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV); in widenPartwordAtomicRMW()
887 PartwordMaskValues PMV = createMaskInstrs( in expandPartwordCmpXchg() local
892 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
894 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt); in expandPartwordCmpXchg()
898 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr); in expandPartwordCmpXchg()
900 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask); in expandPartwordCmpXchg()
905 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2); in expandPartwordCmpXchg()
912 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, CI->getSuccessOrdering(), in expandPartwordCmpXchg()
935 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask); in expandPartwordCmpXchg()
945 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandPartwordCmpXchg()
969 PartwordMaskValues PMV = in expandAtomicRMWToMaskedIntrinsic() local
982 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType), in expandAtomicRMWToMaskedIntrinsic()
983 PMV.ShiftAmt, "ValOperand_Shifted"); in expandAtomicRMWToMaskedIntrinsic()
985 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt, in expandAtomicRMWToMaskedIntrinsic()
987 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV); in expandAtomicRMWToMaskedIntrinsic()
995 PartwordMaskValues PMV = createMaskInstrs( in expandAtomicCmpXchgToMaskedIntrinsic() local
1000 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1003 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt, in expandAtomicCmpXchgToMaskedIntrinsic()
1006 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask, in expandAtomicCmpXchgToMaskedIntrinsic()
1008 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV); in expandAtomicCmpXchgToMaskedIntrinsic()
1012 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success"); in expandAtomicCmpXchgToMaskedIntrinsic()
1212 PartwordMaskValues PMV = in expandAtomicCmpXchg() local
1220 TLI->emitLoadLinked(Builder, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1222 extractMaskedValue(Builder, UnreleasedLoad, PMV); in expandAtomicCmpXchg()
1237 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore"); in expandAtomicCmpXchg()
1240 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV); in expandAtomicCmpXchg()
1242 TLI->emitStoreConditional(Builder, NewValueInsert, PMV.AlignedAddr, in expandAtomicCmpXchg()
1253 SecondLoad = TLI->emitLoadLinked(Builder, PMV.AlignedAddr, MemOpOrder); in expandAtomicCmpXchg()
1254 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV); in expandAtomicCmpXchg()
1314 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV); in expandAtomicCmpXchg()