Lines Matching refs:MMI
651 MatMulInfoTy &MMI) { in isMatMulNonScalarReadAccess() argument
656 if (isMatMulOperandAcc(StmtDomain, AccMap, MMI.i, MMI.j) && !MMI.ReadFromC) { in isMatMulNonScalarReadAccess()
657 MMI.ReadFromC = MemAccess; in isMatMulNonScalarReadAccess()
660 if (isMatMulOperandAcc(StmtDomain, AccMap, MMI.i, MMI.k) && !MMI.A) { in isMatMulNonScalarReadAccess()
661 MMI.A = MemAccess; in isMatMulNonScalarReadAccess()
664 if (isMatMulOperandAcc(StmtDomain, AccMap, MMI.k, MMI.j) && !MMI.B) { in isMatMulNonScalarReadAccess()
665 MMI.B = MemAccess; in isMatMulNonScalarReadAccess()
685 MatMulInfoTy &MMI) { in containsOnlyMatrMultAcc() argument
693 permuteDimensions(PartialSchedule, isl::dim::out, MMI.i, OutDimNum - 1); in containsOnlyMatrMultAcc()
695 permuteDimensions(PartialSchedule, isl::dim::out, MMI.j, OutDimNum - 1); in containsOnlyMatrMultAcc()
697 permuteDimensions(PartialSchedule, isl::dim::out, MMI.k, OutDimNum - 1); in containsOnlyMatrMultAcc()
702 if (MemAccessPtr->isLatestArrayKind() && MemAccessPtr != MMI.WriteToC && in containsOnlyMatrMultAcc()
703 !isMatMulNonScalarReadAccess(MemAccessPtr, MMI) && in containsOnlyMatrMultAcc()
770 MatMulInfoTy &MMI) { in containsMatrMult() argument
784 if (!isMatMulOperandAcc(Stmt->getDomain(), AccMap, MMI.i, MMI.j)) in containsMatrMult()
786 MMI.WriteToC = MemAccessPtr; in containsMatrMult()
790 if (!containsOnlyMatMulDep(PartialSchedule, D, MMI.k)) in containsMatrMult()
793 if (!MMI.WriteToC || !containsOnlyMatrMultAcc(PartialSchedule, MMI)) in containsMatrMult()
796 if (!MMI.A || !MMI.B || !MMI.ReadFromC) in containsMatrMult()
862 static uint64_t getMatMulAlignTypeSize(MatMulInfoTy MMI) { in getMatMulAlignTypeSize() argument
863 auto *S = MMI.A->getStatement()->getParent(); in getMatMulAlignTypeSize()
865 auto ElementSizeA = DL.getTypeAllocSize(MMI.A->getElementType()); in getMatMulAlignTypeSize()
866 auto ElementSizeB = DL.getTypeAllocSize(MMI.B->getElementType()); in getMatMulAlignTypeSize()
867 auto ElementSizeC = DL.getTypeAllocSize(MMI.WriteToC->getElementType()); in getMatMulAlignTypeSize()
877 static uint64_t getMatMulTypeSize(MatMulInfoTy MMI) { in getMatMulTypeSize() argument
878 auto *S = MMI.A->getStatement()->getParent(); in getMatMulTypeSize()
880 auto ElementSizeA = DL.getTypeSizeInBits(MMI.A->getElementType()); in getMatMulTypeSize()
881 auto ElementSizeB = DL.getTypeSizeInBits(MMI.B->getElementType()); in getMatMulTypeSize()
882 auto ElementSizeC = DL.getTypeSizeInBits(MMI.WriteToC->getElementType()); in getMatMulTypeSize()
899 getMicroKernelParams(const TargetTransformInfo *TTI, MatMulInfoTy MMI) { in getMicroKernelParams() argument
908 auto ElementSize = getMatMulTypeSize(MMI); in getMicroKernelParams()
977 MatMulInfoTy MMI) { in getMacroKernelParams() argument
1002 auto ElementSize = getMatMulAlignTypeSize(MMI); in getMacroKernelParams()
1097 MatMulInfoTy &MMI) { in optimizeDataLayoutMatrMulPattern() argument
1110 MMI.B->getElementType(), "Packed_B", in optimizeDataLayoutMatrMulPattern()
1113 auto OldAcc = MMI.B->getLatestAccessRelation(); in optimizeDataLayoutMatrMulPattern()
1114 MMI.B->setNewAccessRelation(AccRel); in optimizeDataLayoutMatrMulPattern()
1118 ExtMap = ExtMap.fix_si(isl::dim::out, MMI.i, 0); in optimizeDataLayoutMatrMulPattern()
1125 OldAcc, MMI.B->getLatestAccessRelation(), Domain); in optimizeDataLayoutMatrMulPattern()
1138 MMI.A->getElementType(), "Packed_A", in optimizeDataLayoutMatrMulPattern()
1141 OldAcc = MMI.A->getLatestAccessRelation(); in optimizeDataLayoutMatrMulPattern()
1142 MMI.A->setNewAccessRelation(AccRel); in optimizeDataLayoutMatrMulPattern()
1146 ExtMap = ExtMap.fix_si(isl::dim::out, MMI.j, 0); in optimizeDataLayoutMatrMulPattern()
1148 OldAcc, MMI.A->getLatestAccessRelation(), Domain); in optimizeDataLayoutMatrMulPattern()
1278 MatMulInfoTy &MMI) { in optimizeMatMulPattern() argument
1281 Node, MMI.WriteToC->getLatestScopArrayInfo()->getBasePtr()); in optimizeMatMulPattern()
1287 Node = permuteBandNodeDimensions(Node, MMI.i, DimOutNum - 3); in optimizeMatMulPattern()
1288 int NewJ = MMI.j == DimOutNum - 3 ? MMI.i : MMI.j; in optimizeMatMulPattern()
1289 int NewK = MMI.k == DimOutNum - 3 ? MMI.i : MMI.k; in optimizeMatMulPattern()
1293 auto MicroKernelParams = getMicroKernelParams(TTI, MMI); in optimizeMatMulPattern()
1294 auto MacroKernelParams = getMacroKernelParams(TTI, MicroKernelParams, MMI); in optimizeMatMulPattern()
1307 MacroKernelParams, MMI); in optimizeMatMulPattern()
1312 MatMulInfoTy &MMI) { in isMatrMultPattern() argument
1324 if (containsMatrMult(NewPartialSchedule, D, MMI)) in isMatrMultPattern()
1338 MatMulInfoTy MMI; in optimizeBand() local
1340 isMatrMultPattern(isl::manage_copy(Node), OAI->D, MMI)) { in optimizeBand()
1343 return optimizeMatMulPattern(isl::manage(Node), OAI->TTI, MMI).release(); in optimizeBand()