• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
17 #define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
18 #include "reg_alloc.h"
19 #include "aarch64_operand.h"
20 #include "aarch64_insn.h"
21 #include "aarch64_abi.h"
22 #include "aarch64_cgfunc.h"
23 #include "loop.h"
24 #include "cg_dominance.h"
25 #include "cg_pre.h"
26 
27 namespace maplebe {
28 #define RESERVED_REGS
29 
30 #define USE_LRA
31 #define USE_SPLIT
32 #undef USE_BB_FREQUENCY
33 #define OPTIMIZE_FOR_PROLOG
34 #define REUSE_SPILLMEM
35 #undef COLOR_SPLIT
36 #define MOVE_COALESCE
37 
38 /* for robust test */
39 #undef CONSISTENT_MEMOPND
40 #undef RANDOM_PRIORITY
41 
42 constexpr uint32 k32 = sizeof(int) * CHAR_BIT;
43 constexpr uint32 k64 = sizeof(int64) * CHAR_BIT;
44 constexpr uint32 kU64 = sizeof(uint64) * CHAR_BIT;
45 
46 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::set<T,Comparator> & set,const T & item)47 inline bool FindNotIn(const std::set<T, Comparator> &set, const T &item)
48 {
49     return set.find(item) == set.end();
50 }
51 
52 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::unordered_set<T,Comparator> & set,const T & item)53 inline bool FindNotIn(const std::unordered_set<T, Comparator> &set, const T &item)
54 {
55     return set.find(item) == set.end();
56 }
57 
58 template <typename T>
FindNotIn(const MapleSet<T> & set,const T & item)59 inline bool FindNotIn(const MapleSet<T> &set, const T &item)
60 {
61     return set.find(item) == set.end();
62 }
63 
64 template <typename T>
FindNotIn(const MapleUnorderedSet<T> & set,const T & item)65 inline bool FindNotIn(const MapleUnorderedSet<T> &set, const T &item)
66 {
67     return set.find(item) == set.end();
68 }
69 
70 template <typename T>
FindNotIn(const MapleList<T> & list,const T & item)71 inline bool FindNotIn(const MapleList<T> &list, const T &item)
72 {
73     return std::find(list.begin(), list.end(), item) == list.end();
74 }
75 
76 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::set<T,Comparator> & set,const T & item)77 inline bool FindIn(const std::set<T, Comparator> &set, const T &item)
78 {
79     return set.find(item) != set.end();
80 }
81 
82 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::unordered_set<T,Comparator> & set,const T & item)83 inline bool FindIn(const std::unordered_set<T, Comparator> &set, const T &item)
84 {
85     return set.find(item) != set.end();
86 }
87 
88 template <typename T>
FindIn(const MapleSet<T> & set,const T & item)89 inline bool FindIn(const MapleSet<T> &set, const T &item)
90 {
91     return set.find(item) != set.end();
92 }
93 
94 template <typename T>
FindIn(const MapleUnorderedSet<T> & set,const T & item)95 inline bool FindIn(const MapleUnorderedSet<T> &set, const T &item)
96 {
97     return set.find(item) != set.end();
98 }
99 
100 template <typename T>
FindIn(const MapleList<T> & list,const T & item)101 inline bool FindIn(const MapleList<T> &list, const T &item)
102 {
103     return std::find(list.begin(), list.end(), item) != list.end();
104 }
105 
IsBitArrElemSet(const uint64 * vec,const uint32 num)106 inline bool IsBitArrElemSet(const uint64 *vec, const uint32 num)
107 {
108     size_t index = num / kU64;
109     uint64 bit = num % kU64;
110     return vec[index] & (1ULL << bit);
111 }
112 
IsBBsetOverlap(const uint64 * vec1,const uint64 * vec2,uint32 bbBuckets)113 inline bool IsBBsetOverlap(const uint64 *vec1, const uint64 *vec2, uint32 bbBuckets)
114 {
115     for (uint32 i = 0; i < bbBuckets; ++i) {
116         if ((vec1[i] & vec2[i]) != 0) {
117             return true;
118         }
119     }
120     return false;
121 }
122 
123 /* For each bb, record info pertain to allocation */
124 /*
125  * This is per bb per LR.
126  * LU info is particular to a bb in a LR.
127  */
128 class LiveUnit {
129 public:
130     LiveUnit() = default;
131     ~LiveUnit() = default;
132 
133     void PrintLiveUnit() const;
134 
GetBegin()135     uint32 GetBegin() const
136     {
137         return begin;
138     }
139 
SetBegin(uint32 val)140     void SetBegin(uint32 val)
141     {
142         begin = val;
143     }
144 
GetEnd()145     uint32 GetEnd() const
146     {
147         return end;
148     }
149 
SetEnd(uint32 end)150     void SetEnd(uint32 end)
151     {
152         this->end = end;
153     }
154 
HasCall()155     bool HasCall() const
156     {
157         return hasCall;
158     }
159 
SetHasCall(bool hasCall)160     void SetHasCall(bool hasCall)
161     {
162         this->hasCall = hasCall;
163     }
164 
GetDefNum()165     uint32 GetDefNum() const
166     {
167         return defNum;
168     }
169 
SetDefNum(uint32 defNum)170     void SetDefNum(uint32 defNum)
171     {
172         this->defNum = defNum;
173     }
174 
IncDefNum()175     void IncDefNum()
176     {
177         ++defNum;
178     }
179 
GetUseNum()180     uint32 GetUseNum() const
181     {
182         return useNum;
183     }
184 
SetUseNum(uint32 useNum)185     void SetUseNum(uint32 useNum)
186     {
187         this->useNum = useNum;
188     }
189 
IncUseNum()190     void IncUseNum()
191     {
192         ++useNum;
193     }
194 
NeedReload()195     bool NeedReload() const
196     {
197         return needReload;
198     }
199 
SetNeedReload(bool needReload)200     void SetNeedReload(bool needReload)
201     {
202         this->needReload = needReload;
203     }
204 
NeedRestore()205     bool NeedRestore() const
206     {
207         return needRestore;
208     }
209 
SetNeedRestore(bool needRestore)210     void SetNeedRestore(bool needRestore)
211     {
212         this->needRestore = needRestore;
213     }
214 
215 private:
216     uint32 begin = 0;     /* first encounter in bb */
217     uint32 end = 0;       /* last encounter in bb */
218     bool hasCall = false; /* bb has a call */
219     uint32 defNum = 0;
220     uint32 useNum = 0; /* used for priority calculation */
221     bool needReload = false;
222     bool needRestore = false;
223 };
224 
225 struct SortedBBCmpFunc {
operatorSortedBBCmpFunc226     bool operator()(const BB *lhs, const BB *rhs) const
227     {
228         return (lhs->GetLevel() < rhs->GetLevel());
229     }
230 };
231 
232 enum refType : uint8 {
233     kIsUse = 0x1,
234     kIsDef = 0x2,
235     kIsCall = 0x4,
236 };
237 
238 /* LR is for each global vreg. */
239 class LiveRange {
240 public:
LiveRange(MapleAllocator & allocator)241     explicit LiveRange(MapleAllocator &allocator)
242         : lrAlloca(&allocator),
243           pregveto(allocator.Adapter()),
244           callDef(allocator.Adapter()),
245           forbidden(allocator.Adapter()),
246           prefs(allocator.Adapter()),
247           refMap(allocator.Adapter()),
248           luMap(allocator.Adapter())
249     {
250     }
251 
252     ~LiveRange() = default;
253 
GetRegNO()254     regno_t GetRegNO() const
255     {
256         return regNO;
257     }
258 
SetRegNO(regno_t val)259     void SetRegNO(regno_t val)
260     {
261         regNO = val;
262     }
263 
GetID()264     uint32 GetID() const
265     {
266         return id;
267     }
268 
SetID(uint32 id)269     void SetID(uint32 id)
270     {
271         this->id = id;
272     }
273 
GetAssignedRegNO()274     regno_t GetAssignedRegNO() const
275     {
276         return assignedRegNO;
277     }
278 
SetAssignedRegNO(regno_t val)279     void SetAssignedRegNO(regno_t val)
280     {
281         assignedRegNO = val;
282     }
283 
GetNumCall()284     uint32 GetNumCall() const
285     {
286         return numCall;
287     }
288 
SetNumCall(uint32 num)289     void SetNumCall(uint32 num)
290     {
291         numCall = num;
292     }
293 
IncNumCall()294     void IncNumCall()
295     {
296         ++numCall;
297     }
298 
GetRegType()299     RegType GetRegType() const
300     {
301         return regType;
302     }
303 
SetRegType(RegType regType)304     void SetRegType(RegType regType)
305     {
306         this->regType = regType;
307     }
308 
GetPriority()309     float GetPriority() const
310     {
311         return priority;
312     }
313 
SetPriority(float priority)314     void SetPriority(float priority)
315     {
316         this->priority = priority;
317     }
318 
IsMustAssigned()319     bool IsMustAssigned() const
320     {
321         return mustAssigned;
322     }
323 
SetMustAssigned()324     void SetMustAssigned()
325     {
326         mustAssigned = true;
327     }
328 
SetBBBuckets(uint32 bucketNum)329     void SetBBBuckets(uint32 bucketNum)
330     {
331         bbBuckets = bucketNum;
332     }
333 
SetRegBuckets(uint32 bucketNum)334     void SetRegBuckets(uint32 bucketNum)
335     {
336         regBuckets = bucketNum;
337     }
338 
GetNumBBMembers()339     uint32 GetNumBBMembers() const
340     {
341         return numBBMembers;
342     }
343 
IncNumBBMembers()344     void IncNumBBMembers()
345     {
346         ++numBBMembers;
347     }
348 
DecNumBBMembers()349     void DecNumBBMembers()
350     {
351         --numBBMembers;
352     }
353 
InitBBMember(MemPool & memPool,size_t size)354     void InitBBMember(MemPool &memPool, size_t size)
355     {
356         bbMember = memPool.NewArray<uint64>(size);
357         errno_t ret = memset_s(bbMember, size * sizeof(uint64), 0, size * sizeof(uint64));
358         CHECK_FATAL(ret == EOK, "call memset_s failed");
359     }
360 
GetBBMember()361     uint64 *GetBBMember()
362     {
363         return bbMember;
364     }
365 
GetBBMember()366     const uint64 *GetBBMember() const
367     {
368         return bbMember;
369     }
370 
GetBBMemberElem(int32 index)371     uint64 GetBBMemberElem(int32 index) const
372     {
373         return bbMember[index];
374     }
375 
SetBBMemberElem(int32 index,uint64 elem)376     void SetBBMemberElem(int32 index, uint64 elem)
377     {
378         bbMember[index] = elem;
379     }
380 
SetMemberBitArrElem(uint32 bbID)381     void SetMemberBitArrElem(uint32 bbID)
382     {
383         uint32 index = bbID / kU64;
384         uint64 bit = bbID % kU64;
385         uint64 mask = 1ULL << bit;
386         if ((GetBBMemberElem(index) & mask) == 0) {
387             IncNumBBMembers();
388             SetBBMemberElem(index, GetBBMemberElem(index) | mask);
389         }
390     }
391 
UnsetMemberBitArrElem(uint32 bbID)392     void UnsetMemberBitArrElem(uint32 bbID)
393     {
394         uint32 index = bbID / kU64;
395         uint64 bit = bbID % kU64;
396         uint64 mask = 1ULL << bit;
397         if ((GetBBMemberElem(index) & mask) != 0) {
398             DecNumBBMembers();
399             SetBBMemberElem(index, GetBBMemberElem(index) & (~mask));
400         }
401     }
402 
SetConflictBitArrElem(regno_t regNO)403     void SetConflictBitArrElem(regno_t regNO)
404     {
405         uint32 index = regNO / kU64;
406         uint64 bit = regNO % kU64;
407         uint64 mask = 1ULL << bit;
408         if ((GetBBConflictElem(index) & mask) == 0) {
409             IncNumBBConflicts();
410             SetBBConflictElem(index, GetBBConflictElem(index) | mask);
411         }
412     }
413 
UnsetConflictBitArrElem(regno_t regNO)414     void UnsetConflictBitArrElem(regno_t regNO)
415     {
416         uint32 index = regNO / kU64;
417         uint64 bit = regNO % kU64;
418         uint64 mask = 1ULL << bit;
419         if ((GetBBConflictElem(index) & mask) != 0) {
420             DecNumBBConflicts();
421             SetBBConflictElem(index, GetBBConflictElem(index) & (~mask));
422         }
423     }
424 
InitPregveto()425     void InitPregveto()
426     {
427         pregveto.clear();
428         pregveto.resize(kMaxRegNum);
429         callDef.clear();
430         callDef.resize(kMaxRegNum);
431     }
432 
GetPregveto(regno_t regno)433     bool GetPregveto(regno_t regno) const
434     {
435         return pregveto[regno];
436     }
437 
GetPregvetoSize()438     size_t GetPregvetoSize() const
439     {
440         return numPregveto;
441     }
442 
InsertElemToPregveto(regno_t regno)443     void InsertElemToPregveto(regno_t regno)
444     {
445         if (!pregveto[regno]) {
446             pregveto[regno] = true;
447             ++numPregveto;
448         }
449     }
450 
GetCallDef(regno_t regno)451     bool GetCallDef(regno_t regno) const
452     {
453         return callDef[regno];
454     }
455 
InsertElemToCallDef(regno_t regno)456     void InsertElemToCallDef(regno_t regno)
457     {
458         if (!callDef[regno]) {
459             callDef[regno] = true;
460             ++numCallDef;
461         }
462     }
463 
SetCrossCall()464     void SetCrossCall()
465     {
466         crossCall = true;
467     }
468 
GetCrossCall()469     bool GetCrossCall() const
470     {
471         return crossCall;
472     }
473 
InitForbidden()474     void InitForbidden()
475     {
476         forbidden.clear();
477         forbidden.resize(kMaxRegNum);
478     }
479 
GetForbidden()480     const MapleVector<bool> &GetForbidden() const
481     {
482         return forbidden;
483     }
484 
GetForbidden(regno_t regno)485     bool GetForbidden(regno_t regno) const
486     {
487         return forbidden[regno];
488     }
489 
GetForbiddenSize()490     size_t GetForbiddenSize() const
491     {
492         return numForbidden;
493     }
494 
InsertElemToForbidden(regno_t regno)495     void InsertElemToForbidden(regno_t regno)
496     {
497         if (!forbidden[regno]) {
498             forbidden[regno] = true;
499             ++numForbidden;
500         }
501     }
502 
EraseElemFromForbidden(regno_t regno)503     void EraseElemFromForbidden(regno_t regno)
504     {
505         if (forbidden[regno]) {
506             forbidden[regno] = false;
507             --numForbidden;
508         }
509     }
510 
ClearForbidden()511     void ClearForbidden()
512     {
513         forbidden.clear();
514     }
515 
GetNumBBConflicts()516     uint32 GetNumBBConflicts() const
517     {
518         return numBBConflicts;
519     }
520 
IncNumBBConflicts()521     void IncNumBBConflicts()
522     {
523         ++numBBConflicts;
524     }
525 
DecNumBBConflicts()526     void DecNumBBConflicts()
527     {
528         --numBBConflicts;
529     }
530 
InitBBConflict(MemPool & memPool,size_t size)531     void InitBBConflict(MemPool &memPool, size_t size)
532     {
533         bbConflict = memPool.NewArray<uint64>(size);
534         errno_t ret = memset_s(bbConflict, size * sizeof(uint64), 0, size * sizeof(uint64));
535         CHECK_FATAL(ret == EOK, "call memset_s failed");
536     }
537 
GetBBConflict()538     const uint64 *GetBBConflict() const
539     {
540         return bbConflict;
541     }
542 
GetBBConflictElem(int32 index)543     uint64 GetBBConflictElem(int32 index) const
544     {
545         DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
546         return bbConflict[index];
547     }
548 
SetBBConflictElem(int32 index,uint64 elem)549     void SetBBConflictElem(int32 index, uint64 elem)
550     {
551         DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
552         bbConflict[index] = elem;
553     }
554 
SetOldConflict(uint64 * conflict)555     void SetOldConflict(uint64 *conflict)
556     {
557         oldConflict = conflict;
558     }
559 
GetOldConflict()560     const uint64 *GetOldConflict() const
561     {
562         return oldConflict;
563     }
564 
GetPrefs()565     const MapleSet<regno_t> &GetPrefs() const
566     {
567         return prefs;
568     }
569 
InsertElemToPrefs(regno_t regNO)570     void InsertElemToPrefs(regno_t regNO)
571     {
572         (void)prefs.insert(regNO);
573     }
574 
GetRefs()575     const MapleMap<uint32, MapleMap<uint32, uint32> *> GetRefs() const
576     {
577         return refMap;
578     }
579 
GetRefs(uint32 bbId)580     const MapleMap<uint32, uint32> GetRefs(uint32 bbId) const
581     {
582         return *(refMap.find(bbId)->second);
583     }
584 
AddRef(uint32 bbId,uint32 pos,uint32 mark)585     void AddRef(uint32 bbId, uint32 pos, uint32 mark)
586     {
587         if (refMap.find(bbId) == refMap.end()) {
588             auto point = lrAlloca->New<MapleMap<uint32, uint32>>(lrAlloca->Adapter());
589             (void)point->emplace(std::pair<uint32, uint32>(pos, mark));
590             (void)refMap.emplace(std::pair<uint32, MapleMap<uint32, uint32> *>(bbId, point));
591         } else {
592             auto &bbPoint = (refMap.find(bbId))->second;
593             if (bbPoint->find(pos) == bbPoint->end()) {
594                 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, mark));
595             } else {
596                 auto posVal = bbPoint->find(pos)->second;
597                 (void)bbPoint->erase(bbPoint->find(pos));
598                 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, posVal | mark));
599             }
600         }
601     }
602 
GetLuMap()603     const MapleMap<uint32, LiveUnit *> &GetLuMap() const
604     {
605         return luMap;
606     }
607 
FindInLuMap(uint32 index)608     MapleMap<uint32, LiveUnit *>::iterator FindInLuMap(uint32 index)
609     {
610         return luMap.find(index);
611     }
612 
EndOfLuMap()613     MapleMap<uint32, LiveUnit *>::iterator EndOfLuMap()
614     {
615         return luMap.end();
616     }
617 
EraseLuMap(MapleMap<uint32,LiveUnit * >::iterator it)618     MapleMap<uint32, LiveUnit *>::iterator EraseLuMap(MapleMap<uint32, LiveUnit *>::iterator it)
619     {
620         return luMap.erase(it);
621     }
622 
SetElemToLuMap(uint32 key,LiveUnit & value)623     void SetElemToLuMap(uint32 key, LiveUnit &value)
624     {
625         luMap[key] = &value;
626     }
627 
GetLiveUnitFromLuMap(uint32 key)628     LiveUnit *GetLiveUnitFromLuMap(uint32 key)
629     {
630         return luMap[key];
631     }
632 
GetLiveUnitFromLuMap(uint32 key)633     const LiveUnit *GetLiveUnitFromLuMap(uint32 key) const
634     {
635         auto it = luMap.find(key);
636         DEBUG_ASSERT(it != luMap.end(), "can't find live unit");
637         return it->second;
638     }
639 
GetSplitLr()640     const LiveRange *GetSplitLr() const
641     {
642         return splitLr;
643     }
644 
SetSplitLr(LiveRange & lr)645     void SetSplitLr(LiveRange &lr)
646     {
647         splitLr = &lr;
648     }
649 
650 #ifdef OPTIMIZE_FOR_PROLOG
GetNumDefs()651     uint32 GetNumDefs() const
652     {
653         return numDefs;
654     }
655 
IncNumDefs()656     void IncNumDefs()
657     {
658         ++numDefs;
659     }
660 
SetNumDefs(uint32 val)661     void SetNumDefs(uint32 val)
662     {
663         numDefs = val;
664     }
665 
GetNumUses()666     uint32 GetNumUses() const
667     {
668         return numUses;
669     }
670 
IncNumUses()671     void IncNumUses()
672     {
673         ++numUses;
674     }
675 
SetNumUses(uint32 val)676     void SetNumUses(uint32 val)
677     {
678         numUses = val;
679     }
680 
GetFrequency()681     uint32 GetFrequency() const
682     {
683         return frequency;
684     }
685 
SetFrequency(uint32 frequency)686     void SetFrequency(uint32 frequency)
687     {
688         this->frequency = frequency;
689     }
690 #endif /* OPTIMIZE_FOR_PROLOG */
691 
GetSpillMem()692     MemOperand *GetSpillMem()
693     {
694         return spillMem;
695     }
696 
GetSpillMem()697     const MemOperand *GetSpillMem() const
698     {
699         return spillMem;
700     }
701 
SetSpillMem(MemOperand & memOpnd)702     void SetSpillMem(MemOperand &memOpnd)
703     {
704         spillMem = &memOpnd;
705     }
706 
GetSpillReg()707     regno_t GetSpillReg() const
708     {
709         return spillReg;
710     }
711 
SetSpillReg(regno_t spillReg)712     void SetSpillReg(regno_t spillReg)
713     {
714         this->spillReg = spillReg;
715     }
716 
GetSpillSize()717     uint32 GetSpillSize() const
718     {
719         return spillSize;
720     }
721 
SetSpillSize(uint32 size)722     void SetSpillSize(uint32 size)
723     {
724         spillSize = size;
725     }
726 
IsSpilled()727     bool IsSpilled() const
728     {
729         return spilled;
730     }
731 
SetSpilled(bool spill)732     void SetSpilled(bool spill)
733     {
734         spilled = spill;
735     }
736 
HasDefUse()737     bool HasDefUse() const
738     {
739         return hasDefUse;
740     }
741 
SetDefUse()742     void SetDefUse()
743     {
744         hasDefUse = true;
745     }
746 
GetProcessed()747     bool GetProcessed() const
748     {
749         return proccessed;
750     }
751 
SetProcessed()752     void SetProcessed()
753     {
754         proccessed = true;
755     }
756 
IsNonLocal()757     bool IsNonLocal() const
758     {
759         return isNonLocal;
760     }
761 
SetIsNonLocal(bool isNonLocal)762     void SetIsNonLocal(bool isNonLocal)
763     {
764         this->isNonLocal = isNonLocal;
765     }
766 
SetRematLevel(uint32 val)767     void SetRematLevel(uint32 val)
768     {
769         this->rematLevel = val;
770     }
771 
GetRematLevel()772     uint32 GetRematLevel() const
773     {
774         return this->rematLevel;
775     }
776 
GetOp()777     Opcode GetOp() const
778     {
779         return op;
780     }
781 
GetRematSymbol()782     const MIRSymbol *GetRematSymbol() const
783     {
784         DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat symbol is invalid");
785         return rematInfo.sym;
786     }
787 
GetRematFieldID()788     FieldID GetRematFieldID() const
789     {
790         DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat field ID is invalid");
791         return fieldID;
792     }
793 
SetRematerializable(const MIRConst * c)794     void SetRematerializable(const MIRConst *c)
795     {
796         op = OP_constval;
797         rematInfo.mirConst = c;
798     }
799 
SetRematerializable(Opcode opcode,const MIRSymbol * symbol,FieldID fieldId,bool addrUp)800     void SetRematerializable(Opcode opcode, const MIRSymbol *symbol, FieldID fieldId, bool addrUp)
801     {
802         this->op = opcode;
803         rematInfo.sym = symbol;
804         this->fieldID = fieldId;
805         this->addrUpper = addrUp;
806     }
807 
CopyRematerialization(const LiveRange & lr)808     void CopyRematerialization(const LiveRange &lr)
809     {
810         op = lr.op;
811         rematInfo = lr.rematInfo;
812         fieldID = lr.fieldID;
813     }
814 
815 private:
816     MapleAllocator *lrAlloca;
817     regno_t regNO = 0;
818     uint32 id = 0;             /* for priority tie breaker */
819     regno_t assignedRegNO = 0; /* color assigned */
820     uint32 numCall = 0;
821     RegType regType = kRegTyUndef;
822     float priority = 0.0;
823     bool mustAssigned = false;
824     uint32 bbBuckets = 0;       /* size of bit array for bb (each bucket == 64 bits) */
825     uint32 regBuckets = 0;      /* size of bit array for reg (each bucket == 64 bits) */
826     uint32 numBBMembers = 0;    /* number of bits set in bbMember */
827     uint64 *bbMember = nullptr; /* Same as smember, but use bit array */
828 
829     MapleVector<bool> pregveto;  /* pregs cannot be assigned   -- SplitLr may clear forbidden */
830     MapleVector<bool> callDef;   /* pregs cannot be assigned   -- SplitLr may clear forbidden */
831     MapleVector<bool> forbidden; /* pregs cannot be assigned */
832     uint32 numPregveto = 0;
833     uint32 numCallDef = 0;
834     uint32 numForbidden = 0;
835     bool crossCall = false;
836 
837     uint32 numBBConflicts = 0;    /* number of bits set in bbConflict */
838     uint64 *bbConflict = nullptr; /* vreg interference from graph neighbors (bit) */
839     uint64 *oldConflict = nullptr;
840     MapleSet<regno_t> prefs; /* pregs that prefer */
841     MapleMap<uint32, MapleMap<uint32, uint32> *> refMap;
842     MapleMap<uint32, LiveUnit *> luMap; /* info for each bb */
843     LiveRange *splitLr = nullptr;       /* The 1st part of the split */
844 #ifdef OPTIMIZE_FOR_PROLOG
845     uint32 numDefs = 0;
846     uint32 numUses = 0;
847     uint32 frequency = 0;
848 #endif                              /* OPTIMIZE_FOR_PROLOG */
849     MemOperand *spillMem = nullptr; /* memory operand used for spill, if any */
850     regno_t spillReg = 0;           /* register operand for spill at current point */
851     uint32 spillSize = 0;           /* 32 or 64 bit spill */
852     bool spilled = false;           /* color assigned */
853     bool hasDefUse = false;         /* has regDS */
854     bool proccessed = false;
855     bool isNonLocal = false;
856     uint32 rematLevel = 0;
857     Opcode op = OP_undef; /* OP_constval, OP_addrof or OP_dread if rematerializable */
858     union RematInfo {
859         const MIRConst *mirConst;
860         const MIRSymbol *sym;
861     } rematInfo;            /* info for rematerializing value */
862     FieldID fieldID = 0;    /* used only when op is OP_addrof or OP_dread */
863     bool addrUpper = false; /* indicates the upper bits of an addrof */
864 };
865 
866 /* One per bb, to communicate local usage to global RA */
867 class LocalRaInfo {
868 public:
LocalRaInfo(MapleAllocator & allocator)869     explicit LocalRaInfo(MapleAllocator &allocator) : defCnt(allocator.Adapter()), useCnt(allocator.Adapter()) {}
870 
871     ~LocalRaInfo() = default;
872 
GetDefCnt()873     const MapleMap<regno_t, uint16> &GetDefCnt() const
874     {
875         return defCnt;
876     }
877 
GetDefCntElem(regno_t regNO)878     uint16 GetDefCntElem(regno_t regNO)
879     {
880         return defCnt[regNO];
881     }
882 
SetDefCntElem(regno_t key,uint16 value)883     void SetDefCntElem(regno_t key, uint16 value)
884     {
885         defCnt[key] = value;
886     }
887 
GetUseCnt()888     const MapleMap<regno_t, uint16> &GetUseCnt() const
889     {
890         return useCnt;
891     }
892 
GetUseCntElem(regno_t regNO)893     uint16 GetUseCntElem(regno_t regNO)
894     {
895         return useCnt[regNO];
896     }
897 
SetUseCntElem(regno_t key,uint16 value)898     void SetUseCntElem(regno_t key, uint16 value)
899     {
900         useCnt[key] = value;
901     }
902 
903 private:
904     MapleMap<regno_t, uint16> defCnt;
905     MapleMap<regno_t, uint16> useCnt;
906 };
907 
908 /* For each bb, record info pertain to allocation */
909 class BBAssignInfo {
910 public:
BBAssignInfo(MapleAllocator & allocator)911     explicit BBAssignInfo(MapleAllocator &allocator) : globalsAssigned(allocator.Adapter()), regMap(allocator.Adapter())
912     {
913     }
914 
915     ~BBAssignInfo() = default;
916 
GetIntLocalRegsNeeded()917     uint32 GetIntLocalRegsNeeded() const
918     {
919         return intLocalRegsNeeded;
920     }
921 
SetIntLocalRegsNeeded(uint32 num)922     void SetIntLocalRegsNeeded(uint32 num)
923     {
924         intLocalRegsNeeded = num;
925     }
926 
GetFpLocalRegsNeeded()927     uint32 GetFpLocalRegsNeeded() const
928     {
929         return fpLocalRegsNeeded;
930     }
931 
SetFpLocalRegsNeeded(uint32 num)932     void SetFpLocalRegsNeeded(uint32 num)
933     {
934         fpLocalRegsNeeded = num;
935     }
936 
InitGlobalAssigned()937     void InitGlobalAssigned()
938     {
939         globalsAssigned.clear();
940         globalsAssigned.resize(kMaxRegNum);
941     }
942 
GetGlobalsAssigned(regno_t regNO)943     bool GetGlobalsAssigned(regno_t regNO) const
944     {
945         return globalsAssigned[regNO];
946     }
947 
InsertElemToGlobalsAssigned(regno_t regNO)948     void InsertElemToGlobalsAssigned(regno_t regNO)
949     {
950         globalsAssigned[regNO] = true;
951     }
952 
EraseElemToGlobalsAssigned(regno_t regNO)953     void EraseElemToGlobalsAssigned(regno_t regNO)
954     {
955         globalsAssigned[regNO] = false;
956     }
957 
GetRegMap()958     const MapleMap<regno_t, regno_t> &GetRegMap() const
959     {
960         return regMap;
961     }
962 
HasRegMap(regno_t regNOKey)963     bool HasRegMap(regno_t regNOKey) const
964     {
965         return (regMap.find(regNOKey) != regMap.end());
966     }
967 
GetRegMapElem(regno_t regNO)968     regno_t GetRegMapElem(regno_t regNO)
969     {
970         return regMap[regNO];
971     }
972 
SetRegMapElem(regno_t regNOKey,regno_t regNOValue)973     void SetRegMapElem(regno_t regNOKey, regno_t regNOValue)
974     {
975         regMap[regNOKey] = regNOValue;
976     }
977 
978 private:
979     uint32 intLocalRegsNeeded = 0;     /* num local reg needs for each bb */
980     uint32 fpLocalRegsNeeded = 0;      /* num local reg needs for each bb */
981     MapleVector<bool> globalsAssigned; /* globals used in a bb */
982     MapleMap<regno_t, regno_t> regMap; /* local vreg to preg mapping */
983 };
984 
985 class FinalizeRegisterInfo {
986 public:
FinalizeRegisterInfo(MapleAllocator & allocator)987     explicit FinalizeRegisterInfo(MapleAllocator &allocator)
988         : defOperands(allocator.Adapter()),
989           defIdx(allocator.Adapter()),
990           useOperands(allocator.Adapter()),
991           useIdx(allocator.Adapter())
992     {
993     }
994 
995     ~FinalizeRegisterInfo() = default;
ClearInfo()996     void ClearInfo()
997     {
998         memOperandIdx = 0;
999         baseOperand = nullptr;
1000         offsetOperand = nullptr;
1001         defOperands.clear();
1002         defIdx.clear();
1003         useOperands.clear();
1004         useIdx.clear();
1005     }
1006 
SetBaseOperand(Operand & opnd,const int32 idx)1007     void SetBaseOperand(Operand &opnd, const int32 idx)
1008     {
1009         baseOperand = &opnd;
1010         memOperandIdx = idx;
1011     }
1012 
SetOffsetOperand(Operand & opnd)1013     void SetOffsetOperand(Operand &opnd)
1014     {
1015         offsetOperand = &opnd;
1016     }
1017 
SetDefOperand(Operand & opnd,const int32 idx)1018     void SetDefOperand(Operand &opnd, const int32 idx)
1019     {
1020         defOperands.emplace_back(&opnd);
1021         defIdx.emplace_back(idx);
1022     }
1023 
SetUseOperand(Operand & opnd,const int32 idx)1024     void SetUseOperand(Operand &opnd, const int32 idx)
1025     {
1026         useOperands.emplace_back(&opnd);
1027         useIdx.emplace_back(idx);
1028     }
1029 
GetMemOperandIdx()1030     int32 GetMemOperandIdx() const
1031     {
1032         return memOperandIdx;
1033     }
1034 
GetBaseOperand()1035     const Operand *GetBaseOperand() const
1036     {
1037         return baseOperand;
1038     }
1039 
GetOffsetOperand()1040     const Operand *GetOffsetOperand() const
1041     {
1042         return offsetOperand;
1043     }
1044 
GetDefOperandsSize()1045     size_t GetDefOperandsSize() const
1046     {
1047         return defOperands.size();
1048     }
1049 
GetDefOperandsElem(size_t index)1050     const Operand *GetDefOperandsElem(size_t index) const
1051     {
1052         return defOperands[index];
1053     }
1054 
GetDefIdxElem(size_t index)1055     int32 GetDefIdxElem(size_t index) const
1056     {
1057         return defIdx[index];
1058     }
1059 
GetUseOperandsSize()1060     size_t GetUseOperandsSize() const
1061     {
1062         return useOperands.size();
1063     }
1064 
GetUseOperandsElem(size_t index)1065     const Operand *GetUseOperandsElem(size_t index) const
1066     {
1067         return useOperands[index];
1068     }
1069 
GetUseIdxElem(size_t index)1070     int32 GetUseIdxElem(size_t index) const
1071     {
1072         return useIdx[index];
1073     }
1074 
1075 private:
1076     int32 memOperandIdx = 0;
1077     Operand *baseOperand = nullptr;
1078     Operand *offsetOperand = nullptr;
1079     MapleVector<Operand *> defOperands;
1080     MapleVector<int32> defIdx;
1081     MapleVector<Operand *> useOperands;
1082     MapleVector<int32> useIdx;
1083 };
1084 
1085 class LocalRegAllocator {
1086 public:
LocalRegAllocator(CGFunc & cgFunc,MapleAllocator & allocator)1087     LocalRegAllocator(CGFunc &cgFunc, MapleAllocator &allocator)
1088         : intRegAssignmentMap(allocator.Adapter()),
1089           fpRegAssignmentMap(allocator.Adapter()),
1090           useInfo(allocator.Adapter()),
1091           defInfo(allocator.Adapter())
1092     {
1093         buckets = (cgFunc.GetMaxRegNum() / kU64) + 1;
1094         intRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1095         fpRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1096         intRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1097         fpRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1098     }
1099 
1100     ~LocalRegAllocator() = default;
1101 
ClearLocalRaInfo()1102     void ClearLocalRaInfo()
1103     {
1104         ClearBitArrElement(intRegAssigned);
1105         ClearBitArrElement(fpRegAssigned);
1106         intRegAssignmentMap.clear();
1107         fpRegAssignmentMap.clear();
1108         intPregUsed = 0;
1109         fpPregUsed = 0;
1110         ClearBitArrElement(intRegSpilled);
1111         ClearBitArrElement(fpRegSpilled);
1112         numIntPregUsed = 0;
1113         numFpPregUsed = 0;
1114     }
1115 
RegBaseUpdate(bool isInt)1116     regno_t RegBaseUpdate(bool isInt) const
1117     {
1118         return isInt ? 0 : V0 - R0;
1119     }
1120 
IsInRegAssigned(regno_t regNO,bool isInt)1121     bool IsInRegAssigned(regno_t regNO, bool isInt) const
1122     {
1123         uint64 *regAssigned = nullptr;
1124         if (isInt) {
1125             regAssigned = intRegAssigned;
1126         } else {
1127             regAssigned = fpRegAssigned;
1128         }
1129         return IsBitArrElemSet(regAssigned, regNO);
1130         ;
1131     }
1132 
SetRegAssigned(regno_t regNO,bool isInt)1133     void SetRegAssigned(regno_t regNO, bool isInt) const
1134     {
1135         if (isInt) {
1136             SetBitArrElement(intRegAssigned, regNO);
1137         } else {
1138             SetBitArrElement(fpRegAssigned, regNO);
1139         }
1140     }
1141 
GetRegAssignmentItem(bool isInt,regno_t regKey)1142     regno_t GetRegAssignmentItem(bool isInt, regno_t regKey)
1143     {
1144         return isInt ? intRegAssignmentMap[regKey] : fpRegAssignmentMap[regKey];
1145     }
1146 
SetRegAssignmentMap(bool isInt,regno_t regKey,regno_t regValue)1147     void SetRegAssignmentMap(bool isInt, regno_t regKey, regno_t regValue)
1148     {
1149         if (isInt) {
1150             intRegAssignmentMap[regKey] = regValue;
1151         } else {
1152             fpRegAssignmentMap[regKey] = regValue;
1153         }
1154     }
1155 
1156     /* only for HandleLocalRaDebug */
GetPregUsed(bool isInt)1157     uint64 GetPregUsed(bool isInt) const
1158     {
1159         if (isInt) {
1160             return intPregUsed;
1161         } else {
1162             return fpPregUsed;
1163         }
1164     }
1165 
SetPregUsed(regno_t regNO,bool isInt)1166     void SetPregUsed(regno_t regNO, bool isInt)
1167     {
1168         uint64 mask = 0;
1169         if (isInt) {
1170             mask = 1ULL << (regNO - R0);
1171             if ((intPregUsed & mask) == 0) {
1172                 ++numIntPregUsed;
1173                 intPregUsed |= mask;
1174             }
1175         } else {
1176             mask = 1ULL << (regNO - V0);
1177             if ((fpPregUsed & mask) == 0) {
1178                 ++numFpPregUsed;
1179                 fpPregUsed |= mask;
1180             }
1181         }
1182     }
1183 
isInRegSpilled(regno_t regNO,bool isInt)1184     bool isInRegSpilled(regno_t regNO, bool isInt) const
1185     {
1186         bool isSet;
1187         if (isInt) {
1188             isSet = IsBitArrElemSet(intRegSpilled, regNO);
1189         } else {
1190             isSet = IsBitArrElemSet(fpRegSpilled, regNO);
1191         }
1192         return isSet;
1193     }
1194 
SetRegSpilled(regno_t regNO,bool isInt)1195     void SetRegSpilled(regno_t regNO, bool isInt) const
1196     {
1197         if (isInt) {
1198             SetBitArrElement(intRegSpilled, regNO);
1199         } else {
1200             SetBitArrElement(fpRegSpilled, regNO);
1201         }
1202     }
1203 
GetPregs(bool isInt)1204     uint64 GetPregs(bool isInt) const
1205     {
1206         if (isInt) {
1207             return intPregs;
1208         } else {
1209             return fpPregs;
1210         }
1211     }
1212 
SetPregs(regno_t regNO,bool isInt)1213     void SetPregs(regno_t regNO, bool isInt)
1214     {
1215         if (isInt) {
1216             intPregs |= 1ULL << (regNO - RegBaseUpdate(true));
1217         } else {
1218             fpPregs |= 1ULL << (regNO - RegBaseUpdate(false));
1219         }
1220     }
1221 
ClearPregs(regno_t regNO,bool isInt)1222     void ClearPregs(regno_t regNO, bool isInt)
1223     {
1224         if (isInt) {
1225             intPregs &= ~(1ULL << (regNO - RegBaseUpdate(true)));
1226         } else {
1227             DEBUG_ASSERT(regNO >= RegBaseUpdate(false), "regNO - RegBaseUpdate(false) unsigned");
1228             fpPregs &= ~(1ULL << (regNO - RegBaseUpdate(false)));
1229         }
1230     }
1231 
IsPregAvailable(regno_t regNO,bool isInt)1232     bool IsPregAvailable(regno_t regNO, bool isInt) const
1233     {
1234         bool isAvailable;
1235         if (isInt) {
1236             isAvailable = intPregs & (1ULL << (regNO - RegBaseUpdate(true)));
1237         } else {
1238             DEBUG_ASSERT(regNO >= RegBaseUpdate(false), "regNO - RegBaseUpdate(false) unsigned");
1239             isAvailable = fpPregs & (1ULL << (regNO - RegBaseUpdate(false)));
1240         }
1241         return isAvailable;
1242     }
1243 
InitPregs(uint32 intMax,uint32 fpMax,bool hasYield,const MapleSet<uint32> & intSpillRegSet,const MapleSet<uint32> & fpSpillRegSet)1244     void InitPregs(uint32 intMax, uint32 fpMax, bool hasYield, const MapleSet<uint32> &intSpillRegSet,
1245                    const MapleSet<uint32> &fpSpillRegSet)
1246     {
1247         uint32 intBase = R0;
1248         uint32 fpBase = V0;
1249         intPregs = (1ULL << (intMax + 1)) - 1;
1250         fpPregs = (1ULL << (((fpMax + 1) + fpBase) - RegBaseUpdate(false))) - 1;
1251         for (uint32 regNO : intSpillRegSet) {
1252             ClearPregs(regNO + intBase, true);
1253         }
1254         for (uint32 regNO : fpSpillRegSet) {
1255             ClearPregs(regNO + fpBase, false);
1256         }
1257         if (hasYield) {
1258             ClearPregs(RYP, true);
1259         }
1260 #ifdef RESERVED_REGS
1261         intPregs &= ~(1ULL << R16);
1262         intPregs &= ~(1ULL << R17);
1263 #endif /* RESERVED_REGS */
1264     }
1265 
GetIntRegAssignmentMap()1266     const MapleMap<regno_t, regno_t> &GetIntRegAssignmentMap() const
1267     {
1268         return intRegAssignmentMap;
1269     }
1270 
GetFpRegAssignmentMap()1271     const MapleMap<regno_t, regno_t> &GetFpRegAssignmentMap() const
1272     {
1273         return fpRegAssignmentMap;
1274     }
1275 
GetUseInfo()1276     const MapleMap<regno_t, uint16> &GetUseInfo() const
1277     {
1278         return useInfo;
1279     }
1280 
SetUseInfoElem(regno_t regNO,uint16 info)1281     void SetUseInfoElem(regno_t regNO, uint16 info)
1282     {
1283         useInfo[regNO] = info;
1284     }
1285 
IncUseInfoElem(regno_t regNO)1286     void IncUseInfoElem(regno_t regNO)
1287     {
1288         if (useInfo.find(regNO) != useInfo.end()) {
1289             ++useInfo[regNO];
1290         }
1291     }
1292 
GetUseInfoElem(regno_t regNO)1293     uint16 GetUseInfoElem(regno_t regNO)
1294     {
1295         return useInfo[regNO];
1296     }
1297 
ClearUseInfo()1298     void ClearUseInfo()
1299     {
1300         useInfo.clear();
1301     }
1302 
GetDefInfo()1303     const MapleMap<regno_t, uint16> &GetDefInfo() const
1304     {
1305         return defInfo;
1306     }
1307 
SetDefInfoElem(regno_t regNO,uint16 info)1308     void SetDefInfoElem(regno_t regNO, uint16 info)
1309     {
1310         defInfo[regNO] = info;
1311     }
1312 
GetDefInfoElem(regno_t regNO)1313     uint16 GetDefInfoElem(regno_t regNO)
1314     {
1315         return defInfo[regNO];
1316     }
1317 
IncDefInfoElem(regno_t regNO)1318     void IncDefInfoElem(regno_t regNO)
1319     {
1320         if (defInfo.find(regNO) != defInfo.end()) {
1321             ++defInfo[regNO];
1322         }
1323     }
1324 
ClearDefInfo()1325     void ClearDefInfo()
1326     {
1327         defInfo.clear();
1328     }
1329 
GetNumIntPregUsed()1330     uint32 GetNumIntPregUsed() const
1331     {
1332         return numIntPregUsed;
1333     }
1334 
GetNumFpPregUsed()1335     uint32 GetNumFpPregUsed() const
1336     {
1337         return numFpPregUsed;
1338     }
1339 
1340 private:
ClearBitArrElement(uint64 * vec)1341     void ClearBitArrElement(uint64 *vec) const
1342     {
1343         for (uint32 i = 0; i < buckets; ++i) {
1344             vec[i] = 0UL;
1345         }
1346     }
1347 
SetBitArrElement(uint64 * vec,regno_t regNO)1348     void SetBitArrElement(uint64 *vec, regno_t regNO) const
1349     {
1350         uint32 index = regNO / kU64;
1351         uint64 bit = regNO % kU64;
1352         vec[index] |= 1ULL << bit;
1353     }
1354 
1355     /* The following local vars keeps track of allocation information in bb. */
1356     uint64 *intRegAssigned; /* in this set if vreg is assigned */
1357     uint64 *fpRegAssigned;
1358     MapleMap<regno_t, regno_t> intRegAssignmentMap; /* vreg -> preg map, which preg is the vreg assigned */
1359     MapleMap<regno_t, regno_t> fpRegAssignmentMap;
1360     uint64 intPregUsed = 0; /* pregs used in bb */
1361     uint64 fpPregUsed = 0;
1362     uint64 *intRegSpilled; /* on this list if vreg is spilled */
1363     uint64 *fpRegSpilled;
1364 
1365     uint64 intPregs = 0; /* available regs for assignement */
1366     uint64 fpPregs = 0;
1367     MapleMap<regno_t, uint16> useInfo; /* copy of local ra info for useCnt */
1368     MapleMap<regno_t, uint16> defInfo; /* copy of local ra info for defCnt */
1369 
1370     uint32 numIntPregUsed = 0;
1371     uint32 numFpPregUsed = 0;
1372     uint32 buckets;
1373 };
1374 
1375 class SplitBBInfo {
1376 public:
1377     SplitBBInfo() = default;
1378 
1379     ~SplitBBInfo() = default;
1380 
GetCandidateBB()1381     BB *GetCandidateBB()
1382     {
1383         return candidateBB;
1384     }
1385 
GetCandidateBB()1386     const BB *GetCandidateBB() const
1387     {
1388         return candidateBB;
1389     }
1390 
GetStartBB()1391     const BB *GetStartBB() const
1392     {
1393         return startBB;
1394     }
1395 
SetCandidateBB(BB & bb)1396     void SetCandidateBB(BB &bb)
1397     {
1398         candidateBB = &bb;
1399     }
1400 
SetStartBB(BB & bb)1401     void SetStartBB(BB &bb)
1402     {
1403         startBB = &bb;
1404     }
1405 
1406 private:
1407     BB *candidateBB = nullptr;
1408     BB *startBB = nullptr;
1409 };
1410 
1411 class GraphColorRegAllocator : public RegAllocator {
1412 public:
GraphColorRegAllocator(CGFunc & cgFunc,MemPool & memPool,DomAnalysis & dom,LoopAnalysis & loop)1413     GraphColorRegAllocator(CGFunc &cgFunc, MemPool &memPool, DomAnalysis &dom, LoopAnalysis &loop)
1414         : RegAllocator(cgFunc, memPool),
1415           domInfo(dom),
1416           loopInfo(loop),
1417           bbVec(alloc.Adapter()),
1418           vregLive(alloc.Adapter()),
1419           pregLive(alloc.Adapter()),
1420           lrMap(alloc.Adapter()),
1421           localRegVec(alloc.Adapter()),
1422           bbRegInfo(alloc.Adapter()),
1423           unconstrained(alloc.Adapter()),
1424           unconstrainedPref(alloc.Adapter()),
1425           constrained(alloc.Adapter()),
1426           mustAssigned(alloc.Adapter()),
1427 #ifdef OPTIMIZE_FOR_PROLOG
1428           intDelayed(alloc.Adapter()),
1429           fpDelayed(alloc.Adapter()),
1430 #endif /* OPTIMIZE_FOR_PROLOG */
1431           intCallerRegSet(alloc.Adapter()),
1432           intCalleeRegSet(alloc.Adapter()),
1433           intSpillRegSet(alloc.Adapter()),
1434           fpCallerRegSet(alloc.Adapter()),
1435           fpCalleeRegSet(alloc.Adapter()),
1436           fpSpillRegSet(alloc.Adapter()),
1437           intCalleeUsed(alloc.Adapter()),
1438           fpCalleeUsed(alloc.Adapter())
1439     {
1440         constexpr uint32 kNumInsnThreashold = 30000;
1441         numVregs = cgFunc.GetMaxVReg();
1442         localRegVec.resize(cgFunc.NumBBs());
1443         bbRegInfo.resize(cgFunc.NumBBs());
1444         if (CGOptions::DoMultiPassColorRA() && cgFunc.GetMirModule().IsCModule()) {
1445             uint32 cnt = 0;
1446             FOR_ALL_BB(bb, &cgFunc) {
1447                 FOR_BB_INSNS(insn, bb) {
1448                     ++cnt;
1449                 }
1450             }
1451             DEBUG_ASSERT(cnt <= cgFunc.GetTotalNumberOfInstructions(), "Incorrect insn count");
1452             if (cnt <= kNumInsnThreashold) {
1453                 doMultiPass = true;
1454                 doLRA = false;
1455                 doOptProlog = false;
1456             }
1457         }
1458     }
1459 
1460     ~GraphColorRegAllocator() override = default;
1461 
AllocateRegisters()1462     bool AllocateRegisters() override
1463     {
1464         return true;
1465     }
1466 
1467     enum SpillMemCheck : uint8 {
1468         kSpillMemPre,
1469         kSpillMemPost,
1470     };
1471 
GetLiveRange(regno_t regNO)1472     LiveRange *GetLiveRange(regno_t regNO)
1473     {
1474         auto it = lrMap.find(regNO);
1475         if (it != lrMap.end()) {
1476             return it->second;
1477         } else {
1478             return nullptr;
1479         }
1480     }
GetLiveRange(regno_t regNO)1481     LiveRange *GetLiveRange(regno_t regNO) const
1482     {
1483         auto it = lrMap.find(regNO);
1484         if (it != lrMap.end()) {
1485             return it->second;
1486         } else {
1487             return nullptr;
1488         }
1489     }
GetLrMap()1490     const MapleMap<regno_t, LiveRange *> &GetLrMap() const
1491     {
1492         return lrMap;
1493     }
1494 
1495 private:
1496     struct SetLiveRangeCmpFunc {
operatorSetLiveRangeCmpFunc1497         bool operator()(const LiveRange *lhs, const LiveRange *rhs) const
1498         {
1499             if (fabs(lhs->GetPriority() - rhs->GetPriority()) <= 1e-6) {
1500                 /*
1501                  * This is to ensure the ordering is consistent as the reg#
1502                  * differs going through VtableImpl.mpl file.
1503                  */
1504                 if (lhs->GetID() == rhs->GetID()) {
1505                     return lhs->GetRegNO() < rhs->GetRegNO();
1506                 } else {
1507                     return lhs->GetID() < rhs->GetID();
1508                 }
1509             }
1510             return (lhs->GetPriority() > rhs->GetPriority());
1511         }
1512     };
1513 
1514     template <typename Func>
1515     void ForEachBBArrElem(const uint64 *vec, Func functor) const;
1516 
1517     template <typename Func>
1518     void ForEachBBArrElemWithInterrupt(const uint64 *vec, Func functor) const;
1519 
1520     template <typename Func>
1521     void ForEachRegArrElem(const uint64 *vec, Func functor) const;
1522 
1523     void PrintLiveUnitMap(const LiveRange &lr) const;
1524     void PrintLiveRangeConflicts(const LiveRange &lr) const;
1525     void PrintLiveBBBit(const LiveRange &li) const;
1526     void PrintLiveRange(const LiveRange &li, const std::string &str) const;
1527     void PrintLiveRanges() const;
1528     void PrintLocalRAInfo(const std::string &str) const;
1529     void PrintBBAssignInfo() const;
1530     void PrintBBs() const;
1531 
1532     uint32 MaxIntPhysRegNum() const;
1533     uint32 MaxFloatPhysRegNum() const;
1534     bool IsReservedReg(AArch64reg regNO) const;
1535     void InitFreeRegPool();
1536     void InitCCReg();
1537     bool IsYieldPointReg(regno_t regNO) const;
1538     bool IsUnconcernedReg(regno_t regNO) const;
1539     bool IsUnconcernedReg(const RegOperand &regOpnd) const;
1540     LiveRange *NewLiveRange();
1541     bool CreateLiveRangeHandleLocal(regno_t regNO, const BB &bb, bool isDef);
1542     LiveRange *CreateLiveRangeAllocateAndUpdate(regno_t regNO, const BB &bb, bool isDef, uint32 currId);
1543     void CreateLiveRange(regno_t regNO, const BB &bb, bool isDef, uint32 currPoint, bool updateCount);
1544     bool SetupLiveRangeByOpHandlePhysicalReg(const RegOperand &op, Insn &insn, regno_t regNO, bool isDef);
1545     void SetupLiveRangeByOp(Operand &op, Insn &insn, bool isDef, uint32 &numUses);
1546     void SetupLiveRangeByRegNO(regno_t liveOut, BB &bb, uint32 currPoint);
1547     bool UpdateInsnCntAndSkipUseless(Insn &insn, uint32 &currPoint) const;
1548     void UpdateCallInfo(uint32 bbId, uint32 currPoint, const Insn &insn);
1549     void ClassifyOperand(std::unordered_set<regno_t> &pregs, std::unordered_set<regno_t> &vregs,
1550                          const Operand &opnd) const;
1551     void SetLrMustAssign(const RegOperand *regOpnd);
1552     void SetupMustAssignedLiveRanges(const Insn &insn);
1553     void ComputeLiveRangesForEachDefOperand(Insn &insn, bool &multiDef);
1554     void ComputeLiveRangesForEachUseOperand(Insn &insn);
1555     void ComputeLiveRangesUpdateIfInsnIsCall(const Insn &insn);
1556     void ComputeLiveRangesUpdateLiveUnitInsnRange(BB &bb, uint32 currPoint);
1557     MemOperand *CreateSpillMem(uint32 spillIdx, SpillMemCheck check);
1558     bool CheckOverlap(uint64 val, uint32 i, LiveRange &lr1, LiveRange &lr2) const;
1559     void CheckInterference(LiveRange &lr1, LiveRange &lr2) const;
1560     void SetBBInfoGlobalAssigned(uint32 bbID, regno_t regNO);
1561     bool HaveAvailableColor(const LiveRange &lr, uint32 num) const;
1562     void SplitAndColorForEachLr(MapleVector<LiveRange *> &targetLrVec);
1563     void SplitAndColor();
1564     void ColorForOptPrologEpilog();
1565     bool IsLocalReg(regno_t regNO) const;
1566     bool IsLocalReg(const LiveRange &lr) const;
1567     void HandleLocalRaDebug(regno_t regNO, const LocalRegAllocator &localRa, bool isInt) const;
1568     void HandleLocalRegAssignment(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1569     void UpdateLocalRegDefUseCount(regno_t regNO, LocalRegAllocator &localRa, bool isDef, bool isInt) const;
1570     void UpdateLocalRegConflict(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1571     void HandleLocalReg(Operand &op, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo, bool isDef, bool isInt);
1572     void LocalRaRegSetEraseReg(LocalRegAllocator &localRa, regno_t regNO) const;
1573     bool LocalRaInitRegSet(LocalRegAllocator &localRa, uint32 bbId);
1574     void LocalRaInitAllocatableRegs(LocalRegAllocator &localRa, uint32 bbId);
1575     void LocalRaForEachDefOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1576     void LocalRaForEachUseOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1577     void LocalRaPrepareBB(BB &bb, LocalRegAllocator &localRa);
1578     void LocalRaFinalAssignment(const LocalRegAllocator &localRa, BBAssignInfo &bbInfo);
1579     void LocalRaDebug(const BB &bb, const LocalRegAllocator &localRa) const;
1580     void LocalRegisterAllocator(bool allocate);
1581     MemOperand *GetSpillOrReuseMem(LiveRange &lr, uint32 regSize, bool &isOutOfRange, Insn &insn, bool isDef);
1582     void SpillOperandForSpillPre(Insn &insn, const Operand &opnd, RegOperand &phyOpnd, uint32 spillIdx, bool needSpill);
1583     MemOperand *GetConsistentReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1584                                       RegType regType);
1585     MemOperand *GetCommonReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1586                                   RegType regType);
1587     MemOperand *GetReuseMem(uint32 vregNO, uint32 size, RegType regType);
1588     MemOperand *GetSpillMem(uint32 vregNO, bool isDest, Insn &insn, AArch64reg regNO, bool &isOutOfRange) const;
1589     bool SetAvailableSpillReg(std::unordered_set<regno_t> &cannotUseReg, LiveRange &lr, uint64 &usedRegMask);
1590     void CollectCannotUseReg(std::unordered_set<regno_t> &cannotUseReg, const LiveRange &lr, Insn &insn);
1591     regno_t PickRegForSpill(uint64 &usedRegMask, RegType regType, uint32 spillIdx, bool &needSpillLr);
1592     bool SetRegForSpill(LiveRange &lr, Insn &insn, uint32 spillIdx, uint64 &usedRegMask, bool isDef);
1593     bool GetSpillReg(Insn &insn, LiveRange &lr, const uint32 &spillIdx, uint64 &usedRegMask, bool isDef);
1594     bool EncountPrevRef(const BB &pred, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1595     bool FoundPrevBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1596     bool EncountNextRef(const BB &succ, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1597     bool FoundNextBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1598     bool HavePrevRefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1599     bool HaveNextDefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1600     bool NeedCallerSave(Insn &insn, LiveRange &lr, bool isDef);
1601     void MarkCalleeSaveRegs();
1602     void MarkUsedRegs(Operand &opnd, uint64 &usedRegMask);
1603     bool LrGetBadReg(const LiveRange &lr) const;
1604     void OptCallerSave();
1605 
1606     MapleVector<LiveRange*>::iterator GetHighPriorityLr(MapleVector<LiveRange*> &lrSet) const;
1607     void UpdateForbiddenForNeighbors(const LiveRange &lr) const;
1608     void UpdatePregvetoForNeighbors(const LiveRange &lr) const;
1609     regno_t TryToAssignCallerSave(const LiveRange &lr) const;
1610     bool ShouldUseCallee(LiveRange &lr, const MapleSet<regno_t> &calleeUsed,
1611                          const MapleVector<LiveRange*> &delayed) const;
1612     void AddCalleeUsed(regno_t regNO, RegType regType);
1613     bool AssignColorToLr(LiveRange &lr, bool isDelayed = false);
1614     void PruneLrForSplit(LiveRange &lr, BB &bb, bool remove,
1615                          std::set<LoopDesc*, LoopDesc::LoopDescCmp> &candidateInLoop,
1616                          std::set<LoopDesc*, LoopDesc::LoopDescCmp> &defInLoop);
1617     bool UseIsUncovered(const BB &bb, const BB &startBB, std::vector<bool> &visitedBB);
1618     void FindUseForSplit(LiveRange &lr, SplitBBInfo &bbInfo, bool &remove,
1619                          std::set<LoopDesc*, LoopDesc::LoopDescCmp> &candidateInLoop,
1620                          std::set<LoopDesc*, LoopDesc::LoopDescCmp> &defInLoop);
1621     void FindBBSharedInSplit(LiveRange &lr, const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &candidateInLoop,
1622                              std::set<LoopDesc*, LoopDesc::LoopDescCmp> &defInLoop);
1623     void ComputeBBForNewSplit(LiveRange &newLr, LiveRange &oldLr);
1624     void ClearLrBBFlags(const std::set<BB*, SortedBBCmpFunc> &member) const;
1625     void ComputeBBForOldSplit(LiveRange &newLr, LiveRange &oldLr);
1626     bool LrCanBeColored(const LiveRange &lr, const BB &bbAdded, std::unordered_set<regno_t> &conflictRegs);
1627     void MoveLrBBInfo(LiveRange &oldLr, LiveRange &newLr, BB &bb) const;
1628     bool ContainsLoop(const LoopDesc &loop, const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &loops) const;
1629     void GetAllLrMemberLoops(LiveRange &lr, std::set<LoopDesc*, LoopDesc::LoopDescCmp> &loop);
1630     bool SplitLrShouldSplit(LiveRange &lr);
1631     bool SplitLrFindCandidateLr(LiveRange &lr, LiveRange &newLr, std::unordered_set<regno_t> &conflictRegs);
1632     void SplitLrHandleLoops(LiveRange &lr, LiveRange &newLr,
1633                             const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &oldLoops,
1634                             const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &newLoops);
1635     void SplitLrFixNewLrCallsAndRlod(LiveRange &newLr, const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &origLoops);
1636     void SplitLrFixOrigLrCalls(LiveRange &lr) const;
1637     void SplitLrUpdateInterference(LiveRange &lr);
1638     void SplitLrUpdateRegInfo(const LiveRange &origLr, LiveRange &newLr,
1639                               std::unordered_set<regno_t> &conflictRegs) const;
1640     void SplitLrErrorCheckAndDebug(const LiveRange &origLr) const;
1641     void SplitLr(LiveRange &lr);
1642 
1643     static constexpr uint16 kMaxUint16 = 0x7fff;
1644 
1645     DomAnalysis &domInfo;
1646     LoopAnalysis &loopInfo;
1647     MapleVector<BB *> bbVec;
1648     MapleUnorderedSet<regno_t> vregLive;
1649     MapleUnorderedSet<regno_t> pregLive;
1650     MapleMap<regno_t, LiveRange *> lrMap;
1651     MapleVector<LocalRaInfo *> localRegVec; /* local reg info for each bb, no local reg if null */
1652     MapleVector<BBAssignInfo *> bbRegInfo;  /* register assignment info for each bb */
1653     MapleVector<LiveRange *> unconstrained;
1654     MapleVector<LiveRange *> unconstrainedPref;
1655     MapleVector<LiveRange *> constrained;
1656     MapleVector<LiveRange *> mustAssigned;
1657 #ifdef OPTIMIZE_FOR_PROLOG
1658     MapleVector<LiveRange *> intDelayed;
1659     MapleVector<LiveRange *> fpDelayed;
1660 #endif                                /* OPTIMIZE_FOR_PROLOG  */
1661     MapleSet<uint32> intCallerRegSet; /* integer caller saved */
1662     MapleSet<uint32> intCalleeRegSet; /*         callee       */
1663     MapleSet<uint32> intSpillRegSet;  /*         spill        */
1664     MapleSet<uint32> fpCallerRegSet;  /* float caller saved   */
1665     MapleSet<uint32> fpCalleeRegSet;  /*       callee         */
1666     MapleSet<uint32> fpSpillRegSet;   /*       spill          */
1667     MapleSet<regno_t> intCalleeUsed;
1668     MapleSet<regno_t> fpCalleeUsed;
1669     Bfs *bfs = nullptr;
1670 
1671     uint32 bbBuckets = 0;  /* size of bit array for bb (each bucket == 64 bits) */
1672     uint32 regBuckets = 0; /* size of bit array for reg (each bucket == 64 bits) */
1673     uint32 intRegNum = 0;  /* total available int preg */
1674     uint32 fpRegNum = 0;   /* total available fp preg */
1675     uint32 numVregs = 0;   /* number of vregs when starting */
1676     regno_t ccReg = 0;
1677     /* For spilling of spill register if there are none available
1678      *   Example, all 3 operands spilled
1679      *                          sp_reg1 -> [spillMemOpnds[1]]
1680      *                          sp_reg2 -> [spillMemOpnds[2]]
1681      *                          ld sp_reg1 <- [addr-reg2]
1682      *                          ld sp_reg2 <- [addr-reg3]
1683      *   reg1 <- reg2, reg3     sp_reg1 <- sp_reg1, sp_reg2
1684      *                          st sp_reg1 -> [addr-reg1]
1685      *                          sp_reg1 <- [spillMemOpnds[1]]
1686      *                          sp_reg2 <- [spillMemOpnds[2]]
1687      */
1688     static constexpr size_t kSpillMemOpndNum = 4;
1689     std::array<MemOperand *, kSpillMemOpndNum> spillMemOpnds = {nullptr};
1690     bool operandSpilled[kSpillMemOpndNum];
1691     bool needExtraSpillReg = false;
1692 #ifdef USE_LRA
1693     bool doLRA = true;
1694 #else
1695     bool doLRA = false;
1696 #endif
1697 #ifdef OPTIMIZE_FOR_PROLOG
1698     bool doOptProlog = true;
1699 #else
1700     bool doOptProlog = false;
1701 #endif
1702     bool hasSpill = false;
1703     bool doMultiPass = false;
1704 };
1705 
1706 class CallerSavePre : public CGPre {
1707 public:
CallerSavePre(GraphColorRegAllocator * regAlloc,CGFunc & cgfunc,DomAnalysis & currDom,const LoopAnalysis & loop,MemPool & memPool,MemPool & mp2,PreKind kind,uint32 limit)1708     CallerSavePre(GraphColorRegAllocator *regAlloc, CGFunc &cgfunc, DomAnalysis &currDom, const LoopAnalysis &loop,
1709                   MemPool &memPool, MemPool &mp2, PreKind kind, uint32 limit)
1710         : CGPre(currDom, memPool, mp2, kind, limit),
1711           func(&cgfunc),
1712           loopInfo(loop),
1713           regAllocator(regAlloc),
1714           loopHeadBBs(ssaPreAllocator.Adapter())
1715     {
1716     }
1717 
1718     ~CallerSavePre() = default;
1719 
1720     void ApplySSAPRE();
SetDump(bool val)1721     void SetDump(bool val)
1722     {
1723         dump = val;
1724     }
1725 
1726 private:
1727     void ComputeAvail();
1728     void ComputeVarAndDfPhis() override;
BuildWorkList()1729     void BuildWorkList() override {};
1730     void DumpWorkCandAndOcc();
1731 
GetBB(uint32 id)1732     BB *GetBB(uint32 id) const override
1733     {
1734         return func->GetBBFromID(id);
1735     }
1736 
GetPUIdx()1737     PUIdx GetPUIdx() const override
1738     {
1739         return func->GetFunction().GetPuidx();
1740     }
1741 
IsLoopHeadBB(uint32 bbId)1742     bool IsLoopHeadBB(uint32 bbId) const
1743     {
1744         return loopHeadBBs.find(bbId) != loopHeadBBs.end();
1745     }
1746     CGFunc *func;
1747     const LoopAnalysis &loopInfo;
1748     bool dump = false;
1749     LiveRange *workLr = nullptr;
1750     GraphColorRegAllocator *regAllocator;
1751     MapleSet<uint32> loopHeadBBs;
1752     bool beyondLimit = false;
1753 };
1754 } /* namespace maplebe */
1755 
1756 #endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H */
1757