• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
17 #define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
18 #include "reg_alloc.h"
19 #include "aarch64_operand.h"
20 #include "aarch64_insn.h"
21 #include "aarch64_abi.h"
22 #include "aarch64_cgfunc.h"
23 #include "loop.h"
24 #include "cg_dominance.h"
25 #include "cg_pre.h"
26 
27 namespace maplebe {
28 #define RESERVED_REGS
29 
30 #define USE_LRA
31 #define USE_SPLIT
32 #undef USE_BB_FREQUENCY
33 #define OPTIMIZE_FOR_PROLOG
34 #define REUSE_SPILLMEM
35 #undef COLOR_SPLIT
36 #define MOVE_COALESCE
37 
38 /* for robust test */
39 #undef CONSISTENT_MEMOPND
40 #undef RANDOM_PRIORITY
41 
42 constexpr uint32 k32 = sizeof(int) * CHAR_BIT;
43 constexpr uint32 k64 = sizeof(int64) * CHAR_BIT;
44 constexpr uint32 kU64 = sizeof(uint64) * CHAR_BIT;
45 
46 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::set<T,Comparator> & set,const T & item)47 inline bool FindNotIn(const std::set<T, Comparator> &set, const T &item)
48 {
49     return set.find(item) == set.end();
50 }
51 
52 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::unordered_set<T,Comparator> & set,const T & item)53 inline bool FindNotIn(const std::unordered_set<T, Comparator> &set, const T &item)
54 {
55     return set.find(item) == set.end();
56 }
57 
58 template <typename T>
FindNotIn(const MapleSet<T> & set,const T & item)59 inline bool FindNotIn(const MapleSet<T> &set, const T &item)
60 {
61     return set.find(item) == set.end();
62 }
63 
64 template <typename T>
FindNotIn(const MapleUnorderedSet<T> & set,const T & item)65 inline bool FindNotIn(const MapleUnorderedSet<T> &set, const T &item)
66 {
67     return set.find(item) == set.end();
68 }
69 
70 template <typename T>
FindNotIn(const MapleList<T> & list,const T & item)71 inline bool FindNotIn(const MapleList<T> &list, const T &item)
72 {
73     return std::find(list.begin(), list.end(), item) == list.end();
74 }
75 
76 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::set<T,Comparator> & set,const T & item)77 inline bool FindIn(const std::set<T, Comparator> &set, const T &item)
78 {
79     return set.find(item) != set.end();
80 }
81 
82 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::unordered_set<T,Comparator> & set,const T & item)83 inline bool FindIn(const std::unordered_set<T, Comparator> &set, const T &item)
84 {
85     return set.find(item) != set.end();
86 }
87 
88 template <typename T>
FindIn(const MapleSet<T> & set,const T & item)89 inline bool FindIn(const MapleSet<T> &set, const T &item)
90 {
91     return set.find(item) != set.end();
92 }
93 
94 template <typename T>
FindIn(const MapleUnorderedSet<T> & set,const T & item)95 inline bool FindIn(const MapleUnorderedSet<T> &set, const T &item)
96 {
97     return set.find(item) != set.end();
98 }
99 
100 template <typename T>
FindIn(const MapleList<T> & list,const T & item)101 inline bool FindIn(const MapleList<T> &list, const T &item)
102 {
103     return std::find(list.begin(), list.end(), item) != list.end();
104 }
105 
IsBitArrElemSet(const uint64 * vec,const uint32 num)106 inline bool IsBitArrElemSet(const uint64 *vec, const uint32 num)
107 {
108     size_t index = num / kU64;
109     uint64 bit = num % kU64;
110     return vec[index] & (1ULL << bit);
111 }
112 
IsBBsetOverlap(const uint64 * vec1,const uint64 * vec2,uint32 bbBuckets)113 inline bool IsBBsetOverlap(const uint64 *vec1, const uint64 *vec2, uint32 bbBuckets)
114 {
115     for (uint32 i = 0; i < bbBuckets; ++i) {
116         if ((vec1[i] & vec2[i]) != 0) {
117             return true;
118         }
119     }
120     return false;
121 }
122 
123 /* For each bb, record info pertain to allocation */
124 /*
125  * This is per bb per LR.
126  * LU info is particular to a bb in a LR.
127  */
128 class LiveUnit {
129 public:
130     LiveUnit() = default;
131     ~LiveUnit() = default;
132 
133     void PrintLiveUnit() const;
134 
GetBegin()135     uint32 GetBegin() const
136     {
137         return begin;
138     }
139 
SetBegin(uint32 val)140     void SetBegin(uint32 val)
141     {
142         begin = val;
143     }
144 
GetEnd()145     uint32 GetEnd() const
146     {
147         return end;
148     }
149 
SetEnd(uint32 end)150     void SetEnd(uint32 end)
151     {
152         this->end = end;
153     }
154 
HasCall()155     bool HasCall() const
156     {
157         return hasCall;
158     }
159 
SetHasCall(bool hasCall)160     void SetHasCall(bool hasCall)
161     {
162         this->hasCall = hasCall;
163     }
164 
GetDefNum()165     uint32 GetDefNum() const
166     {
167         return defNum;
168     }
169 
SetDefNum(uint32 defNum)170     void SetDefNum(uint32 defNum)
171     {
172         this->defNum = defNum;
173     }
174 
IncDefNum()175     void IncDefNum()
176     {
177         ++defNum;
178     }
179 
GetUseNum()180     uint32 GetUseNum() const
181     {
182         return useNum;
183     }
184 
SetUseNum(uint32 useNum)185     void SetUseNum(uint32 useNum)
186     {
187         this->useNum = useNum;
188     }
189 
IncUseNum()190     void IncUseNum()
191     {
192         ++useNum;
193     }
194 
NeedReload()195     bool NeedReload() const
196     {
197         return needReload;
198     }
199 
SetNeedReload(bool needReload)200     void SetNeedReload(bool needReload)
201     {
202         this->needReload = needReload;
203     }
204 
NeedRestore()205     bool NeedRestore() const
206     {
207         return needRestore;
208     }
209 
SetNeedRestore(bool needRestore)210     void SetNeedRestore(bool needRestore)
211     {
212         this->needRestore = needRestore;
213     }
214 
215 private:
216     uint32 begin = 0;     /* first encounter in bb */
217     uint32 end = 0;       /* last encounter in bb */
218     bool hasCall = false; /* bb has a call */
219     uint32 defNum = 0;
220     uint32 useNum = 0; /* used for priority calculation */
221     bool needReload = false;
222     bool needRestore = false;
223 };
224 
225 struct SortedBBCmpFunc {
operatorSortedBBCmpFunc226     bool operator()(const BB *lhs, const BB *rhs) const
227     {
228         return (lhs->GetLevel() < rhs->GetLevel());
229     }
230 };
231 
232 enum refType : uint8 {
233     kIsUse = 0x1,
234     kIsDef = 0x2,
235     kIsCall = 0x4,
236 };
237 
238 /* LR is for each global vreg. */
239 class LiveRange {
240 public:
LiveRange(MapleAllocator & allocator)241     explicit LiveRange(MapleAllocator &allocator)
242         : lrAlloca(&allocator),
243           pregveto(allocator.Adapter()),
244           callDef(allocator.Adapter()),
245           forbidden(allocator.Adapter()),
246           prefs(allocator.Adapter()),
247           refMap(allocator.Adapter()),
248           luMap(allocator.Adapter())
249     {
250     }
251 
252     ~LiveRange() = default;
253 
GetRegNO()254     regno_t GetRegNO() const
255     {
256         return regNO;
257     }
258 
SetRegNO(regno_t val)259     void SetRegNO(regno_t val)
260     {
261         regNO = val;
262     }
263 
GetID()264     uint32 GetID() const
265     {
266         return id;
267     }
268 
SetID(uint32 id)269     void SetID(uint32 id)
270     {
271         this->id = id;
272     }
273 
GetAssignedRegNO()274     regno_t GetAssignedRegNO() const
275     {
276         return assignedRegNO;
277     }
278 
SetAssignedRegNO(regno_t val)279     void SetAssignedRegNO(regno_t val)
280     {
281         assignedRegNO = val;
282     }
283 
GetNumCall()284     uint32 GetNumCall() const
285     {
286         return numCall;
287     }
288 
SetNumCall(uint32 num)289     void SetNumCall(uint32 num)
290     {
291         numCall = num;
292     }
293 
IncNumCall()294     void IncNumCall()
295     {
296         ++numCall;
297     }
298 
GetRegType()299     RegType GetRegType() const
300     {
301         return regType;
302     }
303 
SetRegType(RegType regType)304     void SetRegType(RegType regType)
305     {
306         this->regType = regType;
307     }
308 
GetPriority()309     float GetPriority() const
310     {
311         return priority;
312     }
313 
SetPriority(float priority)314     void SetPriority(float priority)
315     {
316         this->priority = priority;
317     }
318 
IsMustAssigned()319     bool IsMustAssigned() const
320     {
321         return mustAssigned;
322     }
323 
SetMustAssigned()324     void SetMustAssigned()
325     {
326         mustAssigned = true;
327     }
328 
SetBBBuckets(uint32 bucketNum)329     void SetBBBuckets(uint32 bucketNum)
330     {
331         bbBuckets = bucketNum;
332     }
333 
SetRegBuckets(uint32 bucketNum)334     void SetRegBuckets(uint32 bucketNum)
335     {
336         regBuckets = bucketNum;
337     }
338 
GetNumBBMembers()339     uint32 GetNumBBMembers() const
340     {
341         return numBBMembers;
342     }
343 
IncNumBBMembers()344     void IncNumBBMembers()
345     {
346         ++numBBMembers;
347     }
348 
DecNumBBMembers()349     void DecNumBBMembers()
350     {
351         --numBBMembers;
352     }
353 
InitBBMember(MemPool & memPool,size_t size)354     void InitBBMember(MemPool &memPool, size_t size)
355     {
356         bbMember = memPool.NewArray<uint64>(size);
357         errno_t ret = memset_s(bbMember, size * sizeof(uint64), 0, size * sizeof(uint64));
358         CHECK_FATAL(ret == EOK, "call memset_s failed");
359     }
360 
GetBBMember()361     uint64 *GetBBMember()
362     {
363         return bbMember;
364     }
365 
GetBBMember()366     const uint64 *GetBBMember() const
367     {
368         return bbMember;
369     }
370 
GetBBMemberElem(int32 index)371     uint64 GetBBMemberElem(int32 index) const
372     {
373         return bbMember[index];
374     }
375 
SetBBMemberElem(int32 index,uint64 elem)376     void SetBBMemberElem(int32 index, uint64 elem)
377     {
378         bbMember[index] = elem;
379     }
380 
SetMemberBitArrElem(uint32 bbID)381     void SetMemberBitArrElem(uint32 bbID)
382     {
383         uint32 index = bbID / kU64;
384         uint64 bit = bbID % kU64;
385         uint64 mask = 1ULL << bit;
386         if ((GetBBMemberElem(index) & mask) == 0) {
387             IncNumBBMembers();
388             SetBBMemberElem(index, GetBBMemberElem(index) | mask);
389         }
390     }
391 
UnsetMemberBitArrElem(uint32 bbID)392     void UnsetMemberBitArrElem(uint32 bbID)
393     {
394         uint32 index = bbID / kU64;
395         uint64 bit = bbID % kU64;
396         uint64 mask = 1ULL << bit;
397         if ((GetBBMemberElem(index) & mask) != 0) {
398             DecNumBBMembers();
399             SetBBMemberElem(index, GetBBMemberElem(index) & (~mask));
400         }
401     }
402 
SetConflictBitArrElem(regno_t regNO)403     void SetConflictBitArrElem(regno_t regNO)
404     {
405         uint32 index = regNO / kU64;
406         uint64 bit = regNO % kU64;
407         uint64 mask = 1ULL << bit;
408         if ((GetBBConflictElem(index) & mask) == 0) {
409             IncNumBBConflicts();
410             SetBBConflictElem(index, GetBBConflictElem(index) | mask);
411         }
412     }
413 
UnsetConflictBitArrElem(regno_t regNO)414     void UnsetConflictBitArrElem(regno_t regNO)
415     {
416         uint32 index = regNO / kU64;
417         uint64 bit = regNO % kU64;
418         uint64 mask = 1ULL << bit;
419         if ((GetBBConflictElem(index) & mask) != 0) {
420             DecNumBBConflicts();
421             SetBBConflictElem(index, GetBBConflictElem(index) & (~mask));
422         }
423     }
424 
InitPregveto()425     void InitPregveto()
426     {
427         pregveto.clear();
428         pregveto.resize(kMaxRegNum);
429         callDef.clear();
430         callDef.resize(kMaxRegNum);
431     }
432 
GetPregveto(regno_t regno)433     bool GetPregveto(regno_t regno) const
434     {
435         return pregveto[regno];
436     }
437 
GetPregvetoSize()438     size_t GetPregvetoSize() const
439     {
440         return numPregveto;
441     }
442 
InsertElemToPregveto(regno_t regno)443     void InsertElemToPregveto(regno_t regno)
444     {
445         if (!pregveto[regno]) {
446             pregveto[regno] = true;
447             ++numPregveto;
448         }
449     }
450 
GetCallDef(regno_t regno)451     bool GetCallDef(regno_t regno) const
452     {
453         return callDef[regno];
454     }
455 
InsertElemToCallDef(regno_t regno)456     void InsertElemToCallDef(regno_t regno)
457     {
458         if (!callDef[regno]) {
459             callDef[regno] = true;
460             ++numCallDef;
461         }
462     }
463 
SetCrossCall()464     void SetCrossCall()
465     {
466         crossCall = true;
467     }
468 
GetCrossCall()469     bool GetCrossCall() const
470     {
471         return crossCall;
472     }
473 
InitForbidden()474     void InitForbidden()
475     {
476         forbidden.clear();
477         forbidden.resize(kMaxRegNum);
478     }
479 
GetForbidden()480     const MapleVector<bool> &GetForbidden() const
481     {
482         return forbidden;
483     }
484 
GetForbidden(regno_t regno)485     bool GetForbidden(regno_t regno) const
486     {
487         return forbidden[regno];
488     }
489 
GetForbiddenSize()490     size_t GetForbiddenSize() const
491     {
492         return numForbidden;
493     }
494 
InsertElemToForbidden(regno_t regno)495     void InsertElemToForbidden(regno_t regno)
496     {
497         if (!forbidden[regno]) {
498             forbidden[regno] = true;
499             ++numForbidden;
500         }
501     }
502 
EraseElemFromForbidden(regno_t regno)503     void EraseElemFromForbidden(regno_t regno)
504     {
505         if (forbidden[regno]) {
506             forbidden[regno] = false;
507             --numForbidden;
508         }
509     }
510 
ClearForbidden()511     void ClearForbidden()
512     {
513         forbidden.clear();
514     }
515 
GetNumBBConflicts()516     uint32 GetNumBBConflicts() const
517     {
518         return numBBConflicts;
519     }
520 
IncNumBBConflicts()521     void IncNumBBConflicts()
522     {
523         ++numBBConflicts;
524     }
525 
DecNumBBConflicts()526     void DecNumBBConflicts()
527     {
528         --numBBConflicts;
529     }
530 
InitBBConflict(MemPool & memPool,size_t size)531     void InitBBConflict(MemPool &memPool, size_t size)
532     {
533         bbConflict = memPool.NewArray<uint64>(size);
534         errno_t ret = memset_s(bbConflict, size * sizeof(uint64), 0, size * sizeof(uint64));
535         CHECK_FATAL(ret == EOK, "call memset_s failed");
536     }
537 
GetBBConflict()538     const uint64 *GetBBConflict() const
539     {
540         return bbConflict;
541     }
542 
GetBBConflictElem(int32 index)543     uint64 GetBBConflictElem(int32 index) const
544     {
545         DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
546         return bbConflict[index];
547     }
548 
SetBBConflictElem(int32 index,uint64 elem)549     void SetBBConflictElem(int32 index, uint64 elem)
550     {
551         DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
552         bbConflict[index] = elem;
553     }
554 
SetOldConflict(uint64 * conflict)555     void SetOldConflict(uint64 *conflict)
556     {
557         oldConflict = conflict;
558     }
559 
GetOldConflict()560     const uint64 *GetOldConflict() const
561     {
562         return oldConflict;
563     }
564 
GetPrefs()565     const MapleSet<regno_t> &GetPrefs() const
566     {
567         return prefs;
568     }
569 
InsertElemToPrefs(regno_t regNO)570     void InsertElemToPrefs(regno_t regNO)
571     {
572         (void)prefs.insert(regNO);
573     }
574 
GetRefs()575     const MapleMap<uint32, MapleMap<uint32, uint32> *> GetRefs() const
576     {
577         return refMap;
578     }
579 
GetRefs(uint32 bbId)580     const MapleMap<uint32, uint32> GetRefs(uint32 bbId) const
581     {
582         return *(refMap.find(bbId)->second);
583     }
584 
AddRef(uint32 bbId,uint32 pos,uint32 mark)585     void AddRef(uint32 bbId, uint32 pos, uint32 mark)
586     {
587         if (refMap.find(bbId) == refMap.end()) {
588             auto point = lrAlloca->New<MapleMap<uint32, uint32>>(lrAlloca->Adapter());
589             (void)point->emplace(std::pair<uint32, uint32>(pos, mark));
590             (void)refMap.emplace(std::pair<uint32, MapleMap<uint32, uint32> *>(bbId, point));
591         } else {
592             auto &bbPoint = (refMap.find(bbId))->second;
593             if (bbPoint->find(pos) == bbPoint->end()) {
594                 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, mark));
595             } else {
596                 auto posVal = bbPoint->find(pos)->second;
597                 (void)bbPoint->erase(bbPoint->find(pos));
598                 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, posVal | mark));
599             }
600         }
601     }
602 
GetLuMap()603     const MapleMap<uint32, LiveUnit *> &GetLuMap() const
604     {
605         return luMap;
606     }
607 
FindInLuMap(uint32 index)608     MapleMap<uint32, LiveUnit *>::iterator FindInLuMap(uint32 index)
609     {
610         return luMap.find(index);
611     }
612 
EndOfLuMap()613     MapleMap<uint32, LiveUnit *>::iterator EndOfLuMap()
614     {
615         return luMap.end();
616     }
617 
EraseLuMap(MapleMap<uint32,LiveUnit * >::iterator it)618     MapleMap<uint32, LiveUnit *>::iterator EraseLuMap(MapleMap<uint32, LiveUnit *>::iterator it)
619     {
620         return luMap.erase(it);
621     }
622 
SetElemToLuMap(uint32 key,LiveUnit & value)623     void SetElemToLuMap(uint32 key, LiveUnit &value)
624     {
625         luMap[key] = &value;
626     }
627 
GetLiveUnitFromLuMap(uint32 key)628     LiveUnit *GetLiveUnitFromLuMap(uint32 key)
629     {
630         return luMap[key];
631     }
632 
GetLiveUnitFromLuMap(uint32 key)633     const LiveUnit *GetLiveUnitFromLuMap(uint32 key) const
634     {
635         auto it = luMap.find(key);
636         DEBUG_ASSERT(it != luMap.end(), "can't find live unit");
637         return it->second;
638     }
639 
GetSplitLr()640     const LiveRange *GetSplitLr() const
641     {
642         return splitLr;
643     }
644 
SetSplitLr(LiveRange & lr)645     void SetSplitLr(LiveRange &lr)
646     {
647         splitLr = &lr;
648     }
649 
650 #ifdef OPTIMIZE_FOR_PROLOG
GetNumDefs()651     uint32 GetNumDefs() const
652     {
653         return numDefs;
654     }
655 
IncNumDefs()656     void IncNumDefs()
657     {
658         ++numDefs;
659     }
660 
SetNumDefs(uint32 val)661     void SetNumDefs(uint32 val)
662     {
663         numDefs = val;
664     }
665 
GetNumUses()666     uint32 GetNumUses() const
667     {
668         return numUses;
669     }
670 
IncNumUses()671     void IncNumUses()
672     {
673         ++numUses;
674     }
675 
SetNumUses(uint32 val)676     void SetNumUses(uint32 val)
677     {
678         numUses = val;
679     }
680 
GetFrequency()681     uint32 GetFrequency() const
682     {
683         return frequency;
684     }
685 
SetFrequency(uint32 frequency)686     void SetFrequency(uint32 frequency)
687     {
688         this->frequency = frequency;
689     }
690 #endif /* OPTIMIZE_FOR_PROLOG */
691 
GetSpillMem()692     MemOperand *GetSpillMem()
693     {
694         return spillMem;
695     }
696 
GetSpillMem()697     const MemOperand *GetSpillMem() const
698     {
699         return spillMem;
700     }
701 
SetSpillMem(MemOperand & memOpnd)702     void SetSpillMem(MemOperand &memOpnd)
703     {
704         spillMem = &memOpnd;
705     }
706 
GetSpillReg()707     regno_t GetSpillReg() const
708     {
709         return spillReg;
710     }
711 
SetSpillReg(regno_t spillReg)712     void SetSpillReg(regno_t spillReg)
713     {
714         this->spillReg = spillReg;
715     }
716 
GetSpillSize()717     uint32 GetSpillSize() const
718     {
719         return spillSize;
720     }
721 
SetSpillSize(uint32 size)722     void SetSpillSize(uint32 size)
723     {
724         spillSize = size;
725     }
726 
IsSpilled()727     bool IsSpilled() const
728     {
729         return spilled;
730     }
731 
SetSpilled(bool spill)732     void SetSpilled(bool spill)
733     {
734         spilled = spill;
735     }
736 
HasDefUse()737     bool HasDefUse() const
738     {
739         return hasDefUse;
740     }
741 
SetDefUse()742     void SetDefUse()
743     {
744         hasDefUse = true;
745     }
746 
GetProcessed()747     bool GetProcessed() const
748     {
749         return proccessed;
750     }
751 
SetProcessed()752     void SetProcessed()
753     {
754         proccessed = true;
755     }
756 
IsNonLocal()757     bool IsNonLocal() const
758     {
759         return isNonLocal;
760     }
761 
SetIsNonLocal(bool isNonLocal)762     void SetIsNonLocal(bool isNonLocal)
763     {
764         this->isNonLocal = isNonLocal;
765     }
766 
SetRematLevel(uint32 val)767     void SetRematLevel(uint32 val)
768     {
769         this->rematLevel = val;
770     }
771 
GetRematLevel()772     uint32 GetRematLevel() const
773     {
774         return this->rematLevel;
775     }
776 
GetOp()777     Opcode GetOp() const
778     {
779         return op;
780     }
781 
GetRematSymbol()782     const MIRSymbol *GetRematSymbol() const
783     {
784         DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat symbol is invalid");
785         return rematInfo.sym;
786     }
787 
GetRematFieldID()788     FieldID GetRematFieldID() const
789     {
790         DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat field ID is invalid");
791         return fieldID;
792     }
793 
SetRematerializable(const MIRConst * c)794     void SetRematerializable(const MIRConst *c)
795     {
796         op = OP_constval;
797         rematInfo.mirConst = c;
798     }
799 
SetRematerializable(Opcode opcode,const MIRSymbol * symbol,FieldID fieldId,bool addrUp)800     void SetRematerializable(Opcode opcode, const MIRSymbol *symbol, FieldID fieldId, bool addrUp)
801     {
802         this->op = opcode;
803         rematInfo.sym = symbol;
804         this->fieldID = fieldId;
805         this->addrUpper = addrUp;
806     }
807 
CopyRematerialization(const LiveRange & lr)808     void CopyRematerialization(const LiveRange &lr)
809     {
810         op = lr.op;
811         rematInfo = lr.rematInfo;
812         fieldID = lr.fieldID;
813     }
814 
815     bool IsRematerializable(AArch64CGFunc &cgFunc, uint8 rematLevel) const;
816     std::vector<Insn *> Rematerialize(AArch64CGFunc *cgFunc, RegOperand &regOp);
817 
818 private:
819     MapleAllocator *lrAlloca;
820     regno_t regNO = 0;
821     uint32 id = 0;             /* for priority tie breaker */
822     regno_t assignedRegNO = 0; /* color assigned */
823     uint32 numCall = 0;
824     RegType regType = kRegTyUndef;
825     float priority = 0.0;
826     bool mustAssigned = false;
827     uint32 bbBuckets = 0;       /* size of bit array for bb (each bucket == 64 bits) */
828     uint32 regBuckets = 0;      /* size of bit array for reg (each bucket == 64 bits) */
829     uint32 numBBMembers = 0;    /* number of bits set in bbMember */
830     uint64 *bbMember = nullptr; /* Same as smember, but use bit array */
831 
832     MapleVector<bool> pregveto;  /* pregs cannot be assigned   -- SplitLr may clear forbidden */
833     MapleVector<bool> callDef;   /* pregs cannot be assigned   -- SplitLr may clear forbidden */
834     MapleVector<bool> forbidden; /* pregs cannot be assigned */
835     uint32 numPregveto = 0;
836     uint32 numCallDef = 0;
837     uint32 numForbidden = 0;
838     bool crossCall = false;
839 
840     uint32 numBBConflicts = 0;    /* number of bits set in bbConflict */
841     uint64 *bbConflict = nullptr; /* vreg interference from graph neighbors (bit) */
842     uint64 *oldConflict = nullptr;
843     MapleSet<regno_t> prefs; /* pregs that prefer */
844     MapleMap<uint32, MapleMap<uint32, uint32> *> refMap;
845     MapleMap<uint32, LiveUnit *> luMap; /* info for each bb */
846     LiveRange *splitLr = nullptr;       /* The 1st part of the split */
847 #ifdef OPTIMIZE_FOR_PROLOG
848     uint32 numDefs = 0;
849     uint32 numUses = 0;
850     uint32 frequency = 0;
851 #endif                              /* OPTIMIZE_FOR_PROLOG */
852     MemOperand *spillMem = nullptr; /* memory operand used for spill, if any */
853     regno_t spillReg = 0;           /* register operand for spill at current point */
854     uint32 spillSize = 0;           /* 32 or 64 bit spill */
855     bool spilled = false;           /* color assigned */
856     bool hasDefUse = false;         /* has regDS */
857     bool proccessed = false;
858     bool isNonLocal = false;
859     uint32 rematLevel = 0;
860     Opcode op = OP_undef; /* OP_constval, OP_addrof or OP_dread if rematerializable */
861     union RematInfo {
862         const MIRConst *mirConst;
863         const MIRSymbol *sym;
864     } rematInfo;            /* info for rematerializing value */
865     FieldID fieldID = 0;    /* used only when op is OP_addrof or OP_dread */
866     bool addrUpper = false; /* indicates the upper bits of an addrof */
867 };
868 
869 /* One per bb, to communicate local usage to global RA */
870 class LocalRaInfo {
871 public:
LocalRaInfo(MapleAllocator & allocator)872     explicit LocalRaInfo(MapleAllocator &allocator) : defCnt(allocator.Adapter()), useCnt(allocator.Adapter()) {}
873 
874     ~LocalRaInfo() = default;
875 
GetDefCnt()876     const MapleMap<regno_t, uint16> &GetDefCnt() const
877     {
878         return defCnt;
879     }
880 
GetDefCntElem(regno_t regNO)881     uint16 GetDefCntElem(regno_t regNO)
882     {
883         return defCnt[regNO];
884     }
885 
SetDefCntElem(regno_t key,uint16 value)886     void SetDefCntElem(regno_t key, uint16 value)
887     {
888         defCnt[key] = value;
889     }
890 
GetUseCnt()891     const MapleMap<regno_t, uint16> &GetUseCnt() const
892     {
893         return useCnt;
894     }
895 
GetUseCntElem(regno_t regNO)896     uint16 GetUseCntElem(regno_t regNO)
897     {
898         return useCnt[regNO];
899     }
900 
SetUseCntElem(regno_t key,uint16 value)901     void SetUseCntElem(regno_t key, uint16 value)
902     {
903         useCnt[key] = value;
904     }
905 
906 private:
907     MapleMap<regno_t, uint16> defCnt;
908     MapleMap<regno_t, uint16> useCnt;
909 };
910 
911 /* For each bb, record info pertain to allocation */
912 class BBAssignInfo {
913 public:
BBAssignInfo(MapleAllocator & allocator)914     explicit BBAssignInfo(MapleAllocator &allocator) : globalsAssigned(allocator.Adapter()), regMap(allocator.Adapter())
915     {
916     }
917 
918     ~BBAssignInfo() = default;
919 
GetIntLocalRegsNeeded()920     uint32 GetIntLocalRegsNeeded() const
921     {
922         return intLocalRegsNeeded;
923     }
924 
SetIntLocalRegsNeeded(uint32 num)925     void SetIntLocalRegsNeeded(uint32 num)
926     {
927         intLocalRegsNeeded = num;
928     }
929 
GetFpLocalRegsNeeded()930     uint32 GetFpLocalRegsNeeded() const
931     {
932         return fpLocalRegsNeeded;
933     }
934 
SetFpLocalRegsNeeded(uint32 num)935     void SetFpLocalRegsNeeded(uint32 num)
936     {
937         fpLocalRegsNeeded = num;
938     }
939 
InitGlobalAssigned()940     void InitGlobalAssigned()
941     {
942         globalsAssigned.clear();
943         globalsAssigned.resize(kMaxRegNum);
944     }
945 
GetGlobalsAssigned(regno_t regNO)946     bool GetGlobalsAssigned(regno_t regNO) const
947     {
948         return globalsAssigned[regNO];
949     }
950 
InsertElemToGlobalsAssigned(regno_t regNO)951     void InsertElemToGlobalsAssigned(regno_t regNO)
952     {
953         globalsAssigned[regNO] = true;
954     }
955 
EraseElemToGlobalsAssigned(regno_t regNO)956     void EraseElemToGlobalsAssigned(regno_t regNO)
957     {
958         globalsAssigned[regNO] = false;
959     }
960 
GetRegMap()961     const MapleMap<regno_t, regno_t> &GetRegMap() const
962     {
963         return regMap;
964     }
965 
HasRegMap(regno_t regNOKey)966     bool HasRegMap(regno_t regNOKey) const
967     {
968         return (regMap.find(regNOKey) != regMap.end());
969     }
970 
GetRegMapElem(regno_t regNO)971     regno_t GetRegMapElem(regno_t regNO)
972     {
973         return regMap[regNO];
974     }
975 
SetRegMapElem(regno_t regNOKey,regno_t regNOValue)976     void SetRegMapElem(regno_t regNOKey, regno_t regNOValue)
977     {
978         regMap[regNOKey] = regNOValue;
979     }
980 
981 private:
982     uint32 intLocalRegsNeeded = 0;     /* num local reg needs for each bb */
983     uint32 fpLocalRegsNeeded = 0;      /* num local reg needs for each bb */
984     MapleVector<bool> globalsAssigned; /* globals used in a bb */
985     MapleMap<regno_t, regno_t> regMap; /* local vreg to preg mapping */
986 };
987 
988 class FinalizeRegisterInfo {
989 public:
FinalizeRegisterInfo(MapleAllocator & allocator)990     explicit FinalizeRegisterInfo(MapleAllocator &allocator)
991         : defOperands(allocator.Adapter()),
992           defIdx(allocator.Adapter()),
993           useOperands(allocator.Adapter()),
994           useIdx(allocator.Adapter())
995     {
996     }
997 
998     ~FinalizeRegisterInfo() = default;
ClearInfo()999     void ClearInfo()
1000     {
1001         memOperandIdx = 0;
1002         baseOperand = nullptr;
1003         offsetOperand = nullptr;
1004         defOperands.clear();
1005         defIdx.clear();
1006         useOperands.clear();
1007         useIdx.clear();
1008     }
1009 
SetBaseOperand(Operand & opnd,const int32 idx)1010     void SetBaseOperand(Operand &opnd, const int32 idx)
1011     {
1012         baseOperand = &opnd;
1013         memOperandIdx = idx;
1014     }
1015 
SetOffsetOperand(Operand & opnd)1016     void SetOffsetOperand(Operand &opnd)
1017     {
1018         offsetOperand = &opnd;
1019     }
1020 
SetDefOperand(Operand & opnd,const int32 idx)1021     void SetDefOperand(Operand &opnd, const int32 idx)
1022     {
1023         defOperands.emplace_back(&opnd);
1024         defIdx.emplace_back(idx);
1025     }
1026 
SetUseOperand(Operand & opnd,const int32 idx)1027     void SetUseOperand(Operand &opnd, const int32 idx)
1028     {
1029         useOperands.emplace_back(&opnd);
1030         useIdx.emplace_back(idx);
1031     }
1032 
GetMemOperandIdx()1033     int32 GetMemOperandIdx() const
1034     {
1035         return memOperandIdx;
1036     }
1037 
GetBaseOperand()1038     const Operand *GetBaseOperand() const
1039     {
1040         return baseOperand;
1041     }
1042 
GetOffsetOperand()1043     const Operand *GetOffsetOperand() const
1044     {
1045         return offsetOperand;
1046     }
1047 
GetDefOperandsSize()1048     size_t GetDefOperandsSize() const
1049     {
1050         return defOperands.size();
1051     }
1052 
GetDefOperandsElem(size_t index)1053     const Operand *GetDefOperandsElem(size_t index) const
1054     {
1055         return defOperands[index];
1056     }
1057 
GetDefIdxElem(size_t index)1058     int32 GetDefIdxElem(size_t index) const
1059     {
1060         return defIdx[index];
1061     }
1062 
GetUseOperandsSize()1063     size_t GetUseOperandsSize() const
1064     {
1065         return useOperands.size();
1066     }
1067 
GetUseOperandsElem(size_t index)1068     const Operand *GetUseOperandsElem(size_t index) const
1069     {
1070         return useOperands[index];
1071     }
1072 
GetUseIdxElem(size_t index)1073     int32 GetUseIdxElem(size_t index) const
1074     {
1075         return useIdx[index];
1076     }
1077 
1078 private:
1079     int32 memOperandIdx = 0;
1080     Operand *baseOperand = nullptr;
1081     Operand *offsetOperand = nullptr;
1082     MapleVector<Operand *> defOperands;
1083     MapleVector<int32> defIdx;
1084     MapleVector<Operand *> useOperands;
1085     MapleVector<int32> useIdx;
1086 };
1087 
1088 class LocalRegAllocator {
1089 public:
LocalRegAllocator(CGFunc & cgFunc,MapleAllocator & allocator)1090     LocalRegAllocator(CGFunc &cgFunc, MapleAllocator &allocator)
1091         : intRegAssignmentMap(allocator.Adapter()),
1092           fpRegAssignmentMap(allocator.Adapter()),
1093           useInfo(allocator.Adapter()),
1094           defInfo(allocator.Adapter())
1095     {
1096         buckets = (cgFunc.GetMaxRegNum() / kU64) + 1;
1097         intRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1098         fpRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1099         intRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1100         fpRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1101     }
1102 
1103     ~LocalRegAllocator() = default;
1104 
ClearLocalRaInfo()1105     void ClearLocalRaInfo()
1106     {
1107         ClearBitArrElement(intRegAssigned);
1108         ClearBitArrElement(fpRegAssigned);
1109         intRegAssignmentMap.clear();
1110         fpRegAssignmentMap.clear();
1111         intPregUsed = 0;
1112         fpPregUsed = 0;
1113         ClearBitArrElement(intRegSpilled);
1114         ClearBitArrElement(fpRegSpilled);
1115         numIntPregUsed = 0;
1116         numFpPregUsed = 0;
1117     }
1118 
RegBaseUpdate(bool isInt)1119     regno_t RegBaseUpdate(bool isInt) const
1120     {
1121         return isInt ? 0 : V0 - R0;
1122     }
1123 
IsInRegAssigned(regno_t regNO,bool isInt)1124     bool IsInRegAssigned(regno_t regNO, bool isInt) const
1125     {
1126         uint64 *regAssigned = nullptr;
1127         if (isInt) {
1128             regAssigned = intRegAssigned;
1129         } else {
1130             regAssigned = fpRegAssigned;
1131         }
1132         return IsBitArrElemSet(regAssigned, regNO);
1133         ;
1134     }
1135 
SetRegAssigned(regno_t regNO,bool isInt)1136     void SetRegAssigned(regno_t regNO, bool isInt) const
1137     {
1138         if (isInt) {
1139             SetBitArrElement(intRegAssigned, regNO);
1140         } else {
1141             SetBitArrElement(fpRegAssigned, regNO);
1142         }
1143     }
1144 
GetRegAssignmentItem(bool isInt,regno_t regKey)1145     regno_t GetRegAssignmentItem(bool isInt, regno_t regKey)
1146     {
1147         return isInt ? intRegAssignmentMap[regKey] : fpRegAssignmentMap[regKey];
1148     }
1149 
SetRegAssignmentMap(bool isInt,regno_t regKey,regno_t regValue)1150     void SetRegAssignmentMap(bool isInt, regno_t regKey, regno_t regValue)
1151     {
1152         if (isInt) {
1153             intRegAssignmentMap[regKey] = regValue;
1154         } else {
1155             fpRegAssignmentMap[regKey] = regValue;
1156         }
1157     }
1158 
1159     /* only for HandleLocalRaDebug */
GetPregUsed(bool isInt)1160     uint64 GetPregUsed(bool isInt) const
1161     {
1162         if (isInt) {
1163             return intPregUsed;
1164         } else {
1165             return fpPregUsed;
1166         }
1167     }
1168 
SetPregUsed(regno_t regNO,bool isInt)1169     void SetPregUsed(regno_t regNO, bool isInt)
1170     {
1171         uint64 mask = 0;
1172         if (isInt) {
1173             mask = 1ULL << (regNO - R0);
1174             if ((intPregUsed & mask) == 0) {
1175                 ++numIntPregUsed;
1176                 intPregUsed |= mask;
1177             }
1178         } else {
1179             mask = 1ULL << (regNO - V0);
1180             if ((fpPregUsed & mask) == 0) {
1181                 ++numFpPregUsed;
1182                 fpPregUsed |= mask;
1183             }
1184         }
1185     }
1186 
isInRegSpilled(regno_t regNO,bool isInt)1187     bool isInRegSpilled(regno_t regNO, bool isInt) const
1188     {
1189         bool isSet;
1190         if (isInt) {
1191             isSet = IsBitArrElemSet(intRegSpilled, regNO);
1192         } else {
1193             isSet = IsBitArrElemSet(fpRegSpilled, regNO);
1194         }
1195         return isSet;
1196     }
1197 
SetRegSpilled(regno_t regNO,bool isInt)1198     void SetRegSpilled(regno_t regNO, bool isInt) const
1199     {
1200         if (isInt) {
1201             SetBitArrElement(intRegSpilled, regNO);
1202         } else {
1203             SetBitArrElement(fpRegSpilled, regNO);
1204         }
1205     }
1206 
GetPregs(bool isInt)1207     uint64 GetPregs(bool isInt) const
1208     {
1209         if (isInt) {
1210             return intPregs;
1211         } else {
1212             return fpPregs;
1213         }
1214     }
1215 
SetPregs(regno_t regNO,bool isInt)1216     void SetPregs(regno_t regNO, bool isInt)
1217     {
1218         if (isInt) {
1219             intPregs |= 1ULL << (regNO - RegBaseUpdate(true));
1220         } else {
1221             fpPregs |= 1ULL << (regNO - RegBaseUpdate(false));
1222         }
1223     }
1224 
ClearPregs(regno_t regNO,bool isInt)1225     void ClearPregs(regno_t regNO, bool isInt)
1226     {
1227         if (isInt) {
1228             intPregs &= ~(1ULL << (regNO - RegBaseUpdate(true)));
1229         } else {
1230             fpPregs &= ~(1ULL << (regNO - RegBaseUpdate(false)));
1231         }
1232     }
1233 
IsPregAvailable(regno_t regNO,bool isInt)1234     bool IsPregAvailable(regno_t regNO, bool isInt) const
1235     {
1236         bool isAvailable;
1237         if (isInt) {
1238             isAvailable = intPregs & (1ULL << (regNO - RegBaseUpdate(true)));
1239         } else {
1240             isAvailable = fpPregs & (1ULL << (regNO - RegBaseUpdate(false)));
1241         }
1242         return isAvailable;
1243     }
1244 
InitPregs(uint32 intMax,uint32 fpMax,bool hasYield,const MapleSet<uint32> & intSpillRegSet,const MapleSet<uint32> & fpSpillRegSet)1245     void InitPregs(uint32 intMax, uint32 fpMax, bool hasYield, const MapleSet<uint32> &intSpillRegSet,
1246                    const MapleSet<uint32> &fpSpillRegSet)
1247     {
1248         uint32 intBase = R0;
1249         uint32 fpBase = V0;
1250         intPregs = (1ULL << (intMax + 1)) - 1;
1251         fpPregs = (1ULL << (((fpMax + 1) + fpBase) - RegBaseUpdate(false))) - 1;
1252         for (uint32 regNO : intSpillRegSet) {
1253             ClearPregs(regNO + intBase, true);
1254         }
1255         for (uint32 regNO : fpSpillRegSet) {
1256             ClearPregs(regNO + fpBase, false);
1257         }
1258         if (hasYield) {
1259             ClearPregs(RYP, true);
1260         }
1261 #ifdef RESERVED_REGS
1262         intPregs &= ~(1ULL << R16);
1263         intPregs &= ~(1ULL << R17);
1264 #endif /* RESERVED_REGS */
1265     }
1266 
GetIntRegAssignmentMap()1267     const MapleMap<regno_t, regno_t> &GetIntRegAssignmentMap() const
1268     {
1269         return intRegAssignmentMap;
1270     }
1271 
GetFpRegAssignmentMap()1272     const MapleMap<regno_t, regno_t> &GetFpRegAssignmentMap() const
1273     {
1274         return fpRegAssignmentMap;
1275     }
1276 
GetUseInfo()1277     const MapleMap<regno_t, uint16> &GetUseInfo() const
1278     {
1279         return useInfo;
1280     }
1281 
SetUseInfoElem(regno_t regNO,uint16 info)1282     void SetUseInfoElem(regno_t regNO, uint16 info)
1283     {
1284         useInfo[regNO] = info;
1285     }
1286 
IncUseInfoElem(regno_t regNO)1287     void IncUseInfoElem(regno_t regNO)
1288     {
1289         if (useInfo.find(regNO) != useInfo.end()) {
1290             ++useInfo[regNO];
1291         }
1292     }
1293 
GetUseInfoElem(regno_t regNO)1294     uint16 GetUseInfoElem(regno_t regNO)
1295     {
1296         return useInfo[regNO];
1297     }
1298 
ClearUseInfo()1299     void ClearUseInfo()
1300     {
1301         useInfo.clear();
1302     }
1303 
GetDefInfo()1304     const MapleMap<regno_t, uint16> &GetDefInfo() const
1305     {
1306         return defInfo;
1307     }
1308 
SetDefInfoElem(regno_t regNO,uint16 info)1309     void SetDefInfoElem(regno_t regNO, uint16 info)
1310     {
1311         defInfo[regNO] = info;
1312     }
1313 
GetDefInfoElem(regno_t regNO)1314     uint16 GetDefInfoElem(regno_t regNO)
1315     {
1316         return defInfo[regNO];
1317     }
1318 
IncDefInfoElem(regno_t regNO)1319     void IncDefInfoElem(regno_t regNO)
1320     {
1321         if (defInfo.find(regNO) != defInfo.end()) {
1322             ++defInfo[regNO];
1323         }
1324     }
1325 
ClearDefInfo()1326     void ClearDefInfo()
1327     {
1328         defInfo.clear();
1329     }
1330 
GetNumIntPregUsed()1331     uint32 GetNumIntPregUsed() const
1332     {
1333         return numIntPregUsed;
1334     }
1335 
GetNumFpPregUsed()1336     uint32 GetNumFpPregUsed() const
1337     {
1338         return numFpPregUsed;
1339     }
1340 
1341 private:
ClearBitArrElement(uint64 * vec)1342     void ClearBitArrElement(uint64 *vec) const
1343     {
1344         for (uint32 i = 0; i < buckets; ++i) {
1345             vec[i] = 0UL;
1346         }
1347     }
1348 
SetBitArrElement(uint64 * vec,regno_t regNO)1349     void SetBitArrElement(uint64 *vec, regno_t regNO) const
1350     {
1351         uint32 index = regNO / kU64;
1352         uint64 bit = regNO % kU64;
1353         vec[index] |= 1ULL << bit;
1354     }
1355 
1356     /* The following local vars keeps track of allocation information in bb. */
1357     uint64 *intRegAssigned; /* in this set if vreg is assigned */
1358     uint64 *fpRegAssigned;
1359     MapleMap<regno_t, regno_t> intRegAssignmentMap; /* vreg -> preg map, which preg is the vreg assigned */
1360     MapleMap<regno_t, regno_t> fpRegAssignmentMap;
1361     uint64 intPregUsed = 0; /* pregs used in bb */
1362     uint64 fpPregUsed = 0;
1363     uint64 *intRegSpilled; /* on this list if vreg is spilled */
1364     uint64 *fpRegSpilled;
1365 
1366     uint64 intPregs = 0; /* available regs for assignement */
1367     uint64 fpPregs = 0;
1368     MapleMap<regno_t, uint16> useInfo; /* copy of local ra info for useCnt */
1369     MapleMap<regno_t, uint16> defInfo; /* copy of local ra info for defCnt */
1370 
1371     uint32 numIntPregUsed = 0;
1372     uint32 numFpPregUsed = 0;
1373     uint32 buckets;
1374 };
1375 
1376 class SplitBBInfo {
1377 public:
1378     SplitBBInfo() = default;
1379 
1380     ~SplitBBInfo() = default;
1381 
GetCandidateBB()1382     BB *GetCandidateBB()
1383     {
1384         return candidateBB;
1385     }
1386 
GetCandidateBB()1387     const BB *GetCandidateBB() const
1388     {
1389         return candidateBB;
1390     }
1391 
GetStartBB()1392     const BB *GetStartBB() const
1393     {
1394         return startBB;
1395     }
1396 
SetCandidateBB(BB & bb)1397     void SetCandidateBB(BB &bb)
1398     {
1399         candidateBB = &bb;
1400     }
1401 
SetStartBB(BB & bb)1402     void SetStartBB(BB &bb)
1403     {
1404         startBB = &bb;
1405     }
1406 
1407 private:
1408     BB *candidateBB = nullptr;
1409     BB *startBB = nullptr;
1410 };
1411 
1412 class GraphColorRegAllocator : public RegAllocator {
1413 public:
GraphColorRegAllocator(CGFunc & cgFunc,MemPool & memPool,DomAnalysis & dom)1414     GraphColorRegAllocator(CGFunc &cgFunc, MemPool &memPool, DomAnalysis &dom)
1415         : RegAllocator(cgFunc, memPool),
1416           domInfo(dom),
1417           bbVec(alloc.Adapter()),
1418           vregLive(alloc.Adapter()),
1419           pregLive(alloc.Adapter()),
1420           lrMap(alloc.Adapter()),
1421           localRegVec(alloc.Adapter()),
1422           bbRegInfo(alloc.Adapter()),
1423           unconstrained(alloc.Adapter()),
1424           unconstrainedPref(alloc.Adapter()),
1425           constrained(alloc.Adapter()),
1426           mustAssigned(alloc.Adapter()),
1427 #ifdef OPTIMIZE_FOR_PROLOG
1428           intDelayed(alloc.Adapter()),
1429           fpDelayed(alloc.Adapter()),
1430 #endif /* OPTIMIZE_FOR_PROLOG */
1431           intCallerRegSet(alloc.Adapter()),
1432           intCalleeRegSet(alloc.Adapter()),
1433           intSpillRegSet(alloc.Adapter()),
1434           fpCallerRegSet(alloc.Adapter()),
1435           fpCalleeRegSet(alloc.Adapter()),
1436           fpSpillRegSet(alloc.Adapter()),
1437           intCalleeUsed(alloc.Adapter()),
1438           fpCalleeUsed(alloc.Adapter())
1439     {
1440         constexpr uint32 kNumInsnThreashold = 30000;
1441         numVregs = cgFunc.GetMaxVReg();
1442         localRegVec.resize(cgFunc.NumBBs());
1443         bbRegInfo.resize(cgFunc.NumBBs());
1444         if (CGOptions::DoMultiPassColorRA() && cgFunc.GetMirModule().IsCModule()) {
1445             uint32 cnt = 0;
1446             FOR_ALL_BB(bb, &cgFunc) {
1447                 FOR_BB_INSNS(insn, bb) {
1448                     ++cnt;
1449                 }
1450             }
1451             DEBUG_ASSERT(cnt <= cgFunc.GetTotalNumberOfInstructions(), "Incorrect insn count");
1452             if (cnt <= kNumInsnThreashold) {
1453                 doMultiPass = true;
1454                 doLRA = false;
1455                 doOptProlog = false;
1456             }
1457         }
1458     }
1459 
1460     ~GraphColorRegAllocator() override = default;
1461 
1462     bool AllocateRegisters() override;
1463 
1464     enum SpillMemCheck : uint8 {
1465         kSpillMemPre,
1466         kSpillMemPost,
1467     };
1468 
GetLiveRange(regno_t regNO)1469     LiveRange *GetLiveRange(regno_t regNO)
1470     {
1471         auto it = lrMap.find(regNO);
1472         if (it != lrMap.end()) {
1473             return it->second;
1474         } else {
1475             return nullptr;
1476         }
1477     }
GetLiveRange(regno_t regNO)1478     LiveRange *GetLiveRange(regno_t regNO) const
1479     {
1480         auto it = lrMap.find(regNO);
1481         if (it != lrMap.end()) {
1482             return it->second;
1483         } else {
1484             return nullptr;
1485         }
1486     }
GetLrMap()1487     const MapleMap<regno_t, LiveRange *> &GetLrMap() const
1488     {
1489         return lrMap;
1490     }
1491     Insn *SpillOperand(Insn &insn, const Operand &opnd, bool isDef, RegOperand &phyOpnd, bool forCall = false);
1492 
1493 private:
1494     struct SetLiveRangeCmpFunc {
operatorSetLiveRangeCmpFunc1495         bool operator()(const LiveRange *lhs, const LiveRange *rhs) const
1496         {
1497             if (fabs(lhs->GetPriority() - rhs->GetPriority()) <= 1e-6) {
1498                 /*
1499                  * This is to ensure the ordering is consistent as the reg#
1500                  * differs going through VtableImpl.mpl file.
1501                  */
1502                 if (lhs->GetID() == rhs->GetID()) {
1503                     return lhs->GetRegNO() < rhs->GetRegNO();
1504                 } else {
1505                     return lhs->GetID() < rhs->GetID();
1506                 }
1507             }
1508             return (lhs->GetPriority() > rhs->GetPriority());
1509         }
1510     };
1511 
1512     template <typename Func>
1513     void ForEachBBArrElem(const uint64 *vec, Func functor) const;
1514 
1515     template <typename Func>
1516     void ForEachBBArrElemWithInterrupt(const uint64 *vec, Func functor) const;
1517 
1518     template <typename Func>
1519     void ForEachRegArrElem(const uint64 *vec, Func functor) const;
1520 
1521     void PrintLiveUnitMap(const LiveRange &lr) const;
1522     void PrintLiveRangeConflicts(const LiveRange &lr) const;
1523     void PrintLiveBBBit(const LiveRange &li) const;
1524     void PrintLiveRange(const LiveRange &li, const std::string &str) const;
1525     void PrintLiveRanges() const;
1526     void PrintLocalRAInfo(const std::string &str) const;
1527     void PrintBBAssignInfo() const;
1528     void PrintBBs() const;
1529 
1530     uint32 MaxIntPhysRegNum() const;
1531     uint32 MaxFloatPhysRegNum() const;
1532     bool IsReservedReg(AArch64reg regNO) const;
1533     void InitFreeRegPool();
1534     void InitCCReg();
1535     bool IsYieldPointReg(regno_t regNO) const;
1536     bool IsUnconcernedReg(regno_t regNO) const;
1537     bool IsUnconcernedReg(const RegOperand &regOpnd) const;
1538     LiveRange *NewLiveRange();
1539     void CalculatePriority(LiveRange &lr) const;
1540     bool CreateLiveRangeHandleLocal(regno_t regNO, const BB &bb, bool isDef);
1541     LiveRange *CreateLiveRangeAllocateAndUpdate(regno_t regNO, const BB &bb, bool isDef, uint32 currId);
1542     void CreateLiveRange(regno_t regNO, const BB &bb, bool isDef, uint32 currPoint, bool updateCount);
1543     bool SetupLiveRangeByOpHandlePhysicalReg(const RegOperand &op, Insn &insn, regno_t regNO, bool isDef);
1544     void SetupLiveRangeByOp(Operand &op, Insn &insn, bool isDef, uint32 &numUses);
1545     void SetupLiveRangeByRegNO(regno_t liveOut, BB &bb, uint32 currPoint);
1546     bool UpdateInsnCntAndSkipUseless(Insn &insn, uint32 &currPoint) const;
1547     void UpdateCallInfo(uint32 bbId, uint32 currPoint, const Insn &insn);
1548     void ClassifyOperand(std::unordered_set<regno_t> &pregs, std::unordered_set<regno_t> &vregs,
1549                          const Operand &opnd) const;
1550     void SetOpndConflict(const Insn &insn, bool onlyDef);
1551     void UpdateOpndConflict(const Insn &insn, bool multiDef);
1552     void SetLrMustAssign(const RegOperand *regOpnd);
1553     void SetupMustAssignedLiveRanges(const Insn &insn);
1554     void ComputeLiveRangesForEachDefOperand(Insn &insn, bool &multiDef);
1555     void ComputeLiveRangesForEachUseOperand(Insn &insn);
1556     void ComputeLiveRangesUpdateIfInsnIsCall(const Insn &insn);
1557     void ComputeLiveRangesUpdateLiveUnitInsnRange(BB &bb, uint32 currPoint);
1558     void ComputeLiveRanges();
1559     MemOperand *CreateSpillMem(uint32 spillIdx, SpillMemCheck check);
1560     bool CheckOverlap(uint64 val, uint32 i, LiveRange &lr1, LiveRange &lr2) const;
1561     void CheckInterference(LiveRange &lr1, LiveRange &lr2) const;
1562     void BuildInterferenceGraphSeparateIntFp(std::vector<LiveRange *> &intLrVec, std::vector<LiveRange *> &fpLrVec);
1563     void BuildInterferenceGraph();
1564     void SetBBInfoGlobalAssigned(uint32 bbID, regno_t regNO);
1565     bool HaveAvailableColor(const LiveRange &lr, uint32 num) const;
1566     void Separate();
1567     void SplitAndColorForEachLr(MapleVector<LiveRange *> &targetLrVec);
1568     void SplitAndColor();
1569     void ColorForOptPrologEpilog();
1570     bool IsLocalReg(regno_t regNO) const;
1571     bool IsLocalReg(const LiveRange &lr) const;
1572     void HandleLocalRaDebug(regno_t regNO, const LocalRegAllocator &localRa, bool isInt) const;
1573     void HandleLocalRegAssignment(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1574     void UpdateLocalRegDefUseCount(regno_t regNO, LocalRegAllocator &localRa, bool isDef, bool isInt) const;
1575     void UpdateLocalRegConflict(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1576     void HandleLocalReg(Operand &op, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo, bool isDef, bool isInt);
1577     void LocalRaRegSetEraseReg(LocalRegAllocator &localRa, regno_t regNO) const;
1578     bool LocalRaInitRegSet(LocalRegAllocator &localRa, uint32 bbId);
1579     void LocalRaInitAllocatableRegs(LocalRegAllocator &localRa, uint32 bbId);
1580     void LocalRaForEachDefOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1581     void LocalRaForEachUseOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1582     void LocalRaPrepareBB(BB &bb, LocalRegAllocator &localRa);
1583     void LocalRaFinalAssignment(const LocalRegAllocator &localRa, BBAssignInfo &bbInfo);
1584     void LocalRaDebug(const BB &bb, const LocalRegAllocator &localRa) const;
1585     void LocalRegisterAllocator(bool allocate);
1586     MemOperand *GetSpillOrReuseMem(LiveRange &lr, uint32 regSize, bool &isOutOfRange, Insn &insn, bool isDef);
1587     void SpillOperandForSpillPre(Insn &insn, const Operand &opnd, RegOperand &phyOpnd, uint32 spillIdx, bool needSpill);
1588     void SpillOperandForSpillPost(Insn &insn, const Operand &opnd, RegOperand &phyOpnd, uint32 spillIdx,
1589                                   bool needSpill);
1590     MemOperand *GetConsistentReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1591                                       RegType regType);
1592     MemOperand *GetCommonReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1593                                   RegType regType);
1594     MemOperand *GetReuseMem(uint32 vregNO, uint32 size, RegType regType);
1595     MemOperand *GetSpillMem(uint32 vregNO, bool isDest, Insn &insn, AArch64reg regNO, bool &isOutOfRange) const;
1596     bool SetAvailableSpillReg(std::unordered_set<regno_t> &cannotUseReg, LiveRange &lr, uint64 &usedRegMask);
1597     void CollectCannotUseReg(std::unordered_set<regno_t> &cannotUseReg, const LiveRange &lr, Insn &insn);
1598     regno_t PickRegForSpill(uint64 &usedRegMask, RegType regType, uint32 spillIdx, bool &needSpillLr);
1599     bool SetRegForSpill(LiveRange &lr, Insn &insn, uint32 spillIdx, uint64 &usedRegMask, bool isDef);
1600     bool GetSpillReg(Insn &insn, LiveRange &lr, const uint32 &spillIdx, uint64 &usedRegMask, bool isDef);
1601     RegOperand *GetReplaceOpndForLRA(Insn &insn, const Operand &opnd, uint32 &spillIdx, uint64 &usedRegMask,
1602                                      bool isDef);
1603     bool EncountPrevRef(const BB &pred, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1604     bool FoundPrevBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1605     bool EncountNextRef(const BB &succ, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1606     bool FoundNextBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1607     bool HavePrevRefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1608     bool HaveNextDefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1609     bool NeedCallerSave(Insn &insn, LiveRange &lr, bool isDef);
1610     RegOperand *GetReplaceOpnd(Insn &insn, const Operand &opnd, uint32 &spillIdx, uint64 &usedRegMask, bool isDef);
1611     void MarkCalleeSaveRegs();
1612     void MarkUsedRegs(Operand &opnd, uint64 &usedRegMask);
1613     uint64 FinalizeRegisterPreprocess(FinalizeRegisterInfo &fInfo, const Insn &insn, bool &needProcess);
1614     void SplitVregAroundLoop(const CGFuncLoops &loop, const std::vector<LiveRange *> &lrs, BB &headerPred, BB &exitSucc,
1615                              const std::set<regno_t> &cands);
1616     bool LoopNeedSplit(const CGFuncLoops &loop, std::set<regno_t> &cands);
1617     bool LrGetBadReg(const LiveRange &lr) const;
1618     void AnalysisLoopPressureAndSplit(const CGFuncLoops &loop);
1619     void AnalysisLoop(const CGFuncLoops &);
1620     void OptCallerSave();
1621     void FinalizeRegisters();
1622     void GenerateSpillFillRegs(const Insn &insn);
1623     RegOperand *CreateSpillFillCode(const RegOperand &opnd, Insn &insn, uint32 spillCnt, bool isdef = false);
1624     bool SpillLiveRangeForSpills();
1625 
1626     MapleVector<LiveRange *>::iterator GetHighPriorityLr(MapleVector<LiveRange *> &lrSet) const;
1627     void UpdateForbiddenForNeighbors(const LiveRange &lr) const;
1628     void UpdatePregvetoForNeighbors(const LiveRange &lr) const;
1629     regno_t FindColorForLr(const LiveRange &lr) const;
1630     regno_t TryToAssignCallerSave(const LiveRange &lr) const;
1631     bool ShouldUseCallee(LiveRange &lr, const MapleSet<regno_t> &calleeUsed,
1632                          const MapleVector<LiveRange *> &delayed) const;
1633     void AddCalleeUsed(regno_t regNO, RegType regType);
1634     bool AssignColorToLr(LiveRange &lr, bool isDelayed = false);
1635     void PruneLrForSplit(LiveRange &lr, BB &bb, bool remove, std::set<CGFuncLoops *, CGFuncLoopCmp> &candidateInLoop,
1636                          std::set<CGFuncLoops *, CGFuncLoopCmp> &defInLoop);
1637     bool UseIsUncovered(const BB &bb, const BB &startBB, std::vector<bool> &visitedBB);
1638     void FindUseForSplit(LiveRange &lr, SplitBBInfo &bbInfo, bool &remove,
1639                          std::set<CGFuncLoops *, CGFuncLoopCmp> &candidateInLoop,
1640                          std::set<CGFuncLoops *, CGFuncLoopCmp> &defInLoop);
1641     void FindBBSharedInSplit(LiveRange &lr, const std::set<CGFuncLoops *, CGFuncLoopCmp> &candidateInLoop,
1642                              std::set<CGFuncLoops *, CGFuncLoopCmp> &defInLoop);
1643     void ComputeBBForNewSplit(LiveRange &newLr, LiveRange &oldLr);
1644     void ClearLrBBFlags(const std::set<BB *, SortedBBCmpFunc> &member) const;
1645     void ComputeBBForOldSplit(LiveRange &newLr, LiveRange &oldLr);
1646     bool LrCanBeColored(const LiveRange &lr, const BB &bbAdded, std::unordered_set<regno_t> &conflictRegs);
1647     void MoveLrBBInfo(LiveRange &oldLr, LiveRange &newLr, BB &bb) const;
1648     bool ContainsLoop(const CGFuncLoops &loop, const std::set<CGFuncLoops *, CGFuncLoopCmp> &loops) const;
1649     void GetAllLrMemberLoops(LiveRange &lr, std::set<CGFuncLoops *, CGFuncLoopCmp> &loop);
1650     bool SplitLrShouldSplit(LiveRange &lr);
1651     bool SplitLrFindCandidateLr(LiveRange &lr, LiveRange &newLr, std::unordered_set<regno_t> &conflictRegs);
1652     void SplitLrHandleLoops(LiveRange &lr, LiveRange &newLr, const std::set<CGFuncLoops *, CGFuncLoopCmp> &oldLoops,
1653                             const std::set<CGFuncLoops *, CGFuncLoopCmp> &newLoops);
1654     void SplitLrFixNewLrCallsAndRlod(LiveRange &newLr, const std::set<CGFuncLoops *, CGFuncLoopCmp> &origLoops);
1655     void SplitLrFixOrigLrCalls(LiveRange &lr) const;
1656     void SplitLrUpdateInterference(LiveRange &lr);
1657     void SplitLrUpdateRegInfo(const LiveRange &origLr, LiveRange &newLr,
1658                               std::unordered_set<regno_t> &conflictRegs) const;
1659     void SplitLrErrorCheckAndDebug(const LiveRange &origLr) const;
1660     void SplitLr(LiveRange &lr);
1661 
1662     static constexpr uint16 kMaxUint16 = 0x7fff;
1663 
1664     DomAnalysis &domInfo;
1665     MapleVector<BB *> bbVec;
1666     MapleUnorderedSet<regno_t> vregLive;
1667     MapleUnorderedSet<regno_t> pregLive;
1668     MapleMap<regno_t, LiveRange *> lrMap;
1669     MapleVector<LocalRaInfo *> localRegVec; /* local reg info for each bb, no local reg if null */
1670     MapleVector<BBAssignInfo *> bbRegInfo;  /* register assignment info for each bb */
1671     MapleVector<LiveRange *> unconstrained;
1672     MapleVector<LiveRange *> unconstrainedPref;
1673     MapleVector<LiveRange *> constrained;
1674     MapleVector<LiveRange *> mustAssigned;
1675 #ifdef OPTIMIZE_FOR_PROLOG
1676     MapleVector<LiveRange *> intDelayed;
1677     MapleVector<LiveRange *> fpDelayed;
1678 #endif                                /* OPTIMIZE_FOR_PROLOG  */
1679     MapleSet<uint32> intCallerRegSet; /* integer caller saved */
1680     MapleSet<uint32> intCalleeRegSet; /*         callee       */
1681     MapleSet<uint32> intSpillRegSet;  /*         spill        */
1682     MapleSet<uint32> fpCallerRegSet;  /* float caller saved   */
1683     MapleSet<uint32> fpCalleeRegSet;  /*       callee         */
1684     MapleSet<uint32> fpSpillRegSet;   /*       spill          */
1685     MapleSet<regno_t> intCalleeUsed;
1686     MapleSet<regno_t> fpCalleeUsed;
1687     Bfs *bfs = nullptr;
1688 
1689     uint32 bbBuckets = 0;  /* size of bit array for bb (each bucket == 64 bits) */
1690     uint32 regBuckets = 0; /* size of bit array for reg (each bucket == 64 bits) */
1691     uint32 intRegNum = 0;  /* total available int preg */
1692     uint32 fpRegNum = 0;   /* total available fp preg */
1693     uint32 numVregs = 0;   /* number of vregs when starting */
1694     regno_t ccReg = 0;
1695     /* For spilling of spill register if there are none available
1696      *   Example, all 3 operands spilled
1697      *                          sp_reg1 -> [spillMemOpnds[1]]
1698      *                          sp_reg2 -> [spillMemOpnds[2]]
1699      *                          ld sp_reg1 <- [addr-reg2]
1700      *                          ld sp_reg2 <- [addr-reg3]
1701      *   reg1 <- reg2, reg3     sp_reg1 <- sp_reg1, sp_reg2
1702      *                          st sp_reg1 -> [addr-reg1]
1703      *                          sp_reg1 <- [spillMemOpnds[1]]
1704      *                          sp_reg2 <- [spillMemOpnds[2]]
1705      */
1706     static constexpr size_t kSpillMemOpndNum = 4;
1707     std::array<MemOperand *, kSpillMemOpndNum> spillMemOpnds = {nullptr};
1708     bool operandSpilled[kSpillMemOpndNum];
1709     bool needExtraSpillReg = false;
1710 #ifdef USE_LRA
1711     bool doLRA = true;
1712 #else
1713     bool doLRA = false;
1714 #endif
1715 #ifdef OPTIMIZE_FOR_PROLOG
1716     bool doOptProlog = true;
1717 #else
1718     bool doOptProlog = false;
1719 #endif
1720     bool hasSpill = false;
1721     bool doMultiPass = false;
1722     bool seenFP = false;
1723 };
1724 
1725 class CallerSavePre : public CGPre {
1726 public:
CallerSavePre(GraphColorRegAllocator * regAlloc,CGFunc & cgfunc,DomAnalysis & currDom,MemPool & memPool,MemPool & mp2,PreKind kind,uint32 limit)1727     CallerSavePre(GraphColorRegAllocator *regAlloc, CGFunc &cgfunc, DomAnalysis &currDom, MemPool &memPool,
1728                   MemPool &mp2, PreKind kind, uint32 limit)
1729         : CGPre(currDom, memPool, mp2, kind, limit),
1730           func(&cgfunc),
1731           regAllocator(regAlloc),
1732           loopHeadBBs(ssaPreAllocator.Adapter())
1733     {
1734     }
1735 
1736     ~CallerSavePre() = default;
1737 
1738     void ApplySSAPRE();
SetDump(bool val)1739     void SetDump(bool val)
1740     {
1741         dump = val;
1742     }
1743 
1744 private:
1745     void CodeMotion();
1746     void UpdateLoadSite(CgOccur *occ);
1747     void CalLoadSites();
1748     void ComputeAvail();
1749     void Rename1();
1750     void ComputeVarAndDfPhis() override;
1751     void BuildWorkList() override;
1752     void DumpWorkCandAndOcc();
1753 
GetBB(uint32 id)1754     BB *GetBB(uint32 id) const override
1755     {
1756         return func->GetBBFromID(id);
1757     }
1758 
GetPUIdx()1759     PUIdx GetPUIdx() const override
1760     {
1761         return func->GetFunction().GetPuidx();
1762     }
1763 
IsLoopHeadBB(uint32 bbId)1764     bool IsLoopHeadBB(uint32 bbId) const
1765     {
1766         return loopHeadBBs.find(bbId) != loopHeadBBs.end();
1767     }
1768     CGFunc *func;
1769     bool dump = false;
1770     LiveRange *workLr = nullptr;
1771     GraphColorRegAllocator *regAllocator;
1772     MapleSet<uint32> loopHeadBBs;
1773     bool beyondLimit = false;
1774 };
1775 } /* namespace maplebe */
1776 
1777 #endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H */
1778