1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
17 #define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
18 #include "reg_alloc.h"
19 #include "aarch64_operand.h"
20 #include "aarch64_insn.h"
21 #include "aarch64_abi.h"
22 #include "aarch64_cgfunc.h"
23 #include "loop.h"
24 #include "cg_dominance.h"
25 #include "cg_pre.h"
26
27 namespace maplebe {
28 #define RESERVED_REGS
29
30 #define USE_LRA
31 #define USE_SPLIT
32 #undef USE_BB_FREQUENCY
33 #define OPTIMIZE_FOR_PROLOG
34 #define REUSE_SPILLMEM
35 #undef COLOR_SPLIT
36 #define MOVE_COALESCE
37
38 /* for robust test */
39 #undef CONSISTENT_MEMOPND
40 #undef RANDOM_PRIORITY
41
42 constexpr uint32 k32 = sizeof(int) * CHAR_BIT;
43 constexpr uint32 k64 = sizeof(int64) * CHAR_BIT;
44 constexpr uint32 kU64 = sizeof(uint64) * CHAR_BIT;
45
46 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::set<T,Comparator> & set,const T & item)47 inline bool FindNotIn(const std::set<T, Comparator> &set, const T &item)
48 {
49 return set.find(item) == set.end();
50 }
51
52 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::unordered_set<T,Comparator> & set,const T & item)53 inline bool FindNotIn(const std::unordered_set<T, Comparator> &set, const T &item)
54 {
55 return set.find(item) == set.end();
56 }
57
58 template <typename T>
FindNotIn(const MapleSet<T> & set,const T & item)59 inline bool FindNotIn(const MapleSet<T> &set, const T &item)
60 {
61 return set.find(item) == set.end();
62 }
63
64 template <typename T>
FindNotIn(const MapleUnorderedSet<T> & set,const T & item)65 inline bool FindNotIn(const MapleUnorderedSet<T> &set, const T &item)
66 {
67 return set.find(item) == set.end();
68 }
69
70 template <typename T>
FindNotIn(const MapleList<T> & list,const T & item)71 inline bool FindNotIn(const MapleList<T> &list, const T &item)
72 {
73 return std::find(list.begin(), list.end(), item) == list.end();
74 }
75
76 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::set<T,Comparator> & set,const T & item)77 inline bool FindIn(const std::set<T, Comparator> &set, const T &item)
78 {
79 return set.find(item) != set.end();
80 }
81
82 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::unordered_set<T,Comparator> & set,const T & item)83 inline bool FindIn(const std::unordered_set<T, Comparator> &set, const T &item)
84 {
85 return set.find(item) != set.end();
86 }
87
88 template <typename T>
FindIn(const MapleSet<T> & set,const T & item)89 inline bool FindIn(const MapleSet<T> &set, const T &item)
90 {
91 return set.find(item) != set.end();
92 }
93
94 template <typename T>
FindIn(const MapleUnorderedSet<T> & set,const T & item)95 inline bool FindIn(const MapleUnorderedSet<T> &set, const T &item)
96 {
97 return set.find(item) != set.end();
98 }
99
100 template <typename T>
FindIn(const MapleList<T> & list,const T & item)101 inline bool FindIn(const MapleList<T> &list, const T &item)
102 {
103 return std::find(list.begin(), list.end(), item) != list.end();
104 }
105
IsBitArrElemSet(const uint64 * vec,const uint32 num)106 inline bool IsBitArrElemSet(const uint64 *vec, const uint32 num)
107 {
108 size_t index = num / kU64;
109 uint64 bit = num % kU64;
110 return vec[index] & (1ULL << bit);
111 }
112
IsBBsetOverlap(const uint64 * vec1,const uint64 * vec2,uint32 bbBuckets)113 inline bool IsBBsetOverlap(const uint64 *vec1, const uint64 *vec2, uint32 bbBuckets)
114 {
115 for (uint32 i = 0; i < bbBuckets; ++i) {
116 if ((vec1[i] & vec2[i]) != 0) {
117 return true;
118 }
119 }
120 return false;
121 }
122
123 /* For each bb, record info pertain to allocation */
124 /*
125 * This is per bb per LR.
126 * LU info is particular to a bb in a LR.
127 */
128 class LiveUnit {
129 public:
130 LiveUnit() = default;
131 ~LiveUnit() = default;
132
133 void PrintLiveUnit() const;
134
GetBegin()135 uint32 GetBegin() const
136 {
137 return begin;
138 }
139
SetBegin(uint32 val)140 void SetBegin(uint32 val)
141 {
142 begin = val;
143 }
144
GetEnd()145 uint32 GetEnd() const
146 {
147 return end;
148 }
149
SetEnd(uint32 end)150 void SetEnd(uint32 end)
151 {
152 this->end = end;
153 }
154
HasCall()155 bool HasCall() const
156 {
157 return hasCall;
158 }
159
SetHasCall(bool hasCall)160 void SetHasCall(bool hasCall)
161 {
162 this->hasCall = hasCall;
163 }
164
GetDefNum()165 uint32 GetDefNum() const
166 {
167 return defNum;
168 }
169
SetDefNum(uint32 defNum)170 void SetDefNum(uint32 defNum)
171 {
172 this->defNum = defNum;
173 }
174
IncDefNum()175 void IncDefNum()
176 {
177 ++defNum;
178 }
179
GetUseNum()180 uint32 GetUseNum() const
181 {
182 return useNum;
183 }
184
SetUseNum(uint32 useNum)185 void SetUseNum(uint32 useNum)
186 {
187 this->useNum = useNum;
188 }
189
IncUseNum()190 void IncUseNum()
191 {
192 ++useNum;
193 }
194
NeedReload()195 bool NeedReload() const
196 {
197 return needReload;
198 }
199
SetNeedReload(bool needReload)200 void SetNeedReload(bool needReload)
201 {
202 this->needReload = needReload;
203 }
204
NeedRestore()205 bool NeedRestore() const
206 {
207 return needRestore;
208 }
209
SetNeedRestore(bool needRestore)210 void SetNeedRestore(bool needRestore)
211 {
212 this->needRestore = needRestore;
213 }
214
215 private:
216 uint32 begin = 0; /* first encounter in bb */
217 uint32 end = 0; /* last encounter in bb */
218 bool hasCall = false; /* bb has a call */
219 uint32 defNum = 0;
220 uint32 useNum = 0; /* used for priority calculation */
221 bool needReload = false;
222 bool needRestore = false;
223 };
224
225 struct SortedBBCmpFunc {
operatorSortedBBCmpFunc226 bool operator()(const BB *lhs, const BB *rhs) const
227 {
228 return (lhs->GetLevel() < rhs->GetLevel());
229 }
230 };
231
232 enum refType : uint8 {
233 kIsUse = 0x1,
234 kIsDef = 0x2,
235 kIsCall = 0x4,
236 };
237
238 /* LR is for each global vreg. */
239 class LiveRange {
240 public:
LiveRange(MapleAllocator & allocator)241 explicit LiveRange(MapleAllocator &allocator)
242 : lrAlloca(&allocator),
243 pregveto(allocator.Adapter()),
244 callDef(allocator.Adapter()),
245 forbidden(allocator.Adapter()),
246 prefs(allocator.Adapter()),
247 refMap(allocator.Adapter()),
248 luMap(allocator.Adapter())
249 {
250 }
251
252 ~LiveRange() = default;
253
GetRegNO()254 regno_t GetRegNO() const
255 {
256 return regNO;
257 }
258
SetRegNO(regno_t val)259 void SetRegNO(regno_t val)
260 {
261 regNO = val;
262 }
263
GetID()264 uint32 GetID() const
265 {
266 return id;
267 }
268
SetID(uint32 id)269 void SetID(uint32 id)
270 {
271 this->id = id;
272 }
273
GetAssignedRegNO()274 regno_t GetAssignedRegNO() const
275 {
276 return assignedRegNO;
277 }
278
SetAssignedRegNO(regno_t val)279 void SetAssignedRegNO(regno_t val)
280 {
281 assignedRegNO = val;
282 }
283
GetNumCall()284 uint32 GetNumCall() const
285 {
286 return numCall;
287 }
288
SetNumCall(uint32 num)289 void SetNumCall(uint32 num)
290 {
291 numCall = num;
292 }
293
IncNumCall()294 void IncNumCall()
295 {
296 ++numCall;
297 }
298
GetRegType()299 RegType GetRegType() const
300 {
301 return regType;
302 }
303
SetRegType(RegType regType)304 void SetRegType(RegType regType)
305 {
306 this->regType = regType;
307 }
308
GetPriority()309 float GetPriority() const
310 {
311 return priority;
312 }
313
SetPriority(float priority)314 void SetPriority(float priority)
315 {
316 this->priority = priority;
317 }
318
IsMustAssigned()319 bool IsMustAssigned() const
320 {
321 return mustAssigned;
322 }
323
SetMustAssigned()324 void SetMustAssigned()
325 {
326 mustAssigned = true;
327 }
328
SetBBBuckets(uint32 bucketNum)329 void SetBBBuckets(uint32 bucketNum)
330 {
331 bbBuckets = bucketNum;
332 }
333
SetRegBuckets(uint32 bucketNum)334 void SetRegBuckets(uint32 bucketNum)
335 {
336 regBuckets = bucketNum;
337 }
338
GetNumBBMembers()339 uint32 GetNumBBMembers() const
340 {
341 return numBBMembers;
342 }
343
IncNumBBMembers()344 void IncNumBBMembers()
345 {
346 ++numBBMembers;
347 }
348
DecNumBBMembers()349 void DecNumBBMembers()
350 {
351 --numBBMembers;
352 }
353
InitBBMember(MemPool & memPool,size_t size)354 void InitBBMember(MemPool &memPool, size_t size)
355 {
356 bbMember = memPool.NewArray<uint64>(size);
357 errno_t ret = memset_s(bbMember, size * sizeof(uint64), 0, size * sizeof(uint64));
358 CHECK_FATAL(ret == EOK, "call memset_s failed");
359 }
360
GetBBMember()361 uint64 *GetBBMember()
362 {
363 return bbMember;
364 }
365
GetBBMember()366 const uint64 *GetBBMember() const
367 {
368 return bbMember;
369 }
370
GetBBMemberElem(int32 index)371 uint64 GetBBMemberElem(int32 index) const
372 {
373 return bbMember[index];
374 }
375
SetBBMemberElem(int32 index,uint64 elem)376 void SetBBMemberElem(int32 index, uint64 elem)
377 {
378 bbMember[index] = elem;
379 }
380
SetMemberBitArrElem(uint32 bbID)381 void SetMemberBitArrElem(uint32 bbID)
382 {
383 uint32 index = bbID / kU64;
384 uint64 bit = bbID % kU64;
385 uint64 mask = 1ULL << bit;
386 if ((GetBBMemberElem(index) & mask) == 0) {
387 IncNumBBMembers();
388 SetBBMemberElem(index, GetBBMemberElem(index) | mask);
389 }
390 }
391
UnsetMemberBitArrElem(uint32 bbID)392 void UnsetMemberBitArrElem(uint32 bbID)
393 {
394 uint32 index = bbID / kU64;
395 uint64 bit = bbID % kU64;
396 uint64 mask = 1ULL << bit;
397 if ((GetBBMemberElem(index) & mask) != 0) {
398 DecNumBBMembers();
399 SetBBMemberElem(index, GetBBMemberElem(index) & (~mask));
400 }
401 }
402
SetConflictBitArrElem(regno_t regNO)403 void SetConflictBitArrElem(regno_t regNO)
404 {
405 uint32 index = regNO / kU64;
406 uint64 bit = regNO % kU64;
407 uint64 mask = 1ULL << bit;
408 if ((GetBBConflictElem(index) & mask) == 0) {
409 IncNumBBConflicts();
410 SetBBConflictElem(index, GetBBConflictElem(index) | mask);
411 }
412 }
413
UnsetConflictBitArrElem(regno_t regNO)414 void UnsetConflictBitArrElem(regno_t regNO)
415 {
416 uint32 index = regNO / kU64;
417 uint64 bit = regNO % kU64;
418 uint64 mask = 1ULL << bit;
419 if ((GetBBConflictElem(index) & mask) != 0) {
420 DecNumBBConflicts();
421 SetBBConflictElem(index, GetBBConflictElem(index) & (~mask));
422 }
423 }
424
InitPregveto()425 void InitPregveto()
426 {
427 pregveto.clear();
428 pregveto.resize(kMaxRegNum);
429 callDef.clear();
430 callDef.resize(kMaxRegNum);
431 }
432
GetPregveto(regno_t regno)433 bool GetPregveto(regno_t regno) const
434 {
435 return pregveto[regno];
436 }
437
GetPregvetoSize()438 size_t GetPregvetoSize() const
439 {
440 return numPregveto;
441 }
442
InsertElemToPregveto(regno_t regno)443 void InsertElemToPregveto(regno_t regno)
444 {
445 if (!pregveto[regno]) {
446 pregveto[regno] = true;
447 ++numPregveto;
448 }
449 }
450
GetCallDef(regno_t regno)451 bool GetCallDef(regno_t regno) const
452 {
453 return callDef[regno];
454 }
455
InsertElemToCallDef(regno_t regno)456 void InsertElemToCallDef(regno_t regno)
457 {
458 if (!callDef[regno]) {
459 callDef[regno] = true;
460 ++numCallDef;
461 }
462 }
463
SetCrossCall()464 void SetCrossCall()
465 {
466 crossCall = true;
467 }
468
GetCrossCall()469 bool GetCrossCall() const
470 {
471 return crossCall;
472 }
473
InitForbidden()474 void InitForbidden()
475 {
476 forbidden.clear();
477 forbidden.resize(kMaxRegNum);
478 }
479
GetForbidden()480 const MapleVector<bool> &GetForbidden() const
481 {
482 return forbidden;
483 }
484
GetForbidden(regno_t regno)485 bool GetForbidden(regno_t regno) const
486 {
487 return forbidden[regno];
488 }
489
GetForbiddenSize()490 size_t GetForbiddenSize() const
491 {
492 return numForbidden;
493 }
494
InsertElemToForbidden(regno_t regno)495 void InsertElemToForbidden(regno_t regno)
496 {
497 if (!forbidden[regno]) {
498 forbidden[regno] = true;
499 ++numForbidden;
500 }
501 }
502
EraseElemFromForbidden(regno_t regno)503 void EraseElemFromForbidden(regno_t regno)
504 {
505 if (forbidden[regno]) {
506 forbidden[regno] = false;
507 --numForbidden;
508 }
509 }
510
ClearForbidden()511 void ClearForbidden()
512 {
513 forbidden.clear();
514 }
515
GetNumBBConflicts()516 uint32 GetNumBBConflicts() const
517 {
518 return numBBConflicts;
519 }
520
IncNumBBConflicts()521 void IncNumBBConflicts()
522 {
523 ++numBBConflicts;
524 }
525
DecNumBBConflicts()526 void DecNumBBConflicts()
527 {
528 --numBBConflicts;
529 }
530
InitBBConflict(MemPool & memPool,size_t size)531 void InitBBConflict(MemPool &memPool, size_t size)
532 {
533 bbConflict = memPool.NewArray<uint64>(size);
534 errno_t ret = memset_s(bbConflict, size * sizeof(uint64), 0, size * sizeof(uint64));
535 CHECK_FATAL(ret == EOK, "call memset_s failed");
536 }
537
GetBBConflict()538 const uint64 *GetBBConflict() const
539 {
540 return bbConflict;
541 }
542
GetBBConflictElem(int32 index)543 uint64 GetBBConflictElem(int32 index) const
544 {
545 DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
546 return bbConflict[index];
547 }
548
SetBBConflictElem(int32 index,uint64 elem)549 void SetBBConflictElem(int32 index, uint64 elem)
550 {
551 DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
552 bbConflict[index] = elem;
553 }
554
SetOldConflict(uint64 * conflict)555 void SetOldConflict(uint64 *conflict)
556 {
557 oldConflict = conflict;
558 }
559
GetOldConflict()560 const uint64 *GetOldConflict() const
561 {
562 return oldConflict;
563 }
564
GetPrefs()565 const MapleSet<regno_t> &GetPrefs() const
566 {
567 return prefs;
568 }
569
InsertElemToPrefs(regno_t regNO)570 void InsertElemToPrefs(regno_t regNO)
571 {
572 (void)prefs.insert(regNO);
573 }
574
GetRefs()575 const MapleMap<uint32, MapleMap<uint32, uint32> *> GetRefs() const
576 {
577 return refMap;
578 }
579
GetRefs(uint32 bbId)580 const MapleMap<uint32, uint32> GetRefs(uint32 bbId) const
581 {
582 return *(refMap.find(bbId)->second);
583 }
584
AddRef(uint32 bbId,uint32 pos,uint32 mark)585 void AddRef(uint32 bbId, uint32 pos, uint32 mark)
586 {
587 if (refMap.find(bbId) == refMap.end()) {
588 auto point = lrAlloca->New<MapleMap<uint32, uint32>>(lrAlloca->Adapter());
589 (void)point->emplace(std::pair<uint32, uint32>(pos, mark));
590 (void)refMap.emplace(std::pair<uint32, MapleMap<uint32, uint32> *>(bbId, point));
591 } else {
592 auto &bbPoint = (refMap.find(bbId))->second;
593 if (bbPoint->find(pos) == bbPoint->end()) {
594 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, mark));
595 } else {
596 auto posVal = bbPoint->find(pos)->second;
597 (void)bbPoint->erase(bbPoint->find(pos));
598 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, posVal | mark));
599 }
600 }
601 }
602
GetLuMap()603 const MapleMap<uint32, LiveUnit *> &GetLuMap() const
604 {
605 return luMap;
606 }
607
FindInLuMap(uint32 index)608 MapleMap<uint32, LiveUnit *>::iterator FindInLuMap(uint32 index)
609 {
610 return luMap.find(index);
611 }
612
EndOfLuMap()613 MapleMap<uint32, LiveUnit *>::iterator EndOfLuMap()
614 {
615 return luMap.end();
616 }
617
EraseLuMap(MapleMap<uint32,LiveUnit * >::iterator it)618 MapleMap<uint32, LiveUnit *>::iterator EraseLuMap(MapleMap<uint32, LiveUnit *>::iterator it)
619 {
620 return luMap.erase(it);
621 }
622
SetElemToLuMap(uint32 key,LiveUnit & value)623 void SetElemToLuMap(uint32 key, LiveUnit &value)
624 {
625 luMap[key] = &value;
626 }
627
GetLiveUnitFromLuMap(uint32 key)628 LiveUnit *GetLiveUnitFromLuMap(uint32 key)
629 {
630 return luMap[key];
631 }
632
GetLiveUnitFromLuMap(uint32 key)633 const LiveUnit *GetLiveUnitFromLuMap(uint32 key) const
634 {
635 auto it = luMap.find(key);
636 DEBUG_ASSERT(it != luMap.end(), "can't find live unit");
637 return it->second;
638 }
639
GetSplitLr()640 const LiveRange *GetSplitLr() const
641 {
642 return splitLr;
643 }
644
SetSplitLr(LiveRange & lr)645 void SetSplitLr(LiveRange &lr)
646 {
647 splitLr = &lr;
648 }
649
650 #ifdef OPTIMIZE_FOR_PROLOG
GetNumDefs()651 uint32 GetNumDefs() const
652 {
653 return numDefs;
654 }
655
IncNumDefs()656 void IncNumDefs()
657 {
658 ++numDefs;
659 }
660
SetNumDefs(uint32 val)661 void SetNumDefs(uint32 val)
662 {
663 numDefs = val;
664 }
665
GetNumUses()666 uint32 GetNumUses() const
667 {
668 return numUses;
669 }
670
IncNumUses()671 void IncNumUses()
672 {
673 ++numUses;
674 }
675
SetNumUses(uint32 val)676 void SetNumUses(uint32 val)
677 {
678 numUses = val;
679 }
680
GetFrequency()681 uint32 GetFrequency() const
682 {
683 return frequency;
684 }
685
SetFrequency(uint32 frequency)686 void SetFrequency(uint32 frequency)
687 {
688 this->frequency = frequency;
689 }
690 #endif /* OPTIMIZE_FOR_PROLOG */
691
GetSpillMem()692 MemOperand *GetSpillMem()
693 {
694 return spillMem;
695 }
696
GetSpillMem()697 const MemOperand *GetSpillMem() const
698 {
699 return spillMem;
700 }
701
SetSpillMem(MemOperand & memOpnd)702 void SetSpillMem(MemOperand &memOpnd)
703 {
704 spillMem = &memOpnd;
705 }
706
GetSpillReg()707 regno_t GetSpillReg() const
708 {
709 return spillReg;
710 }
711
SetSpillReg(regno_t spillReg)712 void SetSpillReg(regno_t spillReg)
713 {
714 this->spillReg = spillReg;
715 }
716
GetSpillSize()717 uint32 GetSpillSize() const
718 {
719 return spillSize;
720 }
721
SetSpillSize(uint32 size)722 void SetSpillSize(uint32 size)
723 {
724 spillSize = size;
725 }
726
IsSpilled()727 bool IsSpilled() const
728 {
729 return spilled;
730 }
731
SetSpilled(bool spill)732 void SetSpilled(bool spill)
733 {
734 spilled = spill;
735 }
736
HasDefUse()737 bool HasDefUse() const
738 {
739 return hasDefUse;
740 }
741
SetDefUse()742 void SetDefUse()
743 {
744 hasDefUse = true;
745 }
746
GetProcessed()747 bool GetProcessed() const
748 {
749 return proccessed;
750 }
751
SetProcessed()752 void SetProcessed()
753 {
754 proccessed = true;
755 }
756
IsNonLocal()757 bool IsNonLocal() const
758 {
759 return isNonLocal;
760 }
761
SetIsNonLocal(bool isNonLocal)762 void SetIsNonLocal(bool isNonLocal)
763 {
764 this->isNonLocal = isNonLocal;
765 }
766
SetRematLevel(uint32 val)767 void SetRematLevel(uint32 val)
768 {
769 this->rematLevel = val;
770 }
771
GetRematLevel()772 uint32 GetRematLevel() const
773 {
774 return this->rematLevel;
775 }
776
GetOp()777 Opcode GetOp() const
778 {
779 return op;
780 }
781
GetRematSymbol()782 const MIRSymbol *GetRematSymbol() const
783 {
784 DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat symbol is invalid");
785 return rematInfo.sym;
786 }
787
GetRematFieldID()788 FieldID GetRematFieldID() const
789 {
790 DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat field ID is invalid");
791 return fieldID;
792 }
793
SetRematerializable(const MIRConst * c)794 void SetRematerializable(const MIRConst *c)
795 {
796 op = OP_constval;
797 rematInfo.mirConst = c;
798 }
799
SetRematerializable(Opcode opcode,const MIRSymbol * symbol,FieldID fieldId,bool addrUp)800 void SetRematerializable(Opcode opcode, const MIRSymbol *symbol, FieldID fieldId, bool addrUp)
801 {
802 this->op = opcode;
803 rematInfo.sym = symbol;
804 this->fieldID = fieldId;
805 this->addrUpper = addrUp;
806 }
807
CopyRematerialization(const LiveRange & lr)808 void CopyRematerialization(const LiveRange &lr)
809 {
810 op = lr.op;
811 rematInfo = lr.rematInfo;
812 fieldID = lr.fieldID;
813 }
814
815 bool IsRematerializable(AArch64CGFunc &cgFunc, uint8 rematLevel) const;
816 std::vector<Insn *> Rematerialize(AArch64CGFunc *cgFunc, RegOperand ®Op);
817
818 private:
819 MapleAllocator *lrAlloca;
820 regno_t regNO = 0;
821 uint32 id = 0; /* for priority tie breaker */
822 regno_t assignedRegNO = 0; /* color assigned */
823 uint32 numCall = 0;
824 RegType regType = kRegTyUndef;
825 float priority = 0.0;
826 bool mustAssigned = false;
827 uint32 bbBuckets = 0; /* size of bit array for bb (each bucket == 64 bits) */
828 uint32 regBuckets = 0; /* size of bit array for reg (each bucket == 64 bits) */
829 uint32 numBBMembers = 0; /* number of bits set in bbMember */
830 uint64 *bbMember = nullptr; /* Same as smember, but use bit array */
831
832 MapleVector<bool> pregveto; /* pregs cannot be assigned -- SplitLr may clear forbidden */
833 MapleVector<bool> callDef; /* pregs cannot be assigned -- SplitLr may clear forbidden */
834 MapleVector<bool> forbidden; /* pregs cannot be assigned */
835 uint32 numPregveto = 0;
836 uint32 numCallDef = 0;
837 uint32 numForbidden = 0;
838 bool crossCall = false;
839
840 uint32 numBBConflicts = 0; /* number of bits set in bbConflict */
841 uint64 *bbConflict = nullptr; /* vreg interference from graph neighbors (bit) */
842 uint64 *oldConflict = nullptr;
843 MapleSet<regno_t> prefs; /* pregs that prefer */
844 MapleMap<uint32, MapleMap<uint32, uint32> *> refMap;
845 MapleMap<uint32, LiveUnit *> luMap; /* info for each bb */
846 LiveRange *splitLr = nullptr; /* The 1st part of the split */
847 #ifdef OPTIMIZE_FOR_PROLOG
848 uint32 numDefs = 0;
849 uint32 numUses = 0;
850 uint32 frequency = 0;
851 #endif /* OPTIMIZE_FOR_PROLOG */
852 MemOperand *spillMem = nullptr; /* memory operand used for spill, if any */
853 regno_t spillReg = 0; /* register operand for spill at current point */
854 uint32 spillSize = 0; /* 32 or 64 bit spill */
855 bool spilled = false; /* color assigned */
856 bool hasDefUse = false; /* has regDS */
857 bool proccessed = false;
858 bool isNonLocal = false;
859 uint32 rematLevel = 0;
860 Opcode op = OP_undef; /* OP_constval, OP_addrof or OP_dread if rematerializable */
861 union RematInfo {
862 const MIRConst *mirConst;
863 const MIRSymbol *sym;
864 } rematInfo; /* info for rematerializing value */
865 FieldID fieldID = 0; /* used only when op is OP_addrof or OP_dread */
866 bool addrUpper = false; /* indicates the upper bits of an addrof */
867 };
868
869 /* One per bb, to communicate local usage to global RA */
870 class LocalRaInfo {
871 public:
LocalRaInfo(MapleAllocator & allocator)872 explicit LocalRaInfo(MapleAllocator &allocator) : defCnt(allocator.Adapter()), useCnt(allocator.Adapter()) {}
873
874 ~LocalRaInfo() = default;
875
GetDefCnt()876 const MapleMap<regno_t, uint16> &GetDefCnt() const
877 {
878 return defCnt;
879 }
880
GetDefCntElem(regno_t regNO)881 uint16 GetDefCntElem(regno_t regNO)
882 {
883 return defCnt[regNO];
884 }
885
SetDefCntElem(regno_t key,uint16 value)886 void SetDefCntElem(regno_t key, uint16 value)
887 {
888 defCnt[key] = value;
889 }
890
GetUseCnt()891 const MapleMap<regno_t, uint16> &GetUseCnt() const
892 {
893 return useCnt;
894 }
895
GetUseCntElem(regno_t regNO)896 uint16 GetUseCntElem(regno_t regNO)
897 {
898 return useCnt[regNO];
899 }
900
SetUseCntElem(regno_t key,uint16 value)901 void SetUseCntElem(regno_t key, uint16 value)
902 {
903 useCnt[key] = value;
904 }
905
906 private:
907 MapleMap<regno_t, uint16> defCnt;
908 MapleMap<regno_t, uint16> useCnt;
909 };
910
911 /* For each bb, record info pertain to allocation */
912 class BBAssignInfo {
913 public:
BBAssignInfo(MapleAllocator & allocator)914 explicit BBAssignInfo(MapleAllocator &allocator) : globalsAssigned(allocator.Adapter()), regMap(allocator.Adapter())
915 {
916 }
917
918 ~BBAssignInfo() = default;
919
GetIntLocalRegsNeeded()920 uint32 GetIntLocalRegsNeeded() const
921 {
922 return intLocalRegsNeeded;
923 }
924
SetIntLocalRegsNeeded(uint32 num)925 void SetIntLocalRegsNeeded(uint32 num)
926 {
927 intLocalRegsNeeded = num;
928 }
929
GetFpLocalRegsNeeded()930 uint32 GetFpLocalRegsNeeded() const
931 {
932 return fpLocalRegsNeeded;
933 }
934
SetFpLocalRegsNeeded(uint32 num)935 void SetFpLocalRegsNeeded(uint32 num)
936 {
937 fpLocalRegsNeeded = num;
938 }
939
InitGlobalAssigned()940 void InitGlobalAssigned()
941 {
942 globalsAssigned.clear();
943 globalsAssigned.resize(kMaxRegNum);
944 }
945
GetGlobalsAssigned(regno_t regNO)946 bool GetGlobalsAssigned(regno_t regNO) const
947 {
948 return globalsAssigned[regNO];
949 }
950
InsertElemToGlobalsAssigned(regno_t regNO)951 void InsertElemToGlobalsAssigned(regno_t regNO)
952 {
953 globalsAssigned[regNO] = true;
954 }
955
EraseElemToGlobalsAssigned(regno_t regNO)956 void EraseElemToGlobalsAssigned(regno_t regNO)
957 {
958 globalsAssigned[regNO] = false;
959 }
960
GetRegMap()961 const MapleMap<regno_t, regno_t> &GetRegMap() const
962 {
963 return regMap;
964 }
965
HasRegMap(regno_t regNOKey)966 bool HasRegMap(regno_t regNOKey) const
967 {
968 return (regMap.find(regNOKey) != regMap.end());
969 }
970
GetRegMapElem(regno_t regNO)971 regno_t GetRegMapElem(regno_t regNO)
972 {
973 return regMap[regNO];
974 }
975
SetRegMapElem(regno_t regNOKey,regno_t regNOValue)976 void SetRegMapElem(regno_t regNOKey, regno_t regNOValue)
977 {
978 regMap[regNOKey] = regNOValue;
979 }
980
981 private:
982 uint32 intLocalRegsNeeded = 0; /* num local reg needs for each bb */
983 uint32 fpLocalRegsNeeded = 0; /* num local reg needs for each bb */
984 MapleVector<bool> globalsAssigned; /* globals used in a bb */
985 MapleMap<regno_t, regno_t> regMap; /* local vreg to preg mapping */
986 };
987
988 class FinalizeRegisterInfo {
989 public:
FinalizeRegisterInfo(MapleAllocator & allocator)990 explicit FinalizeRegisterInfo(MapleAllocator &allocator)
991 : defOperands(allocator.Adapter()),
992 defIdx(allocator.Adapter()),
993 useOperands(allocator.Adapter()),
994 useIdx(allocator.Adapter())
995 {
996 }
997
998 ~FinalizeRegisterInfo() = default;
ClearInfo()999 void ClearInfo()
1000 {
1001 memOperandIdx = 0;
1002 baseOperand = nullptr;
1003 offsetOperand = nullptr;
1004 defOperands.clear();
1005 defIdx.clear();
1006 useOperands.clear();
1007 useIdx.clear();
1008 }
1009
SetBaseOperand(Operand & opnd,const int32 idx)1010 void SetBaseOperand(Operand &opnd, const int32 idx)
1011 {
1012 baseOperand = &opnd;
1013 memOperandIdx = idx;
1014 }
1015
SetOffsetOperand(Operand & opnd)1016 void SetOffsetOperand(Operand &opnd)
1017 {
1018 offsetOperand = &opnd;
1019 }
1020
SetDefOperand(Operand & opnd,const int32 idx)1021 void SetDefOperand(Operand &opnd, const int32 idx)
1022 {
1023 defOperands.emplace_back(&opnd);
1024 defIdx.emplace_back(idx);
1025 }
1026
SetUseOperand(Operand & opnd,const int32 idx)1027 void SetUseOperand(Operand &opnd, const int32 idx)
1028 {
1029 useOperands.emplace_back(&opnd);
1030 useIdx.emplace_back(idx);
1031 }
1032
GetMemOperandIdx()1033 int32 GetMemOperandIdx() const
1034 {
1035 return memOperandIdx;
1036 }
1037
GetBaseOperand()1038 const Operand *GetBaseOperand() const
1039 {
1040 return baseOperand;
1041 }
1042
GetOffsetOperand()1043 const Operand *GetOffsetOperand() const
1044 {
1045 return offsetOperand;
1046 }
1047
GetDefOperandsSize()1048 size_t GetDefOperandsSize() const
1049 {
1050 return defOperands.size();
1051 }
1052
GetDefOperandsElem(size_t index)1053 const Operand *GetDefOperandsElem(size_t index) const
1054 {
1055 return defOperands[index];
1056 }
1057
GetDefIdxElem(size_t index)1058 int32 GetDefIdxElem(size_t index) const
1059 {
1060 return defIdx[index];
1061 }
1062
GetUseOperandsSize()1063 size_t GetUseOperandsSize() const
1064 {
1065 return useOperands.size();
1066 }
1067
GetUseOperandsElem(size_t index)1068 const Operand *GetUseOperandsElem(size_t index) const
1069 {
1070 return useOperands[index];
1071 }
1072
GetUseIdxElem(size_t index)1073 int32 GetUseIdxElem(size_t index) const
1074 {
1075 return useIdx[index];
1076 }
1077
1078 private:
1079 int32 memOperandIdx = 0;
1080 Operand *baseOperand = nullptr;
1081 Operand *offsetOperand = nullptr;
1082 MapleVector<Operand *> defOperands;
1083 MapleVector<int32> defIdx;
1084 MapleVector<Operand *> useOperands;
1085 MapleVector<int32> useIdx;
1086 };
1087
1088 class LocalRegAllocator {
1089 public:
LocalRegAllocator(CGFunc & cgFunc,MapleAllocator & allocator)1090 LocalRegAllocator(CGFunc &cgFunc, MapleAllocator &allocator)
1091 : intRegAssignmentMap(allocator.Adapter()),
1092 fpRegAssignmentMap(allocator.Adapter()),
1093 useInfo(allocator.Adapter()),
1094 defInfo(allocator.Adapter())
1095 {
1096 buckets = (cgFunc.GetMaxRegNum() / kU64) + 1;
1097 intRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1098 fpRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1099 intRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1100 fpRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1101 }
1102
1103 ~LocalRegAllocator() = default;
1104
ClearLocalRaInfo()1105 void ClearLocalRaInfo()
1106 {
1107 ClearBitArrElement(intRegAssigned);
1108 ClearBitArrElement(fpRegAssigned);
1109 intRegAssignmentMap.clear();
1110 fpRegAssignmentMap.clear();
1111 intPregUsed = 0;
1112 fpPregUsed = 0;
1113 ClearBitArrElement(intRegSpilled);
1114 ClearBitArrElement(fpRegSpilled);
1115 numIntPregUsed = 0;
1116 numFpPregUsed = 0;
1117 }
1118
RegBaseUpdate(bool isInt)1119 regno_t RegBaseUpdate(bool isInt) const
1120 {
1121 return isInt ? 0 : V0 - R0;
1122 }
1123
IsInRegAssigned(regno_t regNO,bool isInt)1124 bool IsInRegAssigned(regno_t regNO, bool isInt) const
1125 {
1126 uint64 *regAssigned = nullptr;
1127 if (isInt) {
1128 regAssigned = intRegAssigned;
1129 } else {
1130 regAssigned = fpRegAssigned;
1131 }
1132 return IsBitArrElemSet(regAssigned, regNO);
1133 ;
1134 }
1135
SetRegAssigned(regno_t regNO,bool isInt)1136 void SetRegAssigned(regno_t regNO, bool isInt) const
1137 {
1138 if (isInt) {
1139 SetBitArrElement(intRegAssigned, regNO);
1140 } else {
1141 SetBitArrElement(fpRegAssigned, regNO);
1142 }
1143 }
1144
GetRegAssignmentItem(bool isInt,regno_t regKey)1145 regno_t GetRegAssignmentItem(bool isInt, regno_t regKey)
1146 {
1147 return isInt ? intRegAssignmentMap[regKey] : fpRegAssignmentMap[regKey];
1148 }
1149
SetRegAssignmentMap(bool isInt,regno_t regKey,regno_t regValue)1150 void SetRegAssignmentMap(bool isInt, regno_t regKey, regno_t regValue)
1151 {
1152 if (isInt) {
1153 intRegAssignmentMap[regKey] = regValue;
1154 } else {
1155 fpRegAssignmentMap[regKey] = regValue;
1156 }
1157 }
1158
1159 /* only for HandleLocalRaDebug */
GetPregUsed(bool isInt)1160 uint64 GetPregUsed(bool isInt) const
1161 {
1162 if (isInt) {
1163 return intPregUsed;
1164 } else {
1165 return fpPregUsed;
1166 }
1167 }
1168
SetPregUsed(regno_t regNO,bool isInt)1169 void SetPregUsed(regno_t regNO, bool isInt)
1170 {
1171 uint64 mask = 0;
1172 if (isInt) {
1173 mask = 1ULL << (regNO - R0);
1174 if ((intPregUsed & mask) == 0) {
1175 ++numIntPregUsed;
1176 intPregUsed |= mask;
1177 }
1178 } else {
1179 mask = 1ULL << (regNO - V0);
1180 if ((fpPregUsed & mask) == 0) {
1181 ++numFpPregUsed;
1182 fpPregUsed |= mask;
1183 }
1184 }
1185 }
1186
isInRegSpilled(regno_t regNO,bool isInt)1187 bool isInRegSpilled(regno_t regNO, bool isInt) const
1188 {
1189 bool isSet;
1190 if (isInt) {
1191 isSet = IsBitArrElemSet(intRegSpilled, regNO);
1192 } else {
1193 isSet = IsBitArrElemSet(fpRegSpilled, regNO);
1194 }
1195 return isSet;
1196 }
1197
SetRegSpilled(regno_t regNO,bool isInt)1198 void SetRegSpilled(regno_t regNO, bool isInt) const
1199 {
1200 if (isInt) {
1201 SetBitArrElement(intRegSpilled, regNO);
1202 } else {
1203 SetBitArrElement(fpRegSpilled, regNO);
1204 }
1205 }
1206
GetPregs(bool isInt)1207 uint64 GetPregs(bool isInt) const
1208 {
1209 if (isInt) {
1210 return intPregs;
1211 } else {
1212 return fpPregs;
1213 }
1214 }
1215
SetPregs(regno_t regNO,bool isInt)1216 void SetPregs(regno_t regNO, bool isInt)
1217 {
1218 if (isInt) {
1219 intPregs |= 1ULL << (regNO - RegBaseUpdate(true));
1220 } else {
1221 fpPregs |= 1ULL << (regNO - RegBaseUpdate(false));
1222 }
1223 }
1224
ClearPregs(regno_t regNO,bool isInt)1225 void ClearPregs(regno_t regNO, bool isInt)
1226 {
1227 if (isInt) {
1228 intPregs &= ~(1ULL << (regNO - RegBaseUpdate(true)));
1229 } else {
1230 fpPregs &= ~(1ULL << (regNO - RegBaseUpdate(false)));
1231 }
1232 }
1233
IsPregAvailable(regno_t regNO,bool isInt)1234 bool IsPregAvailable(regno_t regNO, bool isInt) const
1235 {
1236 bool isAvailable;
1237 if (isInt) {
1238 isAvailable = intPregs & (1ULL << (regNO - RegBaseUpdate(true)));
1239 } else {
1240 isAvailable = fpPregs & (1ULL << (regNO - RegBaseUpdate(false)));
1241 }
1242 return isAvailable;
1243 }
1244
InitPregs(uint32 intMax,uint32 fpMax,bool hasYield,const MapleSet<uint32> & intSpillRegSet,const MapleSet<uint32> & fpSpillRegSet)1245 void InitPregs(uint32 intMax, uint32 fpMax, bool hasYield, const MapleSet<uint32> &intSpillRegSet,
1246 const MapleSet<uint32> &fpSpillRegSet)
1247 {
1248 uint32 intBase = R0;
1249 uint32 fpBase = V0;
1250 intPregs = (1ULL << (intMax + 1)) - 1;
1251 fpPregs = (1ULL << (((fpMax + 1) + fpBase) - RegBaseUpdate(false))) - 1;
1252 for (uint32 regNO : intSpillRegSet) {
1253 ClearPregs(regNO + intBase, true);
1254 }
1255 for (uint32 regNO : fpSpillRegSet) {
1256 ClearPregs(regNO + fpBase, false);
1257 }
1258 if (hasYield) {
1259 ClearPregs(RYP, true);
1260 }
1261 #ifdef RESERVED_REGS
1262 intPregs &= ~(1ULL << R16);
1263 intPregs &= ~(1ULL << R17);
1264 #endif /* RESERVED_REGS */
1265 }
1266
GetIntRegAssignmentMap()1267 const MapleMap<regno_t, regno_t> &GetIntRegAssignmentMap() const
1268 {
1269 return intRegAssignmentMap;
1270 }
1271
GetFpRegAssignmentMap()1272 const MapleMap<regno_t, regno_t> &GetFpRegAssignmentMap() const
1273 {
1274 return fpRegAssignmentMap;
1275 }
1276
GetUseInfo()1277 const MapleMap<regno_t, uint16> &GetUseInfo() const
1278 {
1279 return useInfo;
1280 }
1281
SetUseInfoElem(regno_t regNO,uint16 info)1282 void SetUseInfoElem(regno_t regNO, uint16 info)
1283 {
1284 useInfo[regNO] = info;
1285 }
1286
IncUseInfoElem(regno_t regNO)1287 void IncUseInfoElem(regno_t regNO)
1288 {
1289 if (useInfo.find(regNO) != useInfo.end()) {
1290 ++useInfo[regNO];
1291 }
1292 }
1293
GetUseInfoElem(regno_t regNO)1294 uint16 GetUseInfoElem(regno_t regNO)
1295 {
1296 return useInfo[regNO];
1297 }
1298
ClearUseInfo()1299 void ClearUseInfo()
1300 {
1301 useInfo.clear();
1302 }
1303
GetDefInfo()1304 const MapleMap<regno_t, uint16> &GetDefInfo() const
1305 {
1306 return defInfo;
1307 }
1308
SetDefInfoElem(regno_t regNO,uint16 info)1309 void SetDefInfoElem(regno_t regNO, uint16 info)
1310 {
1311 defInfo[regNO] = info;
1312 }
1313
GetDefInfoElem(regno_t regNO)1314 uint16 GetDefInfoElem(regno_t regNO)
1315 {
1316 return defInfo[regNO];
1317 }
1318
IncDefInfoElem(regno_t regNO)1319 void IncDefInfoElem(regno_t regNO)
1320 {
1321 if (defInfo.find(regNO) != defInfo.end()) {
1322 ++defInfo[regNO];
1323 }
1324 }
1325
ClearDefInfo()1326 void ClearDefInfo()
1327 {
1328 defInfo.clear();
1329 }
1330
GetNumIntPregUsed()1331 uint32 GetNumIntPregUsed() const
1332 {
1333 return numIntPregUsed;
1334 }
1335
GetNumFpPregUsed()1336 uint32 GetNumFpPregUsed() const
1337 {
1338 return numFpPregUsed;
1339 }
1340
1341 private:
ClearBitArrElement(uint64 * vec)1342 void ClearBitArrElement(uint64 *vec) const
1343 {
1344 for (uint32 i = 0; i < buckets; ++i) {
1345 vec[i] = 0UL;
1346 }
1347 }
1348
SetBitArrElement(uint64 * vec,regno_t regNO)1349 void SetBitArrElement(uint64 *vec, regno_t regNO) const
1350 {
1351 uint32 index = regNO / kU64;
1352 uint64 bit = regNO % kU64;
1353 vec[index] |= 1ULL << bit;
1354 }
1355
1356 /* The following local vars keeps track of allocation information in bb. */
1357 uint64 *intRegAssigned; /* in this set if vreg is assigned */
1358 uint64 *fpRegAssigned;
1359 MapleMap<regno_t, regno_t> intRegAssignmentMap; /* vreg -> preg map, which preg is the vreg assigned */
1360 MapleMap<regno_t, regno_t> fpRegAssignmentMap;
1361 uint64 intPregUsed = 0; /* pregs used in bb */
1362 uint64 fpPregUsed = 0;
1363 uint64 *intRegSpilled; /* on this list if vreg is spilled */
1364 uint64 *fpRegSpilled;
1365
1366 uint64 intPregs = 0; /* available regs for assignement */
1367 uint64 fpPregs = 0;
1368 MapleMap<regno_t, uint16> useInfo; /* copy of local ra info for useCnt */
1369 MapleMap<regno_t, uint16> defInfo; /* copy of local ra info for defCnt */
1370
1371 uint32 numIntPregUsed = 0;
1372 uint32 numFpPregUsed = 0;
1373 uint32 buckets;
1374 };
1375
1376 class SplitBBInfo {
1377 public:
1378 SplitBBInfo() = default;
1379
1380 ~SplitBBInfo() = default;
1381
GetCandidateBB()1382 BB *GetCandidateBB()
1383 {
1384 return candidateBB;
1385 }
1386
GetCandidateBB()1387 const BB *GetCandidateBB() const
1388 {
1389 return candidateBB;
1390 }
1391
GetStartBB()1392 const BB *GetStartBB() const
1393 {
1394 return startBB;
1395 }
1396
SetCandidateBB(BB & bb)1397 void SetCandidateBB(BB &bb)
1398 {
1399 candidateBB = &bb;
1400 }
1401
SetStartBB(BB & bb)1402 void SetStartBB(BB &bb)
1403 {
1404 startBB = &bb;
1405 }
1406
1407 private:
1408 BB *candidateBB = nullptr;
1409 BB *startBB = nullptr;
1410 };
1411
1412 class GraphColorRegAllocator : public RegAllocator {
1413 public:
GraphColorRegAllocator(CGFunc & cgFunc,MemPool & memPool,DomAnalysis & dom)1414 GraphColorRegAllocator(CGFunc &cgFunc, MemPool &memPool, DomAnalysis &dom)
1415 : RegAllocator(cgFunc, memPool),
1416 domInfo(dom),
1417 bbVec(alloc.Adapter()),
1418 vregLive(alloc.Adapter()),
1419 pregLive(alloc.Adapter()),
1420 lrMap(alloc.Adapter()),
1421 localRegVec(alloc.Adapter()),
1422 bbRegInfo(alloc.Adapter()),
1423 unconstrained(alloc.Adapter()),
1424 unconstrainedPref(alloc.Adapter()),
1425 constrained(alloc.Adapter()),
1426 mustAssigned(alloc.Adapter()),
1427 #ifdef OPTIMIZE_FOR_PROLOG
1428 intDelayed(alloc.Adapter()),
1429 fpDelayed(alloc.Adapter()),
1430 #endif /* OPTIMIZE_FOR_PROLOG */
1431 intCallerRegSet(alloc.Adapter()),
1432 intCalleeRegSet(alloc.Adapter()),
1433 intSpillRegSet(alloc.Adapter()),
1434 fpCallerRegSet(alloc.Adapter()),
1435 fpCalleeRegSet(alloc.Adapter()),
1436 fpSpillRegSet(alloc.Adapter()),
1437 intCalleeUsed(alloc.Adapter()),
1438 fpCalleeUsed(alloc.Adapter())
1439 {
1440 constexpr uint32 kNumInsnThreashold = 30000;
1441 numVregs = cgFunc.GetMaxVReg();
1442 localRegVec.resize(cgFunc.NumBBs());
1443 bbRegInfo.resize(cgFunc.NumBBs());
1444 if (CGOptions::DoMultiPassColorRA() && cgFunc.GetMirModule().IsCModule()) {
1445 uint32 cnt = 0;
1446 FOR_ALL_BB(bb, &cgFunc) {
1447 FOR_BB_INSNS(insn, bb) {
1448 ++cnt;
1449 }
1450 }
1451 DEBUG_ASSERT(cnt <= cgFunc.GetTotalNumberOfInstructions(), "Incorrect insn count");
1452 if (cnt <= kNumInsnThreashold) {
1453 doMultiPass = true;
1454 doLRA = false;
1455 doOptProlog = false;
1456 }
1457 }
1458 }
1459
1460 ~GraphColorRegAllocator() override = default;
1461
1462 bool AllocateRegisters() override;
1463
1464 enum SpillMemCheck : uint8 {
1465 kSpillMemPre,
1466 kSpillMemPost,
1467 };
1468
GetLiveRange(regno_t regNO)1469 LiveRange *GetLiveRange(regno_t regNO)
1470 {
1471 auto it = lrMap.find(regNO);
1472 if (it != lrMap.end()) {
1473 return it->second;
1474 } else {
1475 return nullptr;
1476 }
1477 }
GetLiveRange(regno_t regNO)1478 LiveRange *GetLiveRange(regno_t regNO) const
1479 {
1480 auto it = lrMap.find(regNO);
1481 if (it != lrMap.end()) {
1482 return it->second;
1483 } else {
1484 return nullptr;
1485 }
1486 }
GetLrMap()1487 const MapleMap<regno_t, LiveRange *> &GetLrMap() const
1488 {
1489 return lrMap;
1490 }
1491 Insn *SpillOperand(Insn &insn, const Operand &opnd, bool isDef, RegOperand &phyOpnd, bool forCall = false);
1492
1493 private:
1494 struct SetLiveRangeCmpFunc {
operatorSetLiveRangeCmpFunc1495 bool operator()(const LiveRange *lhs, const LiveRange *rhs) const
1496 {
1497 if (fabs(lhs->GetPriority() - rhs->GetPriority()) <= 1e-6) {
1498 /*
1499 * This is to ensure the ordering is consistent as the reg#
1500 * differs going through VtableImpl.mpl file.
1501 */
1502 if (lhs->GetID() == rhs->GetID()) {
1503 return lhs->GetRegNO() < rhs->GetRegNO();
1504 } else {
1505 return lhs->GetID() < rhs->GetID();
1506 }
1507 }
1508 return (lhs->GetPriority() > rhs->GetPriority());
1509 }
1510 };
1511
1512 template <typename Func>
1513 void ForEachBBArrElem(const uint64 *vec, Func functor) const;
1514
1515 template <typename Func>
1516 void ForEachBBArrElemWithInterrupt(const uint64 *vec, Func functor) const;
1517
1518 template <typename Func>
1519 void ForEachRegArrElem(const uint64 *vec, Func functor) const;
1520
1521 void PrintLiveUnitMap(const LiveRange &lr) const;
1522 void PrintLiveRangeConflicts(const LiveRange &lr) const;
1523 void PrintLiveBBBit(const LiveRange &li) const;
1524 void PrintLiveRange(const LiveRange &li, const std::string &str) const;
1525 void PrintLiveRanges() const;
1526 void PrintLocalRAInfo(const std::string &str) const;
1527 void PrintBBAssignInfo() const;
1528 void PrintBBs() const;
1529
1530 uint32 MaxIntPhysRegNum() const;
1531 uint32 MaxFloatPhysRegNum() const;
1532 bool IsReservedReg(AArch64reg regNO) const;
1533 void InitFreeRegPool();
1534 void InitCCReg();
1535 bool IsYieldPointReg(regno_t regNO) const;
1536 bool IsUnconcernedReg(regno_t regNO) const;
1537 bool IsUnconcernedReg(const RegOperand ®Opnd) const;
1538 LiveRange *NewLiveRange();
1539 void CalculatePriority(LiveRange &lr) const;
1540 bool CreateLiveRangeHandleLocal(regno_t regNO, const BB &bb, bool isDef);
1541 LiveRange *CreateLiveRangeAllocateAndUpdate(regno_t regNO, const BB &bb, bool isDef, uint32 currId);
1542 void CreateLiveRange(regno_t regNO, const BB &bb, bool isDef, uint32 currPoint, bool updateCount);
1543 bool SetupLiveRangeByOpHandlePhysicalReg(const RegOperand &op, Insn &insn, regno_t regNO, bool isDef);
1544 void SetupLiveRangeByOp(Operand &op, Insn &insn, bool isDef, uint32 &numUses);
1545 void SetupLiveRangeByRegNO(regno_t liveOut, BB &bb, uint32 currPoint);
1546 bool UpdateInsnCntAndSkipUseless(Insn &insn, uint32 &currPoint) const;
1547 void UpdateCallInfo(uint32 bbId, uint32 currPoint, const Insn &insn);
1548 void ClassifyOperand(std::unordered_set<regno_t> &pregs, std::unordered_set<regno_t> &vregs,
1549 const Operand &opnd) const;
1550 void SetOpndConflict(const Insn &insn, bool onlyDef);
1551 void UpdateOpndConflict(const Insn &insn, bool multiDef);
1552 void SetLrMustAssign(const RegOperand *regOpnd);
1553 void SetupMustAssignedLiveRanges(const Insn &insn);
1554 void ComputeLiveRangesForEachDefOperand(Insn &insn, bool &multiDef);
1555 void ComputeLiveRangesForEachUseOperand(Insn &insn);
1556 void ComputeLiveRangesUpdateIfInsnIsCall(const Insn &insn);
1557 void ComputeLiveRangesUpdateLiveUnitInsnRange(BB &bb, uint32 currPoint);
1558 void ComputeLiveRanges();
1559 MemOperand *CreateSpillMem(uint32 spillIdx, SpillMemCheck check);
1560 bool CheckOverlap(uint64 val, uint32 i, LiveRange &lr1, LiveRange &lr2) const;
1561 void CheckInterference(LiveRange &lr1, LiveRange &lr2) const;
1562 void BuildInterferenceGraphSeparateIntFp(std::vector<LiveRange *> &intLrVec, std::vector<LiveRange *> &fpLrVec);
1563 void BuildInterferenceGraph();
1564 void SetBBInfoGlobalAssigned(uint32 bbID, regno_t regNO);
1565 bool HaveAvailableColor(const LiveRange &lr, uint32 num) const;
1566 void Separate();
1567 void SplitAndColorForEachLr(MapleVector<LiveRange *> &targetLrVec);
1568 void SplitAndColor();
1569 void ColorForOptPrologEpilog();
1570 bool IsLocalReg(regno_t regNO) const;
1571 bool IsLocalReg(const LiveRange &lr) const;
1572 void HandleLocalRaDebug(regno_t regNO, const LocalRegAllocator &localRa, bool isInt) const;
1573 void HandleLocalRegAssignment(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1574 void UpdateLocalRegDefUseCount(regno_t regNO, LocalRegAllocator &localRa, bool isDef, bool isInt) const;
1575 void UpdateLocalRegConflict(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1576 void HandleLocalReg(Operand &op, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo, bool isDef, bool isInt);
1577 void LocalRaRegSetEraseReg(LocalRegAllocator &localRa, regno_t regNO) const;
1578 bool LocalRaInitRegSet(LocalRegAllocator &localRa, uint32 bbId);
1579 void LocalRaInitAllocatableRegs(LocalRegAllocator &localRa, uint32 bbId);
1580 void LocalRaForEachDefOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1581 void LocalRaForEachUseOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1582 void LocalRaPrepareBB(BB &bb, LocalRegAllocator &localRa);
1583 void LocalRaFinalAssignment(const LocalRegAllocator &localRa, BBAssignInfo &bbInfo);
1584 void LocalRaDebug(const BB &bb, const LocalRegAllocator &localRa) const;
1585 void LocalRegisterAllocator(bool allocate);
1586 MemOperand *GetSpillOrReuseMem(LiveRange &lr, uint32 regSize, bool &isOutOfRange, Insn &insn, bool isDef);
1587 void SpillOperandForSpillPre(Insn &insn, const Operand &opnd, RegOperand &phyOpnd, uint32 spillIdx, bool needSpill);
1588 void SpillOperandForSpillPost(Insn &insn, const Operand &opnd, RegOperand &phyOpnd, uint32 spillIdx,
1589 bool needSpill);
1590 MemOperand *GetConsistentReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1591 RegType regType);
1592 MemOperand *GetCommonReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1593 RegType regType);
1594 MemOperand *GetReuseMem(uint32 vregNO, uint32 size, RegType regType);
1595 MemOperand *GetSpillMem(uint32 vregNO, bool isDest, Insn &insn, AArch64reg regNO, bool &isOutOfRange) const;
1596 bool SetAvailableSpillReg(std::unordered_set<regno_t> &cannotUseReg, LiveRange &lr, uint64 &usedRegMask);
1597 void CollectCannotUseReg(std::unordered_set<regno_t> &cannotUseReg, const LiveRange &lr, Insn &insn);
1598 regno_t PickRegForSpill(uint64 &usedRegMask, RegType regType, uint32 spillIdx, bool &needSpillLr);
1599 bool SetRegForSpill(LiveRange &lr, Insn &insn, uint32 spillIdx, uint64 &usedRegMask, bool isDef);
1600 bool GetSpillReg(Insn &insn, LiveRange &lr, const uint32 &spillIdx, uint64 &usedRegMask, bool isDef);
1601 RegOperand *GetReplaceOpndForLRA(Insn &insn, const Operand &opnd, uint32 &spillIdx, uint64 &usedRegMask,
1602 bool isDef);
1603 bool EncountPrevRef(const BB &pred, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1604 bool FoundPrevBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1605 bool EncountNextRef(const BB &succ, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1606 bool FoundNextBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1607 bool HavePrevRefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1608 bool HaveNextDefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1609 bool NeedCallerSave(Insn &insn, LiveRange &lr, bool isDef);
1610 RegOperand *GetReplaceOpnd(Insn &insn, const Operand &opnd, uint32 &spillIdx, uint64 &usedRegMask, bool isDef);
1611 void MarkCalleeSaveRegs();
1612 void MarkUsedRegs(Operand &opnd, uint64 &usedRegMask);
1613 uint64 FinalizeRegisterPreprocess(FinalizeRegisterInfo &fInfo, const Insn &insn, bool &needProcess);
1614 void SplitVregAroundLoop(const CGFuncLoops &loop, const std::vector<LiveRange *> &lrs, BB &headerPred, BB &exitSucc,
1615 const std::set<regno_t> &cands);
1616 bool LoopNeedSplit(const CGFuncLoops &loop, std::set<regno_t> &cands);
1617 bool LrGetBadReg(const LiveRange &lr) const;
1618 void AnalysisLoopPressureAndSplit(const CGFuncLoops &loop);
1619 void AnalysisLoop(const CGFuncLoops &);
1620 void OptCallerSave();
1621 void FinalizeRegisters();
1622 void GenerateSpillFillRegs(const Insn &insn);
1623 RegOperand *CreateSpillFillCode(const RegOperand &opnd, Insn &insn, uint32 spillCnt, bool isdef = false);
1624 bool SpillLiveRangeForSpills();
1625
1626 MapleVector<LiveRange *>::iterator GetHighPriorityLr(MapleVector<LiveRange *> &lrSet) const;
1627 void UpdateForbiddenForNeighbors(const LiveRange &lr) const;
1628 void UpdatePregvetoForNeighbors(const LiveRange &lr) const;
1629 regno_t FindColorForLr(const LiveRange &lr) const;
1630 regno_t TryToAssignCallerSave(const LiveRange &lr) const;
1631 bool ShouldUseCallee(LiveRange &lr, const MapleSet<regno_t> &calleeUsed,
1632 const MapleVector<LiveRange *> &delayed) const;
1633 void AddCalleeUsed(regno_t regNO, RegType regType);
1634 bool AssignColorToLr(LiveRange &lr, bool isDelayed = false);
1635 void PruneLrForSplit(LiveRange &lr, BB &bb, bool remove, std::set<CGFuncLoops *, CGFuncLoopCmp> &candidateInLoop,
1636 std::set<CGFuncLoops *, CGFuncLoopCmp> &defInLoop);
1637 bool UseIsUncovered(const BB &bb, const BB &startBB, std::vector<bool> &visitedBB);
1638 void FindUseForSplit(LiveRange &lr, SplitBBInfo &bbInfo, bool &remove,
1639 std::set<CGFuncLoops *, CGFuncLoopCmp> &candidateInLoop,
1640 std::set<CGFuncLoops *, CGFuncLoopCmp> &defInLoop);
1641 void FindBBSharedInSplit(LiveRange &lr, const std::set<CGFuncLoops *, CGFuncLoopCmp> &candidateInLoop,
1642 std::set<CGFuncLoops *, CGFuncLoopCmp> &defInLoop);
1643 void ComputeBBForNewSplit(LiveRange &newLr, LiveRange &oldLr);
1644 void ClearLrBBFlags(const std::set<BB *, SortedBBCmpFunc> &member) const;
1645 void ComputeBBForOldSplit(LiveRange &newLr, LiveRange &oldLr);
1646 bool LrCanBeColored(const LiveRange &lr, const BB &bbAdded, std::unordered_set<regno_t> &conflictRegs);
1647 void MoveLrBBInfo(LiveRange &oldLr, LiveRange &newLr, BB &bb) const;
1648 bool ContainsLoop(const CGFuncLoops &loop, const std::set<CGFuncLoops *, CGFuncLoopCmp> &loops) const;
1649 void GetAllLrMemberLoops(LiveRange &lr, std::set<CGFuncLoops *, CGFuncLoopCmp> &loop);
1650 bool SplitLrShouldSplit(LiveRange &lr);
1651 bool SplitLrFindCandidateLr(LiveRange &lr, LiveRange &newLr, std::unordered_set<regno_t> &conflictRegs);
1652 void SplitLrHandleLoops(LiveRange &lr, LiveRange &newLr, const std::set<CGFuncLoops *, CGFuncLoopCmp> &oldLoops,
1653 const std::set<CGFuncLoops *, CGFuncLoopCmp> &newLoops);
1654 void SplitLrFixNewLrCallsAndRlod(LiveRange &newLr, const std::set<CGFuncLoops *, CGFuncLoopCmp> &origLoops);
1655 void SplitLrFixOrigLrCalls(LiveRange &lr) const;
1656 void SplitLrUpdateInterference(LiveRange &lr);
1657 void SplitLrUpdateRegInfo(const LiveRange &origLr, LiveRange &newLr,
1658 std::unordered_set<regno_t> &conflictRegs) const;
1659 void SplitLrErrorCheckAndDebug(const LiveRange &origLr) const;
1660 void SplitLr(LiveRange &lr);
1661
1662 static constexpr uint16 kMaxUint16 = 0x7fff;
1663
1664 DomAnalysis &domInfo;
1665 MapleVector<BB *> bbVec;
1666 MapleUnorderedSet<regno_t> vregLive;
1667 MapleUnorderedSet<regno_t> pregLive;
1668 MapleMap<regno_t, LiveRange *> lrMap;
1669 MapleVector<LocalRaInfo *> localRegVec; /* local reg info for each bb, no local reg if null */
1670 MapleVector<BBAssignInfo *> bbRegInfo; /* register assignment info for each bb */
1671 MapleVector<LiveRange *> unconstrained;
1672 MapleVector<LiveRange *> unconstrainedPref;
1673 MapleVector<LiveRange *> constrained;
1674 MapleVector<LiveRange *> mustAssigned;
1675 #ifdef OPTIMIZE_FOR_PROLOG
1676 MapleVector<LiveRange *> intDelayed;
1677 MapleVector<LiveRange *> fpDelayed;
1678 #endif /* OPTIMIZE_FOR_PROLOG */
1679 MapleSet<uint32> intCallerRegSet; /* integer caller saved */
1680 MapleSet<uint32> intCalleeRegSet; /* callee */
1681 MapleSet<uint32> intSpillRegSet; /* spill */
1682 MapleSet<uint32> fpCallerRegSet; /* float caller saved */
1683 MapleSet<uint32> fpCalleeRegSet; /* callee */
1684 MapleSet<uint32> fpSpillRegSet; /* spill */
1685 MapleSet<regno_t> intCalleeUsed;
1686 MapleSet<regno_t> fpCalleeUsed;
1687 Bfs *bfs = nullptr;
1688
1689 uint32 bbBuckets = 0; /* size of bit array for bb (each bucket == 64 bits) */
1690 uint32 regBuckets = 0; /* size of bit array for reg (each bucket == 64 bits) */
1691 uint32 intRegNum = 0; /* total available int preg */
1692 uint32 fpRegNum = 0; /* total available fp preg */
1693 uint32 numVregs = 0; /* number of vregs when starting */
1694 regno_t ccReg = 0;
1695 /* For spilling of spill register if there are none available
1696 * Example, all 3 operands spilled
1697 * sp_reg1 -> [spillMemOpnds[1]]
1698 * sp_reg2 -> [spillMemOpnds[2]]
1699 * ld sp_reg1 <- [addr-reg2]
1700 * ld sp_reg2 <- [addr-reg3]
1701 * reg1 <- reg2, reg3 sp_reg1 <- sp_reg1, sp_reg2
1702 * st sp_reg1 -> [addr-reg1]
1703 * sp_reg1 <- [spillMemOpnds[1]]
1704 * sp_reg2 <- [spillMemOpnds[2]]
1705 */
1706 static constexpr size_t kSpillMemOpndNum = 4;
1707 std::array<MemOperand *, kSpillMemOpndNum> spillMemOpnds = {nullptr};
1708 bool operandSpilled[kSpillMemOpndNum];
1709 bool needExtraSpillReg = false;
1710 #ifdef USE_LRA
1711 bool doLRA = true;
1712 #else
1713 bool doLRA = false;
1714 #endif
1715 #ifdef OPTIMIZE_FOR_PROLOG
1716 bool doOptProlog = true;
1717 #else
1718 bool doOptProlog = false;
1719 #endif
1720 bool hasSpill = false;
1721 bool doMultiPass = false;
1722 bool seenFP = false;
1723 };
1724
1725 class CallerSavePre : public CGPre {
1726 public:
CallerSavePre(GraphColorRegAllocator * regAlloc,CGFunc & cgfunc,DomAnalysis & currDom,MemPool & memPool,MemPool & mp2,PreKind kind,uint32 limit)1727 CallerSavePre(GraphColorRegAllocator *regAlloc, CGFunc &cgfunc, DomAnalysis &currDom, MemPool &memPool,
1728 MemPool &mp2, PreKind kind, uint32 limit)
1729 : CGPre(currDom, memPool, mp2, kind, limit),
1730 func(&cgfunc),
1731 regAllocator(regAlloc),
1732 loopHeadBBs(ssaPreAllocator.Adapter())
1733 {
1734 }
1735
1736 ~CallerSavePre() = default;
1737
1738 void ApplySSAPRE();
SetDump(bool val)1739 void SetDump(bool val)
1740 {
1741 dump = val;
1742 }
1743
1744 private:
1745 void CodeMotion();
1746 void UpdateLoadSite(CgOccur *occ);
1747 void CalLoadSites();
1748 void ComputeAvail();
1749 void Rename1();
1750 void ComputeVarAndDfPhis() override;
1751 void BuildWorkList() override;
1752 void DumpWorkCandAndOcc();
1753
GetBB(uint32 id)1754 BB *GetBB(uint32 id) const override
1755 {
1756 return func->GetBBFromID(id);
1757 }
1758
GetPUIdx()1759 PUIdx GetPUIdx() const override
1760 {
1761 return func->GetFunction().GetPuidx();
1762 }
1763
IsLoopHeadBB(uint32 bbId)1764 bool IsLoopHeadBB(uint32 bbId) const
1765 {
1766 return loopHeadBBs.find(bbId) != loopHeadBBs.end();
1767 }
1768 CGFunc *func;
1769 bool dump = false;
1770 LiveRange *workLr = nullptr;
1771 GraphColorRegAllocator *regAllocator;
1772 MapleSet<uint32> loopHeadBBs;
1773 bool beyondLimit = false;
1774 };
1775 } /* namespace maplebe */
1776
1777 #endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H */
1778