1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
17 #define MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H
18 #include "reg_alloc.h"
19 #include "aarch64_operand.h"
20 #include "aarch64_insn.h"
21 #include "aarch64_abi.h"
22 #include "aarch64_cgfunc.h"
23 #include "loop.h"
24 #include "cg_dominance.h"
25 #include "cg_pre.h"
26
27 namespace maplebe {
28 #define RESERVED_REGS
29
30 #define USE_LRA
31 #define USE_SPLIT
32 #undef USE_BB_FREQUENCY
33 #define OPTIMIZE_FOR_PROLOG
34 #define REUSE_SPILLMEM
35 #undef COLOR_SPLIT
36 #define MOVE_COALESCE
37
38 /* for robust test */
39 #undef CONSISTENT_MEMOPND
40 #undef RANDOM_PRIORITY
41
42 constexpr uint32 k32 = sizeof(int) * CHAR_BIT;
43 constexpr uint32 k64 = sizeof(int64) * CHAR_BIT;
44 constexpr uint32 kU64 = sizeof(uint64) * CHAR_BIT;
45
46 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::set<T,Comparator> & set,const T & item)47 inline bool FindNotIn(const std::set<T, Comparator> &set, const T &item)
48 {
49 return set.find(item) == set.end();
50 }
51
52 template <typename T, typename Comparator = std::less<T>>
FindNotIn(const std::unordered_set<T,Comparator> & set,const T & item)53 inline bool FindNotIn(const std::unordered_set<T, Comparator> &set, const T &item)
54 {
55 return set.find(item) == set.end();
56 }
57
58 template <typename T>
FindNotIn(const MapleSet<T> & set,const T & item)59 inline bool FindNotIn(const MapleSet<T> &set, const T &item)
60 {
61 return set.find(item) == set.end();
62 }
63
64 template <typename T>
FindNotIn(const MapleUnorderedSet<T> & set,const T & item)65 inline bool FindNotIn(const MapleUnorderedSet<T> &set, const T &item)
66 {
67 return set.find(item) == set.end();
68 }
69
70 template <typename T>
FindNotIn(const MapleList<T> & list,const T & item)71 inline bool FindNotIn(const MapleList<T> &list, const T &item)
72 {
73 return std::find(list.begin(), list.end(), item) == list.end();
74 }
75
76 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::set<T,Comparator> & set,const T & item)77 inline bool FindIn(const std::set<T, Comparator> &set, const T &item)
78 {
79 return set.find(item) != set.end();
80 }
81
82 template <typename T, typename Comparator = std::less<T>>
FindIn(const std::unordered_set<T,Comparator> & set,const T & item)83 inline bool FindIn(const std::unordered_set<T, Comparator> &set, const T &item)
84 {
85 return set.find(item) != set.end();
86 }
87
88 template <typename T>
FindIn(const MapleSet<T> & set,const T & item)89 inline bool FindIn(const MapleSet<T> &set, const T &item)
90 {
91 return set.find(item) != set.end();
92 }
93
94 template <typename T>
FindIn(const MapleUnorderedSet<T> & set,const T & item)95 inline bool FindIn(const MapleUnorderedSet<T> &set, const T &item)
96 {
97 return set.find(item) != set.end();
98 }
99
100 template <typename T>
FindIn(const MapleList<T> & list,const T & item)101 inline bool FindIn(const MapleList<T> &list, const T &item)
102 {
103 return std::find(list.begin(), list.end(), item) != list.end();
104 }
105
IsBitArrElemSet(const uint64 * vec,const uint32 num)106 inline bool IsBitArrElemSet(const uint64 *vec, const uint32 num)
107 {
108 size_t index = num / kU64;
109 uint64 bit = num % kU64;
110 return vec[index] & (1ULL << bit);
111 }
112
IsBBsetOverlap(const uint64 * vec1,const uint64 * vec2,uint32 bbBuckets)113 inline bool IsBBsetOverlap(const uint64 *vec1, const uint64 *vec2, uint32 bbBuckets)
114 {
115 for (uint32 i = 0; i < bbBuckets; ++i) {
116 if ((vec1[i] & vec2[i]) != 0) {
117 return true;
118 }
119 }
120 return false;
121 }
122
123 /* For each bb, record info pertain to allocation */
124 /*
125 * This is per bb per LR.
126 * LU info is particular to a bb in a LR.
127 */
128 class LiveUnit {
129 public:
130 LiveUnit() = default;
131 ~LiveUnit() = default;
132
133 void PrintLiveUnit() const;
134
GetBegin()135 uint32 GetBegin() const
136 {
137 return begin;
138 }
139
SetBegin(uint32 val)140 void SetBegin(uint32 val)
141 {
142 begin = val;
143 }
144
GetEnd()145 uint32 GetEnd() const
146 {
147 return end;
148 }
149
SetEnd(uint32 end)150 void SetEnd(uint32 end)
151 {
152 this->end = end;
153 }
154
HasCall()155 bool HasCall() const
156 {
157 return hasCall;
158 }
159
SetHasCall(bool hasCall)160 void SetHasCall(bool hasCall)
161 {
162 this->hasCall = hasCall;
163 }
164
GetDefNum()165 uint32 GetDefNum() const
166 {
167 return defNum;
168 }
169
SetDefNum(uint32 defNum)170 void SetDefNum(uint32 defNum)
171 {
172 this->defNum = defNum;
173 }
174
IncDefNum()175 void IncDefNum()
176 {
177 ++defNum;
178 }
179
GetUseNum()180 uint32 GetUseNum() const
181 {
182 return useNum;
183 }
184
SetUseNum(uint32 useNum)185 void SetUseNum(uint32 useNum)
186 {
187 this->useNum = useNum;
188 }
189
IncUseNum()190 void IncUseNum()
191 {
192 ++useNum;
193 }
194
NeedReload()195 bool NeedReload() const
196 {
197 return needReload;
198 }
199
SetNeedReload(bool needReload)200 void SetNeedReload(bool needReload)
201 {
202 this->needReload = needReload;
203 }
204
NeedRestore()205 bool NeedRestore() const
206 {
207 return needRestore;
208 }
209
SetNeedRestore(bool needRestore)210 void SetNeedRestore(bool needRestore)
211 {
212 this->needRestore = needRestore;
213 }
214
215 private:
216 uint32 begin = 0; /* first encounter in bb */
217 uint32 end = 0; /* last encounter in bb */
218 bool hasCall = false; /* bb has a call */
219 uint32 defNum = 0;
220 uint32 useNum = 0; /* used for priority calculation */
221 bool needReload = false;
222 bool needRestore = false;
223 };
224
225 struct SortedBBCmpFunc {
operatorSortedBBCmpFunc226 bool operator()(const BB *lhs, const BB *rhs) const
227 {
228 return (lhs->GetLevel() < rhs->GetLevel());
229 }
230 };
231
232 enum refType : uint8 {
233 kIsUse = 0x1,
234 kIsDef = 0x2,
235 kIsCall = 0x4,
236 };
237
238 /* LR is for each global vreg. */
239 class LiveRange {
240 public:
LiveRange(MapleAllocator & allocator)241 explicit LiveRange(MapleAllocator &allocator)
242 : lrAlloca(&allocator),
243 pregveto(allocator.Adapter()),
244 callDef(allocator.Adapter()),
245 forbidden(allocator.Adapter()),
246 prefs(allocator.Adapter()),
247 refMap(allocator.Adapter()),
248 luMap(allocator.Adapter())
249 {
250 }
251
252 ~LiveRange() = default;
253
GetRegNO()254 regno_t GetRegNO() const
255 {
256 return regNO;
257 }
258
SetRegNO(regno_t val)259 void SetRegNO(regno_t val)
260 {
261 regNO = val;
262 }
263
GetID()264 uint32 GetID() const
265 {
266 return id;
267 }
268
SetID(uint32 id)269 void SetID(uint32 id)
270 {
271 this->id = id;
272 }
273
GetAssignedRegNO()274 regno_t GetAssignedRegNO() const
275 {
276 return assignedRegNO;
277 }
278
SetAssignedRegNO(regno_t val)279 void SetAssignedRegNO(regno_t val)
280 {
281 assignedRegNO = val;
282 }
283
GetNumCall()284 uint32 GetNumCall() const
285 {
286 return numCall;
287 }
288
SetNumCall(uint32 num)289 void SetNumCall(uint32 num)
290 {
291 numCall = num;
292 }
293
IncNumCall()294 void IncNumCall()
295 {
296 ++numCall;
297 }
298
GetRegType()299 RegType GetRegType() const
300 {
301 return regType;
302 }
303
SetRegType(RegType regType)304 void SetRegType(RegType regType)
305 {
306 this->regType = regType;
307 }
308
GetPriority()309 float GetPriority() const
310 {
311 return priority;
312 }
313
SetPriority(float priority)314 void SetPriority(float priority)
315 {
316 this->priority = priority;
317 }
318
IsMustAssigned()319 bool IsMustAssigned() const
320 {
321 return mustAssigned;
322 }
323
SetMustAssigned()324 void SetMustAssigned()
325 {
326 mustAssigned = true;
327 }
328
SetBBBuckets(uint32 bucketNum)329 void SetBBBuckets(uint32 bucketNum)
330 {
331 bbBuckets = bucketNum;
332 }
333
SetRegBuckets(uint32 bucketNum)334 void SetRegBuckets(uint32 bucketNum)
335 {
336 regBuckets = bucketNum;
337 }
338
GetNumBBMembers()339 uint32 GetNumBBMembers() const
340 {
341 return numBBMembers;
342 }
343
IncNumBBMembers()344 void IncNumBBMembers()
345 {
346 ++numBBMembers;
347 }
348
DecNumBBMembers()349 void DecNumBBMembers()
350 {
351 --numBBMembers;
352 }
353
InitBBMember(MemPool & memPool,size_t size)354 void InitBBMember(MemPool &memPool, size_t size)
355 {
356 bbMember = memPool.NewArray<uint64>(size);
357 errno_t ret = memset_s(bbMember, size * sizeof(uint64), 0, size * sizeof(uint64));
358 CHECK_FATAL(ret == EOK, "call memset_s failed");
359 }
360
GetBBMember()361 uint64 *GetBBMember()
362 {
363 return bbMember;
364 }
365
GetBBMember()366 const uint64 *GetBBMember() const
367 {
368 return bbMember;
369 }
370
GetBBMemberElem(int32 index)371 uint64 GetBBMemberElem(int32 index) const
372 {
373 return bbMember[index];
374 }
375
SetBBMemberElem(int32 index,uint64 elem)376 void SetBBMemberElem(int32 index, uint64 elem)
377 {
378 bbMember[index] = elem;
379 }
380
SetMemberBitArrElem(uint32 bbID)381 void SetMemberBitArrElem(uint32 bbID)
382 {
383 uint32 index = bbID / kU64;
384 uint64 bit = bbID % kU64;
385 uint64 mask = 1ULL << bit;
386 if ((GetBBMemberElem(index) & mask) == 0) {
387 IncNumBBMembers();
388 SetBBMemberElem(index, GetBBMemberElem(index) | mask);
389 }
390 }
391
UnsetMemberBitArrElem(uint32 bbID)392 void UnsetMemberBitArrElem(uint32 bbID)
393 {
394 uint32 index = bbID / kU64;
395 uint64 bit = bbID % kU64;
396 uint64 mask = 1ULL << bit;
397 if ((GetBBMemberElem(index) & mask) != 0) {
398 DecNumBBMembers();
399 SetBBMemberElem(index, GetBBMemberElem(index) & (~mask));
400 }
401 }
402
SetConflictBitArrElem(regno_t regNO)403 void SetConflictBitArrElem(regno_t regNO)
404 {
405 uint32 index = regNO / kU64;
406 uint64 bit = regNO % kU64;
407 uint64 mask = 1ULL << bit;
408 if ((GetBBConflictElem(index) & mask) == 0) {
409 IncNumBBConflicts();
410 SetBBConflictElem(index, GetBBConflictElem(index) | mask);
411 }
412 }
413
UnsetConflictBitArrElem(regno_t regNO)414 void UnsetConflictBitArrElem(regno_t regNO)
415 {
416 uint32 index = regNO / kU64;
417 uint64 bit = regNO % kU64;
418 uint64 mask = 1ULL << bit;
419 if ((GetBBConflictElem(index) & mask) != 0) {
420 DecNumBBConflicts();
421 SetBBConflictElem(index, GetBBConflictElem(index) & (~mask));
422 }
423 }
424
InitPregveto()425 void InitPregveto()
426 {
427 pregveto.clear();
428 pregveto.resize(kMaxRegNum);
429 callDef.clear();
430 callDef.resize(kMaxRegNum);
431 }
432
GetPregveto(regno_t regno)433 bool GetPregveto(regno_t regno) const
434 {
435 return pregveto[regno];
436 }
437
GetPregvetoSize()438 size_t GetPregvetoSize() const
439 {
440 return numPregveto;
441 }
442
InsertElemToPregveto(regno_t regno)443 void InsertElemToPregveto(regno_t regno)
444 {
445 if (!pregveto[regno]) {
446 pregveto[regno] = true;
447 ++numPregveto;
448 }
449 }
450
GetCallDef(regno_t regno)451 bool GetCallDef(regno_t regno) const
452 {
453 return callDef[regno];
454 }
455
InsertElemToCallDef(regno_t regno)456 void InsertElemToCallDef(regno_t regno)
457 {
458 if (!callDef[regno]) {
459 callDef[regno] = true;
460 ++numCallDef;
461 }
462 }
463
SetCrossCall()464 void SetCrossCall()
465 {
466 crossCall = true;
467 }
468
GetCrossCall()469 bool GetCrossCall() const
470 {
471 return crossCall;
472 }
473
InitForbidden()474 void InitForbidden()
475 {
476 forbidden.clear();
477 forbidden.resize(kMaxRegNum);
478 }
479
GetForbidden()480 const MapleVector<bool> &GetForbidden() const
481 {
482 return forbidden;
483 }
484
GetForbidden(regno_t regno)485 bool GetForbidden(regno_t regno) const
486 {
487 return forbidden[regno];
488 }
489
GetForbiddenSize()490 size_t GetForbiddenSize() const
491 {
492 return numForbidden;
493 }
494
InsertElemToForbidden(regno_t regno)495 void InsertElemToForbidden(regno_t regno)
496 {
497 if (!forbidden[regno]) {
498 forbidden[regno] = true;
499 ++numForbidden;
500 }
501 }
502
EraseElemFromForbidden(regno_t regno)503 void EraseElemFromForbidden(regno_t regno)
504 {
505 if (forbidden[regno]) {
506 forbidden[regno] = false;
507 --numForbidden;
508 }
509 }
510
ClearForbidden()511 void ClearForbidden()
512 {
513 forbidden.clear();
514 }
515
GetNumBBConflicts()516 uint32 GetNumBBConflicts() const
517 {
518 return numBBConflicts;
519 }
520
IncNumBBConflicts()521 void IncNumBBConflicts()
522 {
523 ++numBBConflicts;
524 }
525
DecNumBBConflicts()526 void DecNumBBConflicts()
527 {
528 --numBBConflicts;
529 }
530
InitBBConflict(MemPool & memPool,size_t size)531 void InitBBConflict(MemPool &memPool, size_t size)
532 {
533 bbConflict = memPool.NewArray<uint64>(size);
534 errno_t ret = memset_s(bbConflict, size * sizeof(uint64), 0, size * sizeof(uint64));
535 CHECK_FATAL(ret == EOK, "call memset_s failed");
536 }
537
GetBBConflict()538 const uint64 *GetBBConflict() const
539 {
540 return bbConflict;
541 }
542
GetBBConflictElem(int32 index)543 uint64 GetBBConflictElem(int32 index) const
544 {
545 DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
546 return bbConflict[index];
547 }
548
SetBBConflictElem(int32 index,uint64 elem)549 void SetBBConflictElem(int32 index, uint64 elem)
550 {
551 DEBUG_ASSERT(static_cast<uint32>(index) < regBuckets, "out of bbConflict");
552 bbConflict[index] = elem;
553 }
554
SetOldConflict(uint64 * conflict)555 void SetOldConflict(uint64 *conflict)
556 {
557 oldConflict = conflict;
558 }
559
GetOldConflict()560 const uint64 *GetOldConflict() const
561 {
562 return oldConflict;
563 }
564
GetPrefs()565 const MapleSet<regno_t> &GetPrefs() const
566 {
567 return prefs;
568 }
569
InsertElemToPrefs(regno_t regNO)570 void InsertElemToPrefs(regno_t regNO)
571 {
572 (void)prefs.insert(regNO);
573 }
574
GetRefs()575 const MapleMap<uint32, MapleMap<uint32, uint32> *> GetRefs() const
576 {
577 return refMap;
578 }
579
GetRefs(uint32 bbId)580 const MapleMap<uint32, uint32> GetRefs(uint32 bbId) const
581 {
582 return *(refMap.find(bbId)->second);
583 }
584
AddRef(uint32 bbId,uint32 pos,uint32 mark)585 void AddRef(uint32 bbId, uint32 pos, uint32 mark)
586 {
587 if (refMap.find(bbId) == refMap.end()) {
588 auto point = lrAlloca->New<MapleMap<uint32, uint32>>(lrAlloca->Adapter());
589 (void)point->emplace(std::pair<uint32, uint32>(pos, mark));
590 (void)refMap.emplace(std::pair<uint32, MapleMap<uint32, uint32> *>(bbId, point));
591 } else {
592 auto &bbPoint = (refMap.find(bbId))->second;
593 if (bbPoint->find(pos) == bbPoint->end()) {
594 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, mark));
595 } else {
596 auto posVal = bbPoint->find(pos)->second;
597 (void)bbPoint->erase(bbPoint->find(pos));
598 (void)bbPoint->emplace(std::pair<uint32, uint32>(pos, posVal | mark));
599 }
600 }
601 }
602
GetLuMap()603 const MapleMap<uint32, LiveUnit *> &GetLuMap() const
604 {
605 return luMap;
606 }
607
FindInLuMap(uint32 index)608 MapleMap<uint32, LiveUnit *>::iterator FindInLuMap(uint32 index)
609 {
610 return luMap.find(index);
611 }
612
EndOfLuMap()613 MapleMap<uint32, LiveUnit *>::iterator EndOfLuMap()
614 {
615 return luMap.end();
616 }
617
EraseLuMap(MapleMap<uint32,LiveUnit * >::iterator it)618 MapleMap<uint32, LiveUnit *>::iterator EraseLuMap(MapleMap<uint32, LiveUnit *>::iterator it)
619 {
620 return luMap.erase(it);
621 }
622
SetElemToLuMap(uint32 key,LiveUnit & value)623 void SetElemToLuMap(uint32 key, LiveUnit &value)
624 {
625 luMap[key] = &value;
626 }
627
GetLiveUnitFromLuMap(uint32 key)628 LiveUnit *GetLiveUnitFromLuMap(uint32 key)
629 {
630 return luMap[key];
631 }
632
GetLiveUnitFromLuMap(uint32 key)633 const LiveUnit *GetLiveUnitFromLuMap(uint32 key) const
634 {
635 auto it = luMap.find(key);
636 DEBUG_ASSERT(it != luMap.end(), "can't find live unit");
637 return it->second;
638 }
639
GetSplitLr()640 const LiveRange *GetSplitLr() const
641 {
642 return splitLr;
643 }
644
SetSplitLr(LiveRange & lr)645 void SetSplitLr(LiveRange &lr)
646 {
647 splitLr = &lr;
648 }
649
650 #ifdef OPTIMIZE_FOR_PROLOG
GetNumDefs()651 uint32 GetNumDefs() const
652 {
653 return numDefs;
654 }
655
IncNumDefs()656 void IncNumDefs()
657 {
658 ++numDefs;
659 }
660
SetNumDefs(uint32 val)661 void SetNumDefs(uint32 val)
662 {
663 numDefs = val;
664 }
665
GetNumUses()666 uint32 GetNumUses() const
667 {
668 return numUses;
669 }
670
IncNumUses()671 void IncNumUses()
672 {
673 ++numUses;
674 }
675
SetNumUses(uint32 val)676 void SetNumUses(uint32 val)
677 {
678 numUses = val;
679 }
680
GetFrequency()681 uint32 GetFrequency() const
682 {
683 return frequency;
684 }
685
SetFrequency(uint32 frequency)686 void SetFrequency(uint32 frequency)
687 {
688 this->frequency = frequency;
689 }
690 #endif /* OPTIMIZE_FOR_PROLOG */
691
GetSpillMem()692 MemOperand *GetSpillMem()
693 {
694 return spillMem;
695 }
696
GetSpillMem()697 const MemOperand *GetSpillMem() const
698 {
699 return spillMem;
700 }
701
SetSpillMem(MemOperand & memOpnd)702 void SetSpillMem(MemOperand &memOpnd)
703 {
704 spillMem = &memOpnd;
705 }
706
GetSpillReg()707 regno_t GetSpillReg() const
708 {
709 return spillReg;
710 }
711
SetSpillReg(regno_t spillReg)712 void SetSpillReg(regno_t spillReg)
713 {
714 this->spillReg = spillReg;
715 }
716
GetSpillSize()717 uint32 GetSpillSize() const
718 {
719 return spillSize;
720 }
721
SetSpillSize(uint32 size)722 void SetSpillSize(uint32 size)
723 {
724 spillSize = size;
725 }
726
IsSpilled()727 bool IsSpilled() const
728 {
729 return spilled;
730 }
731
SetSpilled(bool spill)732 void SetSpilled(bool spill)
733 {
734 spilled = spill;
735 }
736
HasDefUse()737 bool HasDefUse() const
738 {
739 return hasDefUse;
740 }
741
SetDefUse()742 void SetDefUse()
743 {
744 hasDefUse = true;
745 }
746
GetProcessed()747 bool GetProcessed() const
748 {
749 return proccessed;
750 }
751
SetProcessed()752 void SetProcessed()
753 {
754 proccessed = true;
755 }
756
IsNonLocal()757 bool IsNonLocal() const
758 {
759 return isNonLocal;
760 }
761
SetIsNonLocal(bool isNonLocal)762 void SetIsNonLocal(bool isNonLocal)
763 {
764 this->isNonLocal = isNonLocal;
765 }
766
SetRematLevel(uint32 val)767 void SetRematLevel(uint32 val)
768 {
769 this->rematLevel = val;
770 }
771
GetRematLevel()772 uint32 GetRematLevel() const
773 {
774 return this->rematLevel;
775 }
776
GetOp()777 Opcode GetOp() const
778 {
779 return op;
780 }
781
GetRematSymbol()782 const MIRSymbol *GetRematSymbol() const
783 {
784 DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat symbol is invalid");
785 return rematInfo.sym;
786 }
787
GetRematFieldID()788 FieldID GetRematFieldID() const
789 {
790 DEBUG_ASSERT(op == OP_dread || op == OP_addrof, "Remat field ID is invalid");
791 return fieldID;
792 }
793
SetRematerializable(const MIRConst * c)794 void SetRematerializable(const MIRConst *c)
795 {
796 op = OP_constval;
797 rematInfo.mirConst = c;
798 }
799
SetRematerializable(Opcode opcode,const MIRSymbol * symbol,FieldID fieldId,bool addrUp)800 void SetRematerializable(Opcode opcode, const MIRSymbol *symbol, FieldID fieldId, bool addrUp)
801 {
802 this->op = opcode;
803 rematInfo.sym = symbol;
804 this->fieldID = fieldId;
805 this->addrUpper = addrUp;
806 }
807
CopyRematerialization(const LiveRange & lr)808 void CopyRematerialization(const LiveRange &lr)
809 {
810 op = lr.op;
811 rematInfo = lr.rematInfo;
812 fieldID = lr.fieldID;
813 }
814
815 private:
816 MapleAllocator *lrAlloca;
817 regno_t regNO = 0;
818 uint32 id = 0; /* for priority tie breaker */
819 regno_t assignedRegNO = 0; /* color assigned */
820 uint32 numCall = 0;
821 RegType regType = kRegTyUndef;
822 float priority = 0.0;
823 bool mustAssigned = false;
824 uint32 bbBuckets = 0; /* size of bit array for bb (each bucket == 64 bits) */
825 uint32 regBuckets = 0; /* size of bit array for reg (each bucket == 64 bits) */
826 uint32 numBBMembers = 0; /* number of bits set in bbMember */
827 uint64 *bbMember = nullptr; /* Same as smember, but use bit array */
828
829 MapleVector<bool> pregveto; /* pregs cannot be assigned -- SplitLr may clear forbidden */
830 MapleVector<bool> callDef; /* pregs cannot be assigned -- SplitLr may clear forbidden */
831 MapleVector<bool> forbidden; /* pregs cannot be assigned */
832 uint32 numPregveto = 0;
833 uint32 numCallDef = 0;
834 uint32 numForbidden = 0;
835 bool crossCall = false;
836
837 uint32 numBBConflicts = 0; /* number of bits set in bbConflict */
838 uint64 *bbConflict = nullptr; /* vreg interference from graph neighbors (bit) */
839 uint64 *oldConflict = nullptr;
840 MapleSet<regno_t> prefs; /* pregs that prefer */
841 MapleMap<uint32, MapleMap<uint32, uint32> *> refMap;
842 MapleMap<uint32, LiveUnit *> luMap; /* info for each bb */
843 LiveRange *splitLr = nullptr; /* The 1st part of the split */
844 #ifdef OPTIMIZE_FOR_PROLOG
845 uint32 numDefs = 0;
846 uint32 numUses = 0;
847 uint32 frequency = 0;
848 #endif /* OPTIMIZE_FOR_PROLOG */
849 MemOperand *spillMem = nullptr; /* memory operand used for spill, if any */
850 regno_t spillReg = 0; /* register operand for spill at current point */
851 uint32 spillSize = 0; /* 32 or 64 bit spill */
852 bool spilled = false; /* color assigned */
853 bool hasDefUse = false; /* has regDS */
854 bool proccessed = false;
855 bool isNonLocal = false;
856 uint32 rematLevel = 0;
857 Opcode op = OP_undef; /* OP_constval, OP_addrof or OP_dread if rematerializable */
858 union RematInfo {
859 const MIRConst *mirConst;
860 const MIRSymbol *sym;
861 } rematInfo; /* info for rematerializing value */
862 FieldID fieldID = 0; /* used only when op is OP_addrof or OP_dread */
863 bool addrUpper = false; /* indicates the upper bits of an addrof */
864 };
865
866 /* One per bb, to communicate local usage to global RA */
867 class LocalRaInfo {
868 public:
LocalRaInfo(MapleAllocator & allocator)869 explicit LocalRaInfo(MapleAllocator &allocator) : defCnt(allocator.Adapter()), useCnt(allocator.Adapter()) {}
870
871 ~LocalRaInfo() = default;
872
GetDefCnt()873 const MapleMap<regno_t, uint16> &GetDefCnt() const
874 {
875 return defCnt;
876 }
877
GetDefCntElem(regno_t regNO)878 uint16 GetDefCntElem(regno_t regNO)
879 {
880 return defCnt[regNO];
881 }
882
SetDefCntElem(regno_t key,uint16 value)883 void SetDefCntElem(regno_t key, uint16 value)
884 {
885 defCnt[key] = value;
886 }
887
GetUseCnt()888 const MapleMap<regno_t, uint16> &GetUseCnt() const
889 {
890 return useCnt;
891 }
892
GetUseCntElem(regno_t regNO)893 uint16 GetUseCntElem(regno_t regNO)
894 {
895 return useCnt[regNO];
896 }
897
SetUseCntElem(regno_t key,uint16 value)898 void SetUseCntElem(regno_t key, uint16 value)
899 {
900 useCnt[key] = value;
901 }
902
903 private:
904 MapleMap<regno_t, uint16> defCnt;
905 MapleMap<regno_t, uint16> useCnt;
906 };
907
908 /* For each bb, record info pertain to allocation */
909 class BBAssignInfo {
910 public:
BBAssignInfo(MapleAllocator & allocator)911 explicit BBAssignInfo(MapleAllocator &allocator) : globalsAssigned(allocator.Adapter()), regMap(allocator.Adapter())
912 {
913 }
914
915 ~BBAssignInfo() = default;
916
GetIntLocalRegsNeeded()917 uint32 GetIntLocalRegsNeeded() const
918 {
919 return intLocalRegsNeeded;
920 }
921
SetIntLocalRegsNeeded(uint32 num)922 void SetIntLocalRegsNeeded(uint32 num)
923 {
924 intLocalRegsNeeded = num;
925 }
926
GetFpLocalRegsNeeded()927 uint32 GetFpLocalRegsNeeded() const
928 {
929 return fpLocalRegsNeeded;
930 }
931
SetFpLocalRegsNeeded(uint32 num)932 void SetFpLocalRegsNeeded(uint32 num)
933 {
934 fpLocalRegsNeeded = num;
935 }
936
InitGlobalAssigned()937 void InitGlobalAssigned()
938 {
939 globalsAssigned.clear();
940 globalsAssigned.resize(kMaxRegNum);
941 }
942
GetGlobalsAssigned(regno_t regNO)943 bool GetGlobalsAssigned(regno_t regNO) const
944 {
945 return globalsAssigned[regNO];
946 }
947
InsertElemToGlobalsAssigned(regno_t regNO)948 void InsertElemToGlobalsAssigned(regno_t regNO)
949 {
950 globalsAssigned[regNO] = true;
951 }
952
EraseElemToGlobalsAssigned(regno_t regNO)953 void EraseElemToGlobalsAssigned(regno_t regNO)
954 {
955 globalsAssigned[regNO] = false;
956 }
957
GetRegMap()958 const MapleMap<regno_t, regno_t> &GetRegMap() const
959 {
960 return regMap;
961 }
962
HasRegMap(regno_t regNOKey)963 bool HasRegMap(regno_t regNOKey) const
964 {
965 return (regMap.find(regNOKey) != regMap.end());
966 }
967
GetRegMapElem(regno_t regNO)968 regno_t GetRegMapElem(regno_t regNO)
969 {
970 return regMap[regNO];
971 }
972
SetRegMapElem(regno_t regNOKey,regno_t regNOValue)973 void SetRegMapElem(regno_t regNOKey, regno_t regNOValue)
974 {
975 regMap[regNOKey] = regNOValue;
976 }
977
978 private:
979 uint32 intLocalRegsNeeded = 0; /* num local reg needs for each bb */
980 uint32 fpLocalRegsNeeded = 0; /* num local reg needs for each bb */
981 MapleVector<bool> globalsAssigned; /* globals used in a bb */
982 MapleMap<regno_t, regno_t> regMap; /* local vreg to preg mapping */
983 };
984
985 class FinalizeRegisterInfo {
986 public:
FinalizeRegisterInfo(MapleAllocator & allocator)987 explicit FinalizeRegisterInfo(MapleAllocator &allocator)
988 : defOperands(allocator.Adapter()),
989 defIdx(allocator.Adapter()),
990 useOperands(allocator.Adapter()),
991 useIdx(allocator.Adapter())
992 {
993 }
994
995 ~FinalizeRegisterInfo() = default;
ClearInfo()996 void ClearInfo()
997 {
998 memOperandIdx = 0;
999 baseOperand = nullptr;
1000 offsetOperand = nullptr;
1001 defOperands.clear();
1002 defIdx.clear();
1003 useOperands.clear();
1004 useIdx.clear();
1005 }
1006
SetBaseOperand(Operand & opnd,const int32 idx)1007 void SetBaseOperand(Operand &opnd, const int32 idx)
1008 {
1009 baseOperand = &opnd;
1010 memOperandIdx = idx;
1011 }
1012
SetOffsetOperand(Operand & opnd)1013 void SetOffsetOperand(Operand &opnd)
1014 {
1015 offsetOperand = &opnd;
1016 }
1017
SetDefOperand(Operand & opnd,const int32 idx)1018 void SetDefOperand(Operand &opnd, const int32 idx)
1019 {
1020 defOperands.emplace_back(&opnd);
1021 defIdx.emplace_back(idx);
1022 }
1023
SetUseOperand(Operand & opnd,const int32 idx)1024 void SetUseOperand(Operand &opnd, const int32 idx)
1025 {
1026 useOperands.emplace_back(&opnd);
1027 useIdx.emplace_back(idx);
1028 }
1029
GetMemOperandIdx()1030 int32 GetMemOperandIdx() const
1031 {
1032 return memOperandIdx;
1033 }
1034
GetBaseOperand()1035 const Operand *GetBaseOperand() const
1036 {
1037 return baseOperand;
1038 }
1039
GetOffsetOperand()1040 const Operand *GetOffsetOperand() const
1041 {
1042 return offsetOperand;
1043 }
1044
GetDefOperandsSize()1045 size_t GetDefOperandsSize() const
1046 {
1047 return defOperands.size();
1048 }
1049
GetDefOperandsElem(size_t index)1050 const Operand *GetDefOperandsElem(size_t index) const
1051 {
1052 return defOperands[index];
1053 }
1054
GetDefIdxElem(size_t index)1055 int32 GetDefIdxElem(size_t index) const
1056 {
1057 return defIdx[index];
1058 }
1059
GetUseOperandsSize()1060 size_t GetUseOperandsSize() const
1061 {
1062 return useOperands.size();
1063 }
1064
GetUseOperandsElem(size_t index)1065 const Operand *GetUseOperandsElem(size_t index) const
1066 {
1067 return useOperands[index];
1068 }
1069
GetUseIdxElem(size_t index)1070 int32 GetUseIdxElem(size_t index) const
1071 {
1072 return useIdx[index];
1073 }
1074
1075 private:
1076 int32 memOperandIdx = 0;
1077 Operand *baseOperand = nullptr;
1078 Operand *offsetOperand = nullptr;
1079 MapleVector<Operand *> defOperands;
1080 MapleVector<int32> defIdx;
1081 MapleVector<Operand *> useOperands;
1082 MapleVector<int32> useIdx;
1083 };
1084
1085 class LocalRegAllocator {
1086 public:
LocalRegAllocator(CGFunc & cgFunc,MapleAllocator & allocator)1087 LocalRegAllocator(CGFunc &cgFunc, MapleAllocator &allocator)
1088 : intRegAssignmentMap(allocator.Adapter()),
1089 fpRegAssignmentMap(allocator.Adapter()),
1090 useInfo(allocator.Adapter()),
1091 defInfo(allocator.Adapter())
1092 {
1093 buckets = (cgFunc.GetMaxRegNum() / kU64) + 1;
1094 intRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1095 fpRegAssigned = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1096 intRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1097 fpRegSpilled = cgFunc.GetMemoryPool()->NewArray<uint64>(buckets);
1098 }
1099
1100 ~LocalRegAllocator() = default;
1101
ClearLocalRaInfo()1102 void ClearLocalRaInfo()
1103 {
1104 ClearBitArrElement(intRegAssigned);
1105 ClearBitArrElement(fpRegAssigned);
1106 intRegAssignmentMap.clear();
1107 fpRegAssignmentMap.clear();
1108 intPregUsed = 0;
1109 fpPregUsed = 0;
1110 ClearBitArrElement(intRegSpilled);
1111 ClearBitArrElement(fpRegSpilled);
1112 numIntPregUsed = 0;
1113 numFpPregUsed = 0;
1114 }
1115
RegBaseUpdate(bool isInt)1116 regno_t RegBaseUpdate(bool isInt) const
1117 {
1118 return isInt ? 0 : V0 - R0;
1119 }
1120
IsInRegAssigned(regno_t regNO,bool isInt)1121 bool IsInRegAssigned(regno_t regNO, bool isInt) const
1122 {
1123 uint64 *regAssigned = nullptr;
1124 if (isInt) {
1125 regAssigned = intRegAssigned;
1126 } else {
1127 regAssigned = fpRegAssigned;
1128 }
1129 return IsBitArrElemSet(regAssigned, regNO);
1130 ;
1131 }
1132
SetRegAssigned(regno_t regNO,bool isInt)1133 void SetRegAssigned(regno_t regNO, bool isInt) const
1134 {
1135 if (isInt) {
1136 SetBitArrElement(intRegAssigned, regNO);
1137 } else {
1138 SetBitArrElement(fpRegAssigned, regNO);
1139 }
1140 }
1141
GetRegAssignmentItem(bool isInt,regno_t regKey)1142 regno_t GetRegAssignmentItem(bool isInt, regno_t regKey)
1143 {
1144 return isInt ? intRegAssignmentMap[regKey] : fpRegAssignmentMap[regKey];
1145 }
1146
SetRegAssignmentMap(bool isInt,regno_t regKey,regno_t regValue)1147 void SetRegAssignmentMap(bool isInt, regno_t regKey, regno_t regValue)
1148 {
1149 if (isInt) {
1150 intRegAssignmentMap[regKey] = regValue;
1151 } else {
1152 fpRegAssignmentMap[regKey] = regValue;
1153 }
1154 }
1155
1156 /* only for HandleLocalRaDebug */
GetPregUsed(bool isInt)1157 uint64 GetPregUsed(bool isInt) const
1158 {
1159 if (isInt) {
1160 return intPregUsed;
1161 } else {
1162 return fpPregUsed;
1163 }
1164 }
1165
SetPregUsed(regno_t regNO,bool isInt)1166 void SetPregUsed(regno_t regNO, bool isInt)
1167 {
1168 uint64 mask = 0;
1169 if (isInt) {
1170 mask = 1ULL << (regNO - R0);
1171 if ((intPregUsed & mask) == 0) {
1172 ++numIntPregUsed;
1173 intPregUsed |= mask;
1174 }
1175 } else {
1176 mask = 1ULL << (regNO - V0);
1177 if ((fpPregUsed & mask) == 0) {
1178 ++numFpPregUsed;
1179 fpPregUsed |= mask;
1180 }
1181 }
1182 }
1183
isInRegSpilled(regno_t regNO,bool isInt)1184 bool isInRegSpilled(regno_t regNO, bool isInt) const
1185 {
1186 bool isSet;
1187 if (isInt) {
1188 isSet = IsBitArrElemSet(intRegSpilled, regNO);
1189 } else {
1190 isSet = IsBitArrElemSet(fpRegSpilled, regNO);
1191 }
1192 return isSet;
1193 }
1194
SetRegSpilled(regno_t regNO,bool isInt)1195 void SetRegSpilled(regno_t regNO, bool isInt) const
1196 {
1197 if (isInt) {
1198 SetBitArrElement(intRegSpilled, regNO);
1199 } else {
1200 SetBitArrElement(fpRegSpilled, regNO);
1201 }
1202 }
1203
GetPregs(bool isInt)1204 uint64 GetPregs(bool isInt) const
1205 {
1206 if (isInt) {
1207 return intPregs;
1208 } else {
1209 return fpPregs;
1210 }
1211 }
1212
SetPregs(regno_t regNO,bool isInt)1213 void SetPregs(regno_t regNO, bool isInt)
1214 {
1215 if (isInt) {
1216 intPregs |= 1ULL << (regNO - RegBaseUpdate(true));
1217 } else {
1218 fpPregs |= 1ULL << (regNO - RegBaseUpdate(false));
1219 }
1220 }
1221
ClearPregs(regno_t regNO,bool isInt)1222 void ClearPregs(regno_t regNO, bool isInt)
1223 {
1224 if (isInt) {
1225 intPregs &= ~(1ULL << (regNO - RegBaseUpdate(true)));
1226 } else {
1227 DEBUG_ASSERT(regNO >= RegBaseUpdate(false), "regNO - RegBaseUpdate(false) unsigned");
1228 fpPregs &= ~(1ULL << (regNO - RegBaseUpdate(false)));
1229 }
1230 }
1231
IsPregAvailable(regno_t regNO,bool isInt)1232 bool IsPregAvailable(regno_t regNO, bool isInt) const
1233 {
1234 bool isAvailable;
1235 if (isInt) {
1236 isAvailable = intPregs & (1ULL << (regNO - RegBaseUpdate(true)));
1237 } else {
1238 DEBUG_ASSERT(regNO >= RegBaseUpdate(false), "regNO - RegBaseUpdate(false) unsigned");
1239 isAvailable = fpPregs & (1ULL << (regNO - RegBaseUpdate(false)));
1240 }
1241 return isAvailable;
1242 }
1243
InitPregs(uint32 intMax,uint32 fpMax,bool hasYield,const MapleSet<uint32> & intSpillRegSet,const MapleSet<uint32> & fpSpillRegSet)1244 void InitPregs(uint32 intMax, uint32 fpMax, bool hasYield, const MapleSet<uint32> &intSpillRegSet,
1245 const MapleSet<uint32> &fpSpillRegSet)
1246 {
1247 uint32 intBase = R0;
1248 uint32 fpBase = V0;
1249 intPregs = (1ULL << (intMax + 1)) - 1;
1250 fpPregs = (1ULL << (((fpMax + 1) + fpBase) - RegBaseUpdate(false))) - 1;
1251 for (uint32 regNO : intSpillRegSet) {
1252 ClearPregs(regNO + intBase, true);
1253 }
1254 for (uint32 regNO : fpSpillRegSet) {
1255 ClearPregs(regNO + fpBase, false);
1256 }
1257 if (hasYield) {
1258 ClearPregs(RYP, true);
1259 }
1260 #ifdef RESERVED_REGS
1261 intPregs &= ~(1ULL << R16);
1262 intPregs &= ~(1ULL << R17);
1263 #endif /* RESERVED_REGS */
1264 }
1265
GetIntRegAssignmentMap()1266 const MapleMap<regno_t, regno_t> &GetIntRegAssignmentMap() const
1267 {
1268 return intRegAssignmentMap;
1269 }
1270
GetFpRegAssignmentMap()1271 const MapleMap<regno_t, regno_t> &GetFpRegAssignmentMap() const
1272 {
1273 return fpRegAssignmentMap;
1274 }
1275
GetUseInfo()1276 const MapleMap<regno_t, uint16> &GetUseInfo() const
1277 {
1278 return useInfo;
1279 }
1280
SetUseInfoElem(regno_t regNO,uint16 info)1281 void SetUseInfoElem(regno_t regNO, uint16 info)
1282 {
1283 useInfo[regNO] = info;
1284 }
1285
IncUseInfoElem(regno_t regNO)1286 void IncUseInfoElem(regno_t regNO)
1287 {
1288 if (useInfo.find(regNO) != useInfo.end()) {
1289 ++useInfo[regNO];
1290 }
1291 }
1292
GetUseInfoElem(regno_t regNO)1293 uint16 GetUseInfoElem(regno_t regNO)
1294 {
1295 return useInfo[regNO];
1296 }
1297
ClearUseInfo()1298 void ClearUseInfo()
1299 {
1300 useInfo.clear();
1301 }
1302
GetDefInfo()1303 const MapleMap<regno_t, uint16> &GetDefInfo() const
1304 {
1305 return defInfo;
1306 }
1307
SetDefInfoElem(regno_t regNO,uint16 info)1308 void SetDefInfoElem(regno_t regNO, uint16 info)
1309 {
1310 defInfo[regNO] = info;
1311 }
1312
GetDefInfoElem(regno_t regNO)1313 uint16 GetDefInfoElem(regno_t regNO)
1314 {
1315 return defInfo[regNO];
1316 }
1317
IncDefInfoElem(regno_t regNO)1318 void IncDefInfoElem(regno_t regNO)
1319 {
1320 if (defInfo.find(regNO) != defInfo.end()) {
1321 ++defInfo[regNO];
1322 }
1323 }
1324
ClearDefInfo()1325 void ClearDefInfo()
1326 {
1327 defInfo.clear();
1328 }
1329
GetNumIntPregUsed()1330 uint32 GetNumIntPregUsed() const
1331 {
1332 return numIntPregUsed;
1333 }
1334
GetNumFpPregUsed()1335 uint32 GetNumFpPregUsed() const
1336 {
1337 return numFpPregUsed;
1338 }
1339
1340 private:
ClearBitArrElement(uint64 * vec)1341 void ClearBitArrElement(uint64 *vec) const
1342 {
1343 for (uint32 i = 0; i < buckets; ++i) {
1344 vec[i] = 0UL;
1345 }
1346 }
1347
SetBitArrElement(uint64 * vec,regno_t regNO)1348 void SetBitArrElement(uint64 *vec, regno_t regNO) const
1349 {
1350 uint32 index = regNO / kU64;
1351 uint64 bit = regNO % kU64;
1352 vec[index] |= 1ULL << bit;
1353 }
1354
1355 /* The following local vars keeps track of allocation information in bb. */
1356 uint64 *intRegAssigned; /* in this set if vreg is assigned */
1357 uint64 *fpRegAssigned;
1358 MapleMap<regno_t, regno_t> intRegAssignmentMap; /* vreg -> preg map, which preg is the vreg assigned */
1359 MapleMap<regno_t, regno_t> fpRegAssignmentMap;
1360 uint64 intPregUsed = 0; /* pregs used in bb */
1361 uint64 fpPregUsed = 0;
1362 uint64 *intRegSpilled; /* on this list if vreg is spilled */
1363 uint64 *fpRegSpilled;
1364
1365 uint64 intPregs = 0; /* available regs for assignement */
1366 uint64 fpPregs = 0;
1367 MapleMap<regno_t, uint16> useInfo; /* copy of local ra info for useCnt */
1368 MapleMap<regno_t, uint16> defInfo; /* copy of local ra info for defCnt */
1369
1370 uint32 numIntPregUsed = 0;
1371 uint32 numFpPregUsed = 0;
1372 uint32 buckets;
1373 };
1374
1375 class SplitBBInfo {
1376 public:
1377 SplitBBInfo() = default;
1378
1379 ~SplitBBInfo() = default;
1380
GetCandidateBB()1381 BB *GetCandidateBB()
1382 {
1383 return candidateBB;
1384 }
1385
GetCandidateBB()1386 const BB *GetCandidateBB() const
1387 {
1388 return candidateBB;
1389 }
1390
GetStartBB()1391 const BB *GetStartBB() const
1392 {
1393 return startBB;
1394 }
1395
SetCandidateBB(BB & bb)1396 void SetCandidateBB(BB &bb)
1397 {
1398 candidateBB = &bb;
1399 }
1400
SetStartBB(BB & bb)1401 void SetStartBB(BB &bb)
1402 {
1403 startBB = &bb;
1404 }
1405
1406 private:
1407 BB *candidateBB = nullptr;
1408 BB *startBB = nullptr;
1409 };
1410
1411 class GraphColorRegAllocator : public RegAllocator {
1412 public:
GraphColorRegAllocator(CGFunc & cgFunc,MemPool & memPool,DomAnalysis & dom,LoopAnalysis & loop)1413 GraphColorRegAllocator(CGFunc &cgFunc, MemPool &memPool, DomAnalysis &dom, LoopAnalysis &loop)
1414 : RegAllocator(cgFunc, memPool),
1415 domInfo(dom),
1416 loopInfo(loop),
1417 bbVec(alloc.Adapter()),
1418 vregLive(alloc.Adapter()),
1419 pregLive(alloc.Adapter()),
1420 lrMap(alloc.Adapter()),
1421 localRegVec(alloc.Adapter()),
1422 bbRegInfo(alloc.Adapter()),
1423 unconstrained(alloc.Adapter()),
1424 unconstrainedPref(alloc.Adapter()),
1425 constrained(alloc.Adapter()),
1426 mustAssigned(alloc.Adapter()),
1427 #ifdef OPTIMIZE_FOR_PROLOG
1428 intDelayed(alloc.Adapter()),
1429 fpDelayed(alloc.Adapter()),
1430 #endif /* OPTIMIZE_FOR_PROLOG */
1431 intCallerRegSet(alloc.Adapter()),
1432 intCalleeRegSet(alloc.Adapter()),
1433 intSpillRegSet(alloc.Adapter()),
1434 fpCallerRegSet(alloc.Adapter()),
1435 fpCalleeRegSet(alloc.Adapter()),
1436 fpSpillRegSet(alloc.Adapter()),
1437 intCalleeUsed(alloc.Adapter()),
1438 fpCalleeUsed(alloc.Adapter())
1439 {
1440 constexpr uint32 kNumInsnThreashold = 30000;
1441 numVregs = cgFunc.GetMaxVReg();
1442 localRegVec.resize(cgFunc.NumBBs());
1443 bbRegInfo.resize(cgFunc.NumBBs());
1444 if (CGOptions::DoMultiPassColorRA() && cgFunc.GetMirModule().IsCModule()) {
1445 uint32 cnt = 0;
1446 FOR_ALL_BB(bb, &cgFunc) {
1447 FOR_BB_INSNS(insn, bb) {
1448 ++cnt;
1449 }
1450 }
1451 DEBUG_ASSERT(cnt <= cgFunc.GetTotalNumberOfInstructions(), "Incorrect insn count");
1452 if (cnt <= kNumInsnThreashold) {
1453 doMultiPass = true;
1454 doLRA = false;
1455 doOptProlog = false;
1456 }
1457 }
1458 }
1459
1460 ~GraphColorRegAllocator() override = default;
1461
AllocateRegisters()1462 bool AllocateRegisters() override
1463 {
1464 return true;
1465 }
1466
1467 enum SpillMemCheck : uint8 {
1468 kSpillMemPre,
1469 kSpillMemPost,
1470 };
1471
GetLiveRange(regno_t regNO)1472 LiveRange *GetLiveRange(regno_t regNO)
1473 {
1474 auto it = lrMap.find(regNO);
1475 if (it != lrMap.end()) {
1476 return it->second;
1477 } else {
1478 return nullptr;
1479 }
1480 }
GetLiveRange(regno_t regNO)1481 LiveRange *GetLiveRange(regno_t regNO) const
1482 {
1483 auto it = lrMap.find(regNO);
1484 if (it != lrMap.end()) {
1485 return it->second;
1486 } else {
1487 return nullptr;
1488 }
1489 }
GetLrMap()1490 const MapleMap<regno_t, LiveRange *> &GetLrMap() const
1491 {
1492 return lrMap;
1493 }
1494
1495 private:
1496 struct SetLiveRangeCmpFunc {
operatorSetLiveRangeCmpFunc1497 bool operator()(const LiveRange *lhs, const LiveRange *rhs) const
1498 {
1499 if (fabs(lhs->GetPriority() - rhs->GetPriority()) <= 1e-6) {
1500 /*
1501 * This is to ensure the ordering is consistent as the reg#
1502 * differs going through VtableImpl.mpl file.
1503 */
1504 if (lhs->GetID() == rhs->GetID()) {
1505 return lhs->GetRegNO() < rhs->GetRegNO();
1506 } else {
1507 return lhs->GetID() < rhs->GetID();
1508 }
1509 }
1510 return (lhs->GetPriority() > rhs->GetPriority());
1511 }
1512 };
1513
1514 template <typename Func>
1515 void ForEachBBArrElem(const uint64 *vec, Func functor) const;
1516
1517 template <typename Func>
1518 void ForEachBBArrElemWithInterrupt(const uint64 *vec, Func functor) const;
1519
1520 template <typename Func>
1521 void ForEachRegArrElem(const uint64 *vec, Func functor) const;
1522
1523 void PrintLiveUnitMap(const LiveRange &lr) const;
1524 void PrintLiveRangeConflicts(const LiveRange &lr) const;
1525 void PrintLiveBBBit(const LiveRange &li) const;
1526 void PrintLiveRange(const LiveRange &li, const std::string &str) const;
1527 void PrintLiveRanges() const;
1528 void PrintLocalRAInfo(const std::string &str) const;
1529 void PrintBBAssignInfo() const;
1530 void PrintBBs() const;
1531
1532 uint32 MaxIntPhysRegNum() const;
1533 uint32 MaxFloatPhysRegNum() const;
1534 bool IsReservedReg(AArch64reg regNO) const;
1535 void InitFreeRegPool();
1536 void InitCCReg();
1537 bool IsYieldPointReg(regno_t regNO) const;
1538 bool IsUnconcernedReg(regno_t regNO) const;
1539 bool IsUnconcernedReg(const RegOperand ®Opnd) const;
1540 LiveRange *NewLiveRange();
1541 bool CreateLiveRangeHandleLocal(regno_t regNO, const BB &bb, bool isDef);
1542 LiveRange *CreateLiveRangeAllocateAndUpdate(regno_t regNO, const BB &bb, bool isDef, uint32 currId);
1543 void CreateLiveRange(regno_t regNO, const BB &bb, bool isDef, uint32 currPoint, bool updateCount);
1544 bool SetupLiveRangeByOpHandlePhysicalReg(const RegOperand &op, Insn &insn, regno_t regNO, bool isDef);
1545 void SetupLiveRangeByOp(Operand &op, Insn &insn, bool isDef, uint32 &numUses);
1546 void SetupLiveRangeByRegNO(regno_t liveOut, BB &bb, uint32 currPoint);
1547 bool UpdateInsnCntAndSkipUseless(Insn &insn, uint32 &currPoint) const;
1548 void UpdateCallInfo(uint32 bbId, uint32 currPoint, const Insn &insn);
1549 void ClassifyOperand(std::unordered_set<regno_t> &pregs, std::unordered_set<regno_t> &vregs,
1550 const Operand &opnd) const;
1551 void SetLrMustAssign(const RegOperand *regOpnd);
1552 void SetupMustAssignedLiveRanges(const Insn &insn);
1553 void ComputeLiveRangesForEachDefOperand(Insn &insn, bool &multiDef);
1554 void ComputeLiveRangesForEachUseOperand(Insn &insn);
1555 void ComputeLiveRangesUpdateIfInsnIsCall(const Insn &insn);
1556 void ComputeLiveRangesUpdateLiveUnitInsnRange(BB &bb, uint32 currPoint);
1557 MemOperand *CreateSpillMem(uint32 spillIdx, SpillMemCheck check);
1558 bool CheckOverlap(uint64 val, uint32 i, LiveRange &lr1, LiveRange &lr2) const;
1559 void CheckInterference(LiveRange &lr1, LiveRange &lr2) const;
1560 void SetBBInfoGlobalAssigned(uint32 bbID, regno_t regNO);
1561 bool HaveAvailableColor(const LiveRange &lr, uint32 num) const;
1562 void SplitAndColorForEachLr(MapleVector<LiveRange *> &targetLrVec);
1563 void SplitAndColor();
1564 void ColorForOptPrologEpilog();
1565 bool IsLocalReg(regno_t regNO) const;
1566 bool IsLocalReg(const LiveRange &lr) const;
1567 void HandleLocalRaDebug(regno_t regNO, const LocalRegAllocator &localRa, bool isInt) const;
1568 void HandleLocalRegAssignment(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1569 void UpdateLocalRegDefUseCount(regno_t regNO, LocalRegAllocator &localRa, bool isDef, bool isInt) const;
1570 void UpdateLocalRegConflict(regno_t regNO, LocalRegAllocator &localRa, bool isInt);
1571 void HandleLocalReg(Operand &op, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo, bool isDef, bool isInt);
1572 void LocalRaRegSetEraseReg(LocalRegAllocator &localRa, regno_t regNO) const;
1573 bool LocalRaInitRegSet(LocalRegAllocator &localRa, uint32 bbId);
1574 void LocalRaInitAllocatableRegs(LocalRegAllocator &localRa, uint32 bbId);
1575 void LocalRaForEachDefOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1576 void LocalRaForEachUseOperand(const Insn &insn, LocalRegAllocator &localRa, const BBAssignInfo *bbInfo);
1577 void LocalRaPrepareBB(BB &bb, LocalRegAllocator &localRa);
1578 void LocalRaFinalAssignment(const LocalRegAllocator &localRa, BBAssignInfo &bbInfo);
1579 void LocalRaDebug(const BB &bb, const LocalRegAllocator &localRa) const;
1580 void LocalRegisterAllocator(bool allocate);
1581 MemOperand *GetSpillOrReuseMem(LiveRange &lr, uint32 regSize, bool &isOutOfRange, Insn &insn, bool isDef);
1582 void SpillOperandForSpillPre(Insn &insn, const Operand &opnd, RegOperand &phyOpnd, uint32 spillIdx, bool needSpill);
1583 MemOperand *GetConsistentReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1584 RegType regType);
1585 MemOperand *GetCommonReuseMem(const uint64 *conflict, const std::set<MemOperand *> &usedMemOpnd, uint32 size,
1586 RegType regType);
1587 MemOperand *GetReuseMem(uint32 vregNO, uint32 size, RegType regType);
1588 MemOperand *GetSpillMem(uint32 vregNO, bool isDest, Insn &insn, AArch64reg regNO, bool &isOutOfRange) const;
1589 bool SetAvailableSpillReg(std::unordered_set<regno_t> &cannotUseReg, LiveRange &lr, uint64 &usedRegMask);
1590 void CollectCannotUseReg(std::unordered_set<regno_t> &cannotUseReg, const LiveRange &lr, Insn &insn);
1591 regno_t PickRegForSpill(uint64 &usedRegMask, RegType regType, uint32 spillIdx, bool &needSpillLr);
1592 bool SetRegForSpill(LiveRange &lr, Insn &insn, uint32 spillIdx, uint64 &usedRegMask, bool isDef);
1593 bool GetSpillReg(Insn &insn, LiveRange &lr, const uint32 &spillIdx, uint64 &usedRegMask, bool isDef);
1594 bool EncountPrevRef(const BB &pred, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1595 bool FoundPrevBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1596 bool EncountNextRef(const BB &succ, LiveRange &lr, bool isDef, std::vector<bool> &visitedMap);
1597 bool FoundNextBeforeCall(Insn &insn, LiveRange &lr, bool isDef);
1598 bool HavePrevRefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1599 bool HaveNextDefInCurBB(Insn &insn, LiveRange &lr, bool &contSearch) const;
1600 bool NeedCallerSave(Insn &insn, LiveRange &lr, bool isDef);
1601 void MarkCalleeSaveRegs();
1602 void MarkUsedRegs(Operand &opnd, uint64 &usedRegMask);
1603 bool LrGetBadReg(const LiveRange &lr) const;
1604 void OptCallerSave();
1605
1606 MapleVector<LiveRange*>::iterator GetHighPriorityLr(MapleVector<LiveRange*> &lrSet) const;
1607 void UpdateForbiddenForNeighbors(const LiveRange &lr) const;
1608 void UpdatePregvetoForNeighbors(const LiveRange &lr) const;
1609 regno_t TryToAssignCallerSave(const LiveRange &lr) const;
1610 bool ShouldUseCallee(LiveRange &lr, const MapleSet<regno_t> &calleeUsed,
1611 const MapleVector<LiveRange*> &delayed) const;
1612 void AddCalleeUsed(regno_t regNO, RegType regType);
1613 bool AssignColorToLr(LiveRange &lr, bool isDelayed = false);
1614 void PruneLrForSplit(LiveRange &lr, BB &bb, bool remove,
1615 std::set<LoopDesc*, LoopDesc::LoopDescCmp> &candidateInLoop,
1616 std::set<LoopDesc*, LoopDesc::LoopDescCmp> &defInLoop);
1617 bool UseIsUncovered(const BB &bb, const BB &startBB, std::vector<bool> &visitedBB);
1618 void FindUseForSplit(LiveRange &lr, SplitBBInfo &bbInfo, bool &remove,
1619 std::set<LoopDesc*, LoopDesc::LoopDescCmp> &candidateInLoop,
1620 std::set<LoopDesc*, LoopDesc::LoopDescCmp> &defInLoop);
1621 void FindBBSharedInSplit(LiveRange &lr, const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &candidateInLoop,
1622 std::set<LoopDesc*, LoopDesc::LoopDescCmp> &defInLoop);
1623 void ComputeBBForNewSplit(LiveRange &newLr, LiveRange &oldLr);
1624 void ClearLrBBFlags(const std::set<BB*, SortedBBCmpFunc> &member) const;
1625 void ComputeBBForOldSplit(LiveRange &newLr, LiveRange &oldLr);
1626 bool LrCanBeColored(const LiveRange &lr, const BB &bbAdded, std::unordered_set<regno_t> &conflictRegs);
1627 void MoveLrBBInfo(LiveRange &oldLr, LiveRange &newLr, BB &bb) const;
1628 bool ContainsLoop(const LoopDesc &loop, const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &loops) const;
1629 void GetAllLrMemberLoops(LiveRange &lr, std::set<LoopDesc*, LoopDesc::LoopDescCmp> &loop);
1630 bool SplitLrShouldSplit(LiveRange &lr);
1631 bool SplitLrFindCandidateLr(LiveRange &lr, LiveRange &newLr, std::unordered_set<regno_t> &conflictRegs);
1632 void SplitLrHandleLoops(LiveRange &lr, LiveRange &newLr,
1633 const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &oldLoops,
1634 const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &newLoops);
1635 void SplitLrFixNewLrCallsAndRlod(LiveRange &newLr, const std::set<LoopDesc*, LoopDesc::LoopDescCmp> &origLoops);
1636 void SplitLrFixOrigLrCalls(LiveRange &lr) const;
1637 void SplitLrUpdateInterference(LiveRange &lr);
1638 void SplitLrUpdateRegInfo(const LiveRange &origLr, LiveRange &newLr,
1639 std::unordered_set<regno_t> &conflictRegs) const;
1640 void SplitLrErrorCheckAndDebug(const LiveRange &origLr) const;
1641 void SplitLr(LiveRange &lr);
1642
1643 static constexpr uint16 kMaxUint16 = 0x7fff;
1644
1645 DomAnalysis &domInfo;
1646 LoopAnalysis &loopInfo;
1647 MapleVector<BB *> bbVec;
1648 MapleUnorderedSet<regno_t> vregLive;
1649 MapleUnorderedSet<regno_t> pregLive;
1650 MapleMap<regno_t, LiveRange *> lrMap;
1651 MapleVector<LocalRaInfo *> localRegVec; /* local reg info for each bb, no local reg if null */
1652 MapleVector<BBAssignInfo *> bbRegInfo; /* register assignment info for each bb */
1653 MapleVector<LiveRange *> unconstrained;
1654 MapleVector<LiveRange *> unconstrainedPref;
1655 MapleVector<LiveRange *> constrained;
1656 MapleVector<LiveRange *> mustAssigned;
1657 #ifdef OPTIMIZE_FOR_PROLOG
1658 MapleVector<LiveRange *> intDelayed;
1659 MapleVector<LiveRange *> fpDelayed;
1660 #endif /* OPTIMIZE_FOR_PROLOG */
1661 MapleSet<uint32> intCallerRegSet; /* integer caller saved */
1662 MapleSet<uint32> intCalleeRegSet; /* callee */
1663 MapleSet<uint32> intSpillRegSet; /* spill */
1664 MapleSet<uint32> fpCallerRegSet; /* float caller saved */
1665 MapleSet<uint32> fpCalleeRegSet; /* callee */
1666 MapleSet<uint32> fpSpillRegSet; /* spill */
1667 MapleSet<regno_t> intCalleeUsed;
1668 MapleSet<regno_t> fpCalleeUsed;
1669 Bfs *bfs = nullptr;
1670
1671 uint32 bbBuckets = 0; /* size of bit array for bb (each bucket == 64 bits) */
1672 uint32 regBuckets = 0; /* size of bit array for reg (each bucket == 64 bits) */
1673 uint32 intRegNum = 0; /* total available int preg */
1674 uint32 fpRegNum = 0; /* total available fp preg */
1675 uint32 numVregs = 0; /* number of vregs when starting */
1676 regno_t ccReg = 0;
1677 /* For spilling of spill register if there are none available
1678 * Example, all 3 operands spilled
1679 * sp_reg1 -> [spillMemOpnds[1]]
1680 * sp_reg2 -> [spillMemOpnds[2]]
1681 * ld sp_reg1 <- [addr-reg2]
1682 * ld sp_reg2 <- [addr-reg3]
1683 * reg1 <- reg2, reg3 sp_reg1 <- sp_reg1, sp_reg2
1684 * st sp_reg1 -> [addr-reg1]
1685 * sp_reg1 <- [spillMemOpnds[1]]
1686 * sp_reg2 <- [spillMemOpnds[2]]
1687 */
1688 static constexpr size_t kSpillMemOpndNum = 4;
1689 std::array<MemOperand *, kSpillMemOpndNum> spillMemOpnds = {nullptr};
1690 bool operandSpilled[kSpillMemOpndNum];
1691 bool needExtraSpillReg = false;
1692 #ifdef USE_LRA
1693 bool doLRA = true;
1694 #else
1695 bool doLRA = false;
1696 #endif
1697 #ifdef OPTIMIZE_FOR_PROLOG
1698 bool doOptProlog = true;
1699 #else
1700 bool doOptProlog = false;
1701 #endif
1702 bool hasSpill = false;
1703 bool doMultiPass = false;
1704 };
1705
1706 class CallerSavePre : public CGPre {
1707 public:
CallerSavePre(GraphColorRegAllocator * regAlloc,CGFunc & cgfunc,DomAnalysis & currDom,const LoopAnalysis & loop,MemPool & memPool,MemPool & mp2,PreKind kind,uint32 limit)1708 CallerSavePre(GraphColorRegAllocator *regAlloc, CGFunc &cgfunc, DomAnalysis &currDom, const LoopAnalysis &loop,
1709 MemPool &memPool, MemPool &mp2, PreKind kind, uint32 limit)
1710 : CGPre(currDom, memPool, mp2, kind, limit),
1711 func(&cgfunc),
1712 loopInfo(loop),
1713 regAllocator(regAlloc),
1714 loopHeadBBs(ssaPreAllocator.Adapter())
1715 {
1716 }
1717
1718 ~CallerSavePre() = default;
1719
1720 void ApplySSAPRE();
SetDump(bool val)1721 void SetDump(bool val)
1722 {
1723 dump = val;
1724 }
1725
1726 private:
1727 void ComputeAvail();
1728 void ComputeVarAndDfPhis() override;
BuildWorkList()1729 void BuildWorkList() override {};
1730 void DumpWorkCandAndOcc();
1731
GetBB(uint32 id)1732 BB *GetBB(uint32 id) const override
1733 {
1734 return func->GetBBFromID(id);
1735 }
1736
GetPUIdx()1737 PUIdx GetPUIdx() const override
1738 {
1739 return func->GetFunction().GetPuidx();
1740 }
1741
IsLoopHeadBB(uint32 bbId)1742 bool IsLoopHeadBB(uint32 bbId) const
1743 {
1744 return loopHeadBBs.find(bbId) != loopHeadBBs.end();
1745 }
1746 CGFunc *func;
1747 const LoopAnalysis &loopInfo;
1748 bool dump = false;
1749 LiveRange *workLr = nullptr;
1750 GraphColorRegAllocator *regAllocator;
1751 MapleSet<uint32> loopHeadBBs;
1752 bool beyondLimit = false;
1753 };
1754 } /* namespace maplebe */
1755
1756 #endif /* MAPLEBE_INCLUDE_CG_AARCH64_AARCH64_COLOR_RA_H */
1757