• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "optimizer/ir/inst.h"
17 #include "optimizer/ir/basicblock.h"
18 #include "optimizer/ir/graph.h"
19 #include "liveness_analyzer.h"
20 #include "optimizer/analysis/dominators_tree.h"
21 #include "optimizer/analysis/loop_analyzer.h"
22 #include "optimizer/optimizations/locations_builder.h"
23 #include "optimizer/optimizations/regalloc/reg_type.h"
24 
25 namespace ark::compiler {
LivenessAnalyzer(Graph * graph)26 LivenessAnalyzer::LivenessAnalyzer(Graph *graph)
27     : Analysis(graph),
28       allocator_(graph->GetAllocator()),
29       linearBlocks_(graph->GetAllocator()->Adapter()),
30       instLifeNumbers_(graph->GetAllocator()->Adapter()),
31       instLifeIntervals_(graph->GetAllocator()->Adapter()),
32       instsByLifeNumber_(graph->GetAllocator()->Adapter()),
33       blockLiveRanges_(graph->GetAllocator()->Adapter()),
34       blockLiveSets_(graph->GetLocalAllocator()->Adapter()),
35       pendingCatchPhiInputs_(graph->GetAllocator()->Adapter()),
36       physicalGeneralIntervals_(graph->GetAllocator()->Adapter()),
37       physicalVectorIntervals_(graph->GetAllocator()->Adapter()),
38       intervalsForTemps_(graph->GetAllocator()->Adapter()),
39       useTable_(graph->GetAllocator()),
40       hasSafepointDuringCall_(graph->GetRuntime()->HasSafepointDuringCall())
41 {
42 }
43 
RunImpl()44 bool LivenessAnalyzer::RunImpl()
45 {
46     if (!RunLocationsBuilder(GetGraph())) {
47         return false;
48     }
49     GetGraph()->RunPass<DominatorsTree>();
50     GetGraph()->RunPass<LoopAnalyzer>();
51     ResetLiveness();
52     BuildBlocksLinearOrder();
53     BuildInstLifeNumbers();
54     BuildInstLifeIntervals();
55     Finalize();
56     if (!pendingCatchPhiInputs_.empty()) {
57         COMPILER_LOG(ERROR, LIVENESS_ANALYZER)
58             << "Graph contains CatchPhi instructions whose inputs were not processed";
59         return false;
60     }
61     std::copy_if(physicalGeneralIntervals_.begin(), physicalGeneralIntervals_.end(),
62                  std::back_inserter(instLifeIntervals_), [](auto li) { return li != nullptr; });
63     std::copy_if(physicalVectorIntervals_.begin(), physicalVectorIntervals_.end(),
64                  std::back_inserter(instLifeIntervals_), [](auto li) { return li != nullptr; });
65     std::copy(intervalsForTemps_.begin(), intervalsForTemps_.end(), std::back_inserter(instLifeIntervals_));
66     COMPILER_LOG(DEBUG, LIVENESS_ANALYZER) << "Liveness analysis is constructed";
67     return true;
68 }
69 
ResetLiveness()70 void LivenessAnalyzer::ResetLiveness()
71 {
72     instLifeNumbers_.clear();
73     instLifeIntervals_.clear();
74     blockLiveSets_.clear();
75     blockLiveRanges_.clear();
76     physicalGeneralIntervals_.clear();
77     physicalVectorIntervals_.clear();
78     intervalsForTemps_.clear();
79     if (GetGraph()->GetArch() != Arch::NONE) {
80         physicalGeneralIntervals_.resize(REGISTERS_NUM);
81         physicalVectorIntervals_.resize(VREGISTERS_NUM);
82     }
83 #ifndef NDEBUG
84     finalized_ = false;
85 #endif
86 }
87 
88 /*
89  * Linear blocks order means:
90  * - all dominators of a block are visiting before this block;
91  * - all blocks belonging to the same loop are contiguous;
92  */
BuildBlocksLinearOrder()93 void LivenessAnalyzer::BuildBlocksLinearOrder()
94 {
95     ASSERT_PRINT(GetGraph()->IsAnalysisValid<DominatorsTree>(), "Liveness Analyzer needs valid Dom Tree");
96     auto size = GetGraph()->GetBlocksRPO().size();
97     linearBlocks_.reserve(size);
98     linearBlocks_.clear();
99     marker_ = GetGraph()->NewMarker();
100     ASSERT_PRINT(marker_ != UNDEF_MARKER, "There are no free markers");
101     if (GetGraph()->IsBytecodeOptimizer() && !GetGraph()->GetTryBeginBlocks().empty()) {
102         LinearizeBlocks<true>();
103     } else {
104         LinearizeBlocks<false>();
105     }
106     ASSERT(linearBlocks_.size() == size);
107     GetGraph()->EraseMarker(marker_);
108     ASSERT_PRINT(CheckLinearOrder(), "Linear block order isn't correct");
109 
110     blockLiveSets_.resize(GetGraph()->GetVectorBlocks().size());
111     blockLiveRanges_.resize(GetGraph()->GetVectorBlocks().size());
112 }
113 
114 /*
115  * Check if all forward edges of loop header were visited to get the resulting block order in RPO.
116  * Predecessors which are not in the same loop with a header - are forward edges.
117  */
AllForwardEdgesVisited(BasicBlock * block)118 bool LivenessAnalyzer::AllForwardEdgesVisited(BasicBlock *block)
119 {
120     if (!block->IsLoopHeader()) {
121         for (auto pred : block->GetPredsBlocks()) {
122             if (!pred->IsMarked(marker_)) {
123                 return false;
124             }
125         }
126     } else {
127         // Head of irreducible loop can not dominate other blocks in the loop
128         if (block->GetLoop()->IsIrreducible()) {
129             return true;
130         }
131         // Predecessors which are not dominated - are forward edges,
132         for (auto pred : block->GetPredsBlocks()) {
133             if (!block->IsDominate(pred) && !pred->IsMarked(marker_)) {
134                 return false;
135             }
136         }
137     }
138     return true;
139 }
140 
141 template <bool USE_PC_ORDER>
LinearizeBlocks()142 void LivenessAnalyzer::LinearizeBlocks()
143 {
144     ArenaList<BasicBlock *> pending {GetGraph()->GetLocalAllocator()->Adapter()};
145     pending.push_back(GetGraph()->GetStartBlock());
146 
147     while (!pending.empty()) {
148         auto current = pending.front();
149         pending.pop_front();
150 
151         linearBlocks_.push_back(current);
152         current->SetMarker(marker_);
153 
154         auto succs = current->GetSuccsBlocks();
155         // Each block is inserted into pending list before all already inserted blocks
156         // from the same loop. To process edges forwarding to a "true" branches successors
157         // should be processed in reversed order.
158         for (auto it = succs.rbegin(); it != succs.rend(); ++it) {
159             auto succ = *it;
160             if (succ->IsMarked(marker_) || !AllForwardEdgesVisited(succ)) {
161                 continue;
162             }
163             InsertSuccToPendingList<USE_PC_ORDER>(pending, succ);
164         }
165     }
166 }
167 
168 template <bool USE_PC_ORDER>
InsertSuccToPendingList(ArenaList<BasicBlock * > & pending,BasicBlock * succ)169 void LivenessAnalyzer::InsertSuccToPendingList(ArenaList<BasicBlock *> &pending, BasicBlock *succ)
170 {
171     if constexpr (USE_PC_ORDER) {  // NOLINT(readability-braces-around-statements)
172         auto pcCompare = [succ](auto block) { return block->GetGuestPc() > succ->GetGuestPc(); };
173         auto insertBefore = std::find_if(pending.begin(), pending.end(), pcCompare);
174         pending.insert(insertBefore, succ);
175     } else {
176         // Insert successor right before the first block not from an inner loop.
177         // Such ordering guarantee that a loop and all it's inner loops will be processed
178         // before following edges leading to outer loop blocks.
179         auto isNotInnerLoop = [succ](auto block) { return !block->GetLoop()->IsInside(succ->GetLoop()); };
180         auto insertBefore = std::find_if(pending.begin(), pending.end(), isNotInnerLoop);
181         pending.insert(insertBefore, succ);
182     }
183 }
184 
185 /*
186  * Check linear order correctness, using dominators tree
187  */
CheckLinearOrder()188 bool LivenessAnalyzer::CheckLinearOrder()
189 {
190     ArenaVector<size_t> blockPos(GetGraph()->GetVectorBlocks().size(), 0, GetAllocator()->Adapter());
191     size_t position = 0;
192     for (auto block : linearBlocks_) {
193         blockPos[block->GetId()] = position++;
194     }
195 
196     for (auto block : linearBlocks_) {
197         if (block->GetDominator() != nullptr) {
198             ASSERT_PRINT(blockPos[block->GetDominator()->GetId()] < blockPos[block->GetId()],
199                          "Each block should be visited after its dominator");
200         }
201         if (!block->IsTryEnd()) {
202             continue;
203         }
204         ASSERT(block->GetTryId() != INVALID_ID);
205         for (auto bb : linearBlocks_) {
206             if (bb->IsTry() && bb->GetTryId() == block->GetTryId()) {
207                 ASSERT_PRINT(blockPos[bb->GetId()] < blockPos[block->GetId()],
208                              "Each try-block should be visited before its try-end block");
209             }
210         }
211     }
212 
213     return true;
214 }
215 
216 /*
217  * Set lifetime number and visiting number for each instruction
218  */
BuildInstLifeNumbers()219 void LivenessAnalyzer::BuildInstLifeNumbers()
220 {
221     LifeNumber blockBegin;
222     LifeNumber lifeNumber = 0;
223     LinearNumber linearNumber = 0;
224 
225     for (auto block : GetLinearizedBlocks()) {
226         blockBegin = lifeNumber;
227         // set the same number for each phi in the block
228         for (auto phi : block->PhiInsts()) {
229             phi->SetLinearNumber(linearNumber++);
230             SetInstLifeNumber(phi, lifeNumber);
231             CreateLifeIntervals(phi);
232         }
233         // ignore PHI instructions
234         instsByLifeNumber_.push_back(nullptr);
235         // set a unique number for each instruction in the block, differing by 2
236         // for the reason of adding spill/fill instructions
237         for (auto inst : block->Insts()) {
238             inst->SetLinearNumber(linearNumber++);
239             CreateLifeIntervals(inst);
240             if (IsPseudoUserOfMultiOutput(inst)) {
241                 // Should be the same life number as pseudo-user, since actually they have the same definition
242                 SetInstLifeNumber(inst, lifeNumber);
243                 GetInstLifeIntervals(inst)->AddUsePosition(lifeNumber);
244                 continue;
245             }
246             lifeNumber += LIFE_NUMBER_GAP;
247             SetInstLifeNumber(inst, lifeNumber);
248             SetUsePositions(inst, lifeNumber);
249             instsByLifeNumber_.push_back(inst);
250 
251             if (inst->RequireTmpReg()) {
252                 CreateIntervalForTemp(lifeNumber);
253             }
254         }
255         lifeNumber += LIFE_NUMBER_GAP;
256         SetBlockLiveRange(block, {blockBegin, lifeNumber});
257     }
258 }
259 
SetUsePositions(Inst * userInst,LifeNumber lifeNumber)260 void LivenessAnalyzer::SetUsePositions(Inst *userInst, LifeNumber lifeNumber)
261 {
262     if (GetGraph()->IsBytecodeOptimizer()) {
263         return;
264     }
265     if (userInst->IsCatchPhi() || userInst->IsSaveState()) {
266         return;
267     }
268     for (size_t i = 0; i < userInst->GetInputsCount(); i++) {
269         auto location = userInst->GetLocation(i);
270         if (!location.IsAnyRegister()) {
271             continue;
272         }
273         auto inputInst = userInst->GetDataFlowInput(i);
274         auto li = GetInstLifeIntervals(inputInst);
275         if (location.IsRegisterValid()) {
276             useTable_.AddUseOnFixedLocation(inputInst, location, lifeNumber);
277         } else if (location.IsUnallocatedRegister()) {
278             li->AddUsePosition(lifeNumber);
279         }
280     }
281 
282     // Constant can be defined without register
283     if (userInst->IsConst() && g_options.IsCompilerRematConst()) {
284         return;
285     }
286 
287     // If instruction required dst register, set use position in the beginning of the interval
288     auto li = GetInstLifeIntervals(userInst);
289     if (!li->NoDest()) {
290         li->AddUsePosition(lifeNumber);
291     }
292 }
293 
294 /*
295  * Get lifetime intervals for each instruction
296  */
BuildInstLifeIntervals()297 void LivenessAnalyzer::BuildInstLifeIntervals()
298 {
299     for (auto it = GetLinearizedBlocks().rbegin(); it != GetLinearizedBlocks().rend(); it++) {
300         auto block = *it;
301         auto liveSet = GetInitInstLiveSet(block);
302         ProcessBlockLiveInstructions(block, liveSet);
303     }
304 }
305 
306 /*
307  * The initial set of live instructions in the `block` is the the union of all live instructions at the beginning of
308  * the block's successors.
309  * Also for each phi-instruction of the successors: input corresponding to the `block` is added to the live set.
310  */
GetInitInstLiveSet(BasicBlock * block)311 InstLiveSet *LivenessAnalyzer::GetInitInstLiveSet(BasicBlock *block)
312 {
313     unsigned instructionCount = instLifeIntervals_.size();
314     auto liveSet = GetAllocator()->New<InstLiveSet>(instructionCount, GetAllocator());
315     for (auto succ : block->GetSuccsBlocks()) {
316         // catch-begin is pseudo successor, its live set will be processed for blocks with throwable instructions
317         if (succ->IsCatchBegin()) {
318             continue;
319         }
320         liveSet->Union(GetBlockLiveSet(succ));
321         for (auto phi : succ->PhiInsts()) {
322             auto phiInput = phi->CastToPhi()->GetPhiDataflowInput(block);
323             liveSet->Add(phiInput->GetLinearNumber());
324         }
325     }
326 
327     // if basic block contains throwable instruction, all instructions live in the catch-handlers should be live in this
328     // block
329     if (auto inst = block->GetFistThrowableInst(); inst != nullptr) {
330         auto handlers = GetGraph()->GetThrowableInstHandlers(inst);
331         for (auto catchHandler : handlers) {
332             liveSet->Union(GetBlockLiveSet(catchHandler));
333         }
334     }
335     return liveSet;
336 }
337 
GetLoopEnd(Loop * loop)338 LifeNumber LivenessAnalyzer::GetLoopEnd(Loop *loop)
339 {
340     LifeNumber loopEnd = 0;
341     // find max LifeNumber of inner loops
342     for (auto inner : loop->GetInnerLoops()) {
343         loopEnd = std::max(loopEnd, GetLoopEnd(inner));
344     }
345     // find max LifeNumber of back_edges
346     for (auto backEdge : loop->GetBackEdges()) {
347         loopEnd = std::max(loopEnd, GetBlockLiveRange(backEdge).GetEnd());
348     }
349     return loopEnd;
350 }
351 
352 /*
353  * Append and adjust the lifetime intervals for each instruction live in the block and for their inputs
354  */
ProcessBlockLiveInstructions(BasicBlock * block,InstLiveSet * liveSet)355 void LivenessAnalyzer::ProcessBlockLiveInstructions(BasicBlock *block, InstLiveSet *liveSet)
356 {
357     // For each live instruction set initial life range equals to the block life range
358     for (auto &interval : instLifeIntervals_) {
359         if (liveSet->IsSet(interval->GetInst()->GetLinearNumber())) {
360             interval->AppendRange(GetBlockLiveRange(block));
361         }
362     }
363 
364     for (Inst *inst : block->InstsSafeReverse()) {
365         // Shorten instruction lifetime to the position where its defined
366         // and remove from the block's live set
367         auto instLifeNumber = GetInstLifeNumber(inst);
368         auto interval = GetInstLifeIntervals(inst);
369         interval->StartFrom(instLifeNumber);
370         liveSet->Remove(inst->GetLinearNumber());
371         if (inst->IsCatchPhi()) {
372             // catch-phi's liveness should overlap all linked try blocks' livenesses
373             for (auto pred : inst->GetBasicBlock()->GetPredsBlocks()) {
374                 instLifeNumber = std::min(instLifeNumber, GetBlockLiveRange(pred).GetBegin());
375             }
376             interval->StartFrom(instLifeNumber);
377             AdjustCatchPhiInputsLifetime(inst);
378         } else {
379             if (inst->GetOpcode() == Opcode::LiveOut) {
380                 ProcessOpcodeLiveOut(block, interval, instLifeNumber);
381             }
382             auto currentLiveRange = LiveRange {GetBlockLiveRange(block).GetBegin(), instLifeNumber};
383             AdjustInputsLifetime(inst, currentLiveRange, liveSet);
384         }
385 
386         BlockFixedRegisters(inst);
387     }
388 
389     // The lifetime interval of phis instructions starts at the beginning of the block
390     for (auto phi : block->PhiInsts()) {
391         liveSet->Remove(phi->GetLinearNumber());
392     }
393 
394     // All instructions live at the beginning ot the loop header
395     // must be live for the entire loop
396     if (block->IsLoopHeader()) {
397         LifeNumber loopEndPosition = GetLoopEnd(block->GetLoop());
398 
399         for (auto &interval : instLifeIntervals_) {
400             if (liveSet->IsSet(interval->GetInst()->GetLinearNumber())) {
401                 interval->AppendGroupRange({GetBlockLiveRange(block).GetBegin(), loopEndPosition});
402             }
403         }
404     }
405     SetBlockLiveSet(block, liveSet);
406 }
407 
ProcessOpcodeLiveOut(BasicBlock * block,LifeIntervals * interval,LifeNumber instLifeNumber)408 void LivenessAnalyzer::ProcessOpcodeLiveOut(BasicBlock *block, LifeIntervals *interval, LifeNumber instLifeNumber)
409 {
410     if (block->GetSuccsBlocks().size() == 1 && block->GetSuccessor(0)->IsEndBlock()) {
411         interval->AppendRange({instLifeNumber, GetBlockLiveRange(block).GetEnd()});
412     } else {
413         interval->AppendRange({instLifeNumber, GetBlockLiveRange(GetGraph()->GetEndBlock()).GetBegin()});
414     }
415 }
416 
417 /* static */
GetPropagatedLiveRange(Inst * inst,LiveRange liveRange)418 LiveRange LivenessAnalyzer::GetPropagatedLiveRange(Inst *inst, LiveRange liveRange)
419 {
420     /*
421      * Implicit null check encoded as no-op and if the reference to check is null
422      * then SIGSEGV will be raised at the first (closest) user. Regmap generated for
423      * NullCheck's SaveState should be valid at that user so we need to extend
424      * life intervals of SaveState's inputs until NullCheck user.
425      */
426     if (inst->IsNullCheck() && !inst->GetUsers().Empty() && inst->CastToNullCheck()->IsImplicit()) {
427         auto extendUntil = std::numeric_limits<LifeNumber>::max();
428         for (auto &user : inst->GetUsers()) {
429             // Skip the users dominating the NullCheck as their live ranges
430             // start earlier than NullCheck's live range.
431             if (!user.GetInst()->IsDominate(inst)) {
432                 auto li = GetInstLifeIntervals(user.GetInst());
433                 ASSERT(li != nullptr);
434                 extendUntil = std::min<LifeNumber>(extendUntil, li->GetBegin() + 1);
435             }
436         }
437         liveRange.SetEnd(extendUntil);
438         return liveRange;
439     }
440     /*
441      * We need to propagate liveness for instruction with CallRuntime to save registers before call;
442      * Otherwise, we will not be able to restore the value of the virtual registers
443      */
444     if (inst->IsPropagateLiveness()) {
445         liveRange.SetEnd(liveRange.GetEnd() + 1);
446     } else if (inst->GetOpcode() == Opcode::ReturnInlined && inst->CastToReturnInlined()->IsExtendedLiveness()) {
447         /*
448          * [ReturnInlined]
449          * [ReturnInlined]
450          * ...
451          * [Deoptimize/Throw]
452          *
453          * In this case we propagate ReturnInlined inputs liveness up to the end of basic block
454          */
455         liveRange.SetEnd(GetBlockLiveRange(inst->GetBasicBlock()).GetEnd());
456     }
457     return liveRange;
458 }
459 
460 /*
461  * Adjust instruction inputs lifetime and add them to the block's live set
462  */
AdjustInputsLifetime(Inst * inst,LiveRange liveRange,InstLiveSet * liveSet)463 void LivenessAnalyzer::AdjustInputsLifetime(Inst *inst, LiveRange liveRange, InstLiveSet *liveSet)
464 {
465     for (auto input : inst->GetInputs()) {
466         auto inputInst = inst->GetDataFlowInput(input.GetInst());
467         liveSet->Add(inputInst->GetLinearNumber());
468         SetInputRange(inst, inputInst, liveRange);
469     }
470 
471     // Extend SaveState inputs lifetime to the end of SaveState's lifetime
472     if (inst->RequireState()) {
473         auto saveState = inst->GetSaveState();
474         auto propagatedRange = GetPropagatedLiveRange(inst, liveRange);
475         while (true) {
476             ASSERT(saveState != nullptr);
477             for (auto ssInput : saveState->GetInputs()) {
478                 auto inputInst = saveState->GetDataFlowInput(ssInput.GetInst());
479                 liveSet->Add(inputInst->GetLinearNumber());
480                 GetInstLifeIntervals(inputInst)->AppendRange(propagatedRange);
481             }
482             auto callerInst = saveState->GetCallerInst();
483             if (callerInst == nullptr) {
484                 break;
485             }
486             saveState = callerInst->GetSaveState();
487         }
488     }
489 
490     // Handle CatchPhi inputs associated with inst
491     auto range = pendingCatchPhiInputs_.equal_range(inst);
492     for (auto it = range.first; it != range.second; ++it) {
493         auto throwableInput = it->second;
494         auto throwableInputInterval = GetInstLifeIntervals(throwableInput);
495         liveSet->Add(throwableInput->GetLinearNumber());
496         throwableInputInterval->AppendRange(liveRange);
497     }
498     pendingCatchPhiInputs_.erase(inst);
499 }
500 
501 /*
502  * Increase ref-input liveness in the 'no-async-jit' mode, since GC can be triggered and delete ref during callee-method
503  * compilation
504  */
SetInputRange(const Inst * inst,const Inst * input,LiveRange liveRange) const505 void LivenessAnalyzer::SetInputRange(const Inst *inst, const Inst *input, LiveRange liveRange) const
506 {
507     if (hasSafepointDuringCall_ && inst->IsCall() && DataType::IsReference(input->GetType())) {
508         GetInstLifeIntervals(input)->AppendRange(liveRange.GetBegin(), liveRange.GetEnd() + 1U);
509     } else {
510         GetInstLifeIntervals(input)->AppendRange(liveRange);
511     }
512 }
513 
514 /*
515  * CatchPhi does not handle inputs as regular instructions - instead of performing
516  * some action at CatchPhi's definition copy instruction are added before throwable instructions.
517  * Instead of extending input life interval until CatchPhi it is extended until throwable instruction.
518  */
AdjustCatchPhiInputsLifetime(Inst * inst)519 void LivenessAnalyzer::AdjustCatchPhiInputsLifetime(Inst *inst)
520 {
521     auto catchPhi = inst->CastToCatchPhi();
522 
523     for (ssize_t inputIdx = catchPhi->GetInputsCount() - 1; inputIdx >= 0; inputIdx--) {
524         auto inputInst = catchPhi->GetDataFlowInput(inputIdx);
525         auto throwableInst = const_cast<Inst *>(catchPhi->GetThrowableInst(inputIdx));
526 
527         pendingCatchPhiInputs_.insert({throwableInst, inputInst});
528     }
529 }
530 
SetInstLifeNumber(const Inst * inst,LifeNumber number)531 void LivenessAnalyzer::SetInstLifeNumber([[maybe_unused]] const Inst *inst, LifeNumber number)
532 {
533     ASSERT(instLifeNumbers_.size() == inst->GetLinearNumber());
534     instLifeNumbers_.push_back(number);
535 }
536 
GetInstLifeNumber(const Inst * inst) const537 LifeNumber LivenessAnalyzer::GetInstLifeNumber(const Inst *inst) const
538 {
539     return instLifeNumbers_[inst->GetLinearNumber()];
540 }
541 
542 /*
543  * Create new lifetime intervals for instruction, check that instruction linear number is equal to intervals
544  * position in vector
545  */
CreateLifeIntervals(Inst * inst)546 void LivenessAnalyzer::CreateLifeIntervals(Inst *inst)
547 {
548     ASSERT(!finalized_);
549     ASSERT(inst->GetLinearNumber() == instLifeIntervals_.size());
550     auto interval = GetAllocator()->New<LifeIntervals>(GetAllocator(), inst);
551     instLifeIntervals_.push_back(interval);
552 }
553 
CreateIntervalForTemp(LifeNumber ln)554 void LivenessAnalyzer::CreateIntervalForTemp(LifeNumber ln)
555 {
556     ASSERT(!finalized_);
557     auto interval = GetAllocator()->New<LifeIntervals>(GetAllocator());
558     interval->AppendRange({ln - 1, ln});
559     interval->AddUsePosition(ln);
560     // DataType is INT64, since general register is reserved (for 32-bits arch will be converted to INT32)
561     interval->SetType(ConvertRegType(GetGraph(), DataType::INT64));
562     intervalsForTemps_.push_back(interval);
563 }
564 
GetInstLifeIntervals(const Inst * inst) const565 LifeIntervals *LivenessAnalyzer::GetInstLifeIntervals(const Inst *inst) const
566 {
567     ASSERT(inst->GetLinearNumber() != INVALID_LINEAR_NUM);
568     return instLifeIntervals_[inst->GetLinearNumber()];
569 }
570 
SetBlockLiveRange(BasicBlock * block,LiveRange lifeRange)571 void LivenessAnalyzer::SetBlockLiveRange(BasicBlock *block, LiveRange lifeRange)
572 {
573     blockLiveRanges_[block->GetId()] = lifeRange;
574 }
575 
GetLinearizedBlocks() const576 const ArenaVector<BasicBlock *> &LivenessAnalyzer::GetLinearizedBlocks() const
577 {
578     return linearBlocks_;
579 }
580 
GetInstByLifeNumber(LifeNumber ln) const581 Inst *LivenessAnalyzer::GetInstByLifeNumber(LifeNumber ln) const
582 {
583     return instsByLifeNumber_[ln / LIFE_NUMBER_GAP];
584 }
585 
GetBlockCoversPoint(LifeNumber ln) const586 BasicBlock *LivenessAnalyzer::GetBlockCoversPoint(LifeNumber ln) const
587 {
588     for (auto bb : linearBlocks_) {
589         if (GetBlockLiveRange(bb).Contains(ln)) {
590             return bb;
591         }
592     }
593     return nullptr;
594 }
595 
Cleanup()596 void LivenessAnalyzer::Cleanup()
597 {
598     for (auto *interv : instLifeIntervals_) {
599         if (!interv->IsPhysical() && !interv->IsPreassigned()) {
600             interv->ClearLocation();
601         }
602         if (interv->GetSibling() != nullptr) {
603             interv->MergeSibling();
604         }
605     }
606 }
607 
Finalize()608 void LivenessAnalyzer::Finalize()
609 {
610     for (auto *interv : instLifeIntervals_) {
611         interv->Finalize();
612     }
613     for (auto *interv : intervalsForTemps_) {
614         interv->Finalize();
615     }
616 #ifndef NDEBUG
617     finalized_ = true;
618 #endif
619 }
620 
GetLifeIntervals() const621 const ArenaVector<LifeIntervals *> &LivenessAnalyzer::GetLifeIntervals() const
622 {
623     return instLifeIntervals_;
624 }
625 
GetBlockLiveRange(const BasicBlock * block) const626 LiveRange LivenessAnalyzer::GetBlockLiveRange(const BasicBlock *block) const
627 {
628     return blockLiveRanges_[block->GetId()];
629 }
630 
SetBlockLiveSet(BasicBlock * block,InstLiveSet * liveSet)631 void LivenessAnalyzer::SetBlockLiveSet(BasicBlock *block, InstLiveSet *liveSet)
632 {
633     blockLiveSets_[block->GetId()] = liveSet;
634 }
635 
GetBlockLiveSet(BasicBlock * block) const636 InstLiveSet *LivenessAnalyzer::GetBlockLiveSet(BasicBlock *block) const
637 {
638     return blockLiveSets_[block->GetId()];
639 }
640 
GetUseTable() const641 const UseTable &LivenessAnalyzer::GetUseTable() const
642 {
643     return useTable_;
644 }
645 
GetTmpRegInterval(const Inst * inst)646 LifeIntervals *LivenessAnalyzer::GetTmpRegInterval(const Inst *inst)
647 {
648     auto instLn = GetInstLifeNumber(inst);
649     auto it = std::find_if(intervalsForTemps_.begin(), intervalsForTemps_.end(),
650                            [instLn](auto li) { return li->GetBegin() == instLn - 1; });
651     return it == intervalsForTemps_.end() ? nullptr : *it;
652 }
653 
GetPassName() const654 const char *LivenessAnalyzer::GetPassName() const
655 {
656     return "LivenessAnalysis";
657 }
658 
GetBlocksCount() const659 size_t LivenessAnalyzer::GetBlocksCount() const
660 {
661     return blockLiveRanges_.size();
662 }
663 
GetAllocator()664 ArenaAllocator *LivenessAnalyzer::GetAllocator()
665 {
666     return allocator_;
667 }
668 
DumpLifeIntervals(std::ostream & out) const669 void LivenessAnalyzer::DumpLifeIntervals(std::ostream &out) const
670 {
671     for (auto bb : GetGraph()->GetBlocksRPO()) {
672         if (bb->GetId() >= GetBlocksCount()) {
673             continue;
674         }
675         auto blockRange = GetBlockLiveRange(bb);
676         out << "BB " << bb->GetId() << "\t" << blockRange.ToString() << std::endl;
677 
678         for (auto inst : bb->AllInsts()) {
679             if (inst->GetLinearNumber() == INVALID_LINEAR_NUM) {
680                 continue;
681             }
682             auto interval = GetInstLifeIntervals(inst);
683 
684             out << "v" << inst->GetId() << "\t";
685             for (auto sibling = interval; sibling != nullptr; sibling = sibling->GetSibling()) {
686                 out << sibling->ToString<false>() << "@ " << sibling->GetLocation().ToString(GetGraph()->GetArch())
687                     << "; ";
688             }
689             out << std::endl;
690         }
691     }
692 
693     out << "Temps:" << std::endl;
694     for (auto interval : intervalsForTemps_) {
695         out << interval->ToString<false>() << "@ " << interval->GetLocation().ToString(GetGraph()->GetArch())
696             << std::endl;
697     }
698     DumpLocationsUsage(out);
699 }
700 
DumpLocationsUsage(std::ostream & out) const701 void LivenessAnalyzer::DumpLocationsUsage(std::ostream &out) const
702 {
703     std::map<Register, std::vector<LifeIntervals *>> regsIntervals;
704     std::map<Register, std::vector<LifeIntervals *>> vregsIntervals;
705     std::map<Register, std::vector<LifeIntervals *>> slotsIntervals;
706     for (auto &interval : instLifeIntervals_) {
707         for (auto sibling = interval; sibling != nullptr; sibling = sibling->GetSibling()) {
708             auto location = sibling->GetLocation();
709             if (location.IsFpRegister()) {
710                 ASSERT(DataType::IsFloatType(interval->GetType()));
711                 vregsIntervals[location.GetValue()].push_back(sibling);
712             } else if (location.IsRegister()) {
713                 regsIntervals[location.GetValue()].push_back(sibling);
714             } else if (location.IsStack()) {
715                 slotsIntervals[location.GetValue()].push_back(sibling);
716             }
717         }
718     }
719 
720     for (auto intervalsMap : {&regsIntervals, &vregsIntervals, &slotsIntervals}) {
721         std::string locSymbol;
722         if (intervalsMap == &regsIntervals) {
723             out << std::endl << "Registers intervals" << std::endl;
724             locSymbol = "r";
725         } else if (intervalsMap == &vregsIntervals) {
726             out << std::endl << "Vector registers intervals" << std::endl;
727             locSymbol = "vr";
728         } else {
729             ASSERT(intervalsMap == &slotsIntervals);
730             out << std::endl << "Stack slots intervals" << std::endl;
731             locSymbol = "s";
732         }
733 
734         if (intervalsMap->empty()) {
735             out << "-" << std::endl;
736             continue;
737         }
738         for (auto &[reg, intervals] : *intervalsMap) {
739             std::sort(intervals.begin(), intervals.end(),
740                       [](const auto &lhs, const auto &rhs) { return lhs->GetBegin() < rhs->GetBegin(); });
741             out << locSymbol << std::to_string(reg) << ": ";
742             auto delim = "";
743             for (auto &interval : intervals) {
744                 out << delim << interval->ToString<false>();
745                 delim = "; ";
746             }
747             out << std::endl;
748         }
749     }
750 }
751 
BlockFixedRegisters(Inst * inst)752 void LivenessAnalyzer::BlockFixedRegisters(Inst *inst)
753 {
754     if (GetGraph()->IsBytecodeOptimizer() || GetGraph()->GetMode().IsFastPath()) {
755         return;
756     }
757 
758     auto blockFrom = GetInstLifeNumber(inst);
759     if (IsCallBlockingRegisters(inst)) {
760         BlockPhysicalRegisters<false>(blockFrom);
761         BlockPhysicalRegisters<true>(blockFrom);
762     }
763     for (auto i = 0U; i < inst->GetInputsCount(); ++i) {
764         BlockFixedLocationRegister(inst->GetLocation(i), blockFrom);
765     }
766     BlockFixedLocationRegister(inst->GetDstLocation(), blockFrom);
767 
768     if (inst->IsParameter()) {
769         // Block a register starting from the position preceding entry block start to
770         // correctly handle register blocked by "current" instruction from registers blocked
771         // by other instructions during register allocation.
772         constexpr auto FIRST_AVAILABLE_LIFE_NUMBER = LIFE_NUMBER_GAP - 1;
773         // Registers holding parameters should be blocked starting from the beginning of entry block
774         // to avoid its assignment to parameter instructions in case of high register pressure.
775         // The required interval is blocked using two calls to correctly mark first use position of a
776         // blocked register.
777         BlockFixedLocationRegister(inst->CastToParameter()->GetLocationData().GetSrc(), blockFrom);
778         BlockFixedLocationRegister(inst->CastToParameter()->GetLocationData().GetSrc(), FIRST_AVAILABLE_LIFE_NUMBER,
779                                    blockFrom - 1, false);
780         // Block second parameter register that contains number of actual arguments in case of
781         // dynamic methods as it is used in parameter's code generation
782         if (GetGraph()->GetMode().IsDynamicMethod() &&
783             GetGraph()->FindParameter(ParameterInst::DYNAMIC_NUM_ARGS) == nullptr) {
784             BlockReg<false>(Target(GetGraph()->GetArch()).GetParamRegId(1), blockFrom, blockFrom + 1U, true);
785             BlockReg<false>(Target(GetGraph()->GetArch()).GetParamRegId(1), FIRST_AVAILABLE_LIFE_NUMBER, blockFrom - 1,
786                             false);
787         }
788     }
789 }
790 
791 template <bool IS_FP>
BlockPhysicalRegisters(LifeNumber blockFrom)792 void LivenessAnalyzer::BlockPhysicalRegisters(LifeNumber blockFrom)
793 {
794     auto arch = GetGraph()->GetArch();
795     RegMask callerRegs {GetCallerRegsMask(arch, IS_FP)};
796     for (auto reg = GetFirstCallerReg(arch, IS_FP); reg <= GetLastCallerReg(arch, IS_FP); ++reg) {
797         if (callerRegs.test(reg)) {
798             BlockReg<IS_FP>(reg, blockFrom, blockFrom + 1U, true);
799         }
800     }
801 }
802 
BlockFixedLocationRegister(Location location,LifeNumber ln)803 void LivenessAnalyzer::BlockFixedLocationRegister(Location location, LifeNumber ln)
804 {
805     BlockFixedLocationRegister(location, ln, ln + 1U, true);
806 }
807 
BlockFixedLocationRegister(Location location,LifeNumber blockFrom,LifeNumber blockTo,bool isUse)808 void LivenessAnalyzer::BlockFixedLocationRegister(Location location, LifeNumber blockFrom, LifeNumber blockTo,
809                                                   bool isUse)
810 {
811     if (location.IsRegister() && location.IsRegisterValid()) {
812         BlockReg<false>(location.GetValue(), blockFrom, blockTo, isUse);
813     } else if (location.IsFpRegister() && location.IsRegisterValid()) {
814         BlockReg<true>(location.GetValue(), blockFrom, blockTo, isUse);
815     }
816 }
817 
818 template <bool IS_FP>
BlockReg(Register reg,LifeNumber blockFrom,LifeNumber blockTo,bool isUse)819 void LivenessAnalyzer::BlockReg(Register reg, LifeNumber blockFrom, LifeNumber blockTo, bool isUse)
820 {
821     ASSERT(!finalized_);
822     auto &intervals = IS_FP ? physicalVectorIntervals_ : physicalGeneralIntervals_;
823     auto interval = intervals.at(reg);
824     if (interval == nullptr) {
825         interval = GetGraph()->GetAllocator()->New<LifeIntervals>(GetGraph()->GetAllocator());
826         interval->SetPhysicalReg(reg, IS_FP ? DataType::FLOAT64 : DataType::UINT64);
827         intervals.at(reg) = interval;
828     }
829     interval->AppendRange(blockFrom, blockTo);
830     if (isUse) {
831         interval->AddUsePosition(blockFrom);
832     }
833 }
834 
IsCallBlockingRegisters(Inst * inst) const835 bool LivenessAnalyzer::IsCallBlockingRegisters(Inst *inst) const
836 {
837     if (inst->IsCall() && !inst->IsLaunchCall() && !static_cast<CallInst *>(inst)->IsInlined()) {
838         return true;
839     }
840     if (inst->IsIntrinsic() && inst->CastToIntrinsic()->IsNativeCall()) {
841         return true;
842     }
843     return false;
844 }
845 
SplitAt(LifeNumber ln,ArenaAllocator * alloc)846 LifeIntervals *LifeIntervals::SplitAt(LifeNumber ln, ArenaAllocator *alloc)
847 {
848     ASSERT(!IsPhysical());
849     ASSERT(ln > GetBegin() && ln <= GetEnd());
850     auto splitChild = alloc->New<LifeIntervals>(alloc, GetInst());
851 #ifndef NDEBUG
852     ASSERT(finalized_);
853     splitChild->finalized_ = true;
854 #endif
855     if (sibling_ != nullptr) {
856         splitChild->sibling_ = sibling_;
857     }
858     splitChild->isSplitSibling_ = true;
859 
860     sibling_ = splitChild;
861     splitChild->SetType(GetType());
862 
863     auto i = liveRanges_.size();
864     while (i > 0 && liveRanges_[i - 1].GetEnd() <= ln) {
865         i--;
866     }
867     if (i > 0) {
868         auto &range = liveRanges_[i - 1];
869         if (range.GetBegin() < ln) {
870             splitChild->AppendRange(range.GetBegin(), ln);
871             range.SetBegin(ln);
872         }
873     }
874     splitChild->liveRanges_.insert(splitChild->liveRanges_.end(), liveRanges_.begin() + i, liveRanges_.end());
875     liveRanges_.erase(liveRanges_.begin() + i, liveRanges_.end());
876     std::swap(liveRanges_, splitChild->liveRanges_);
877 
878     // Move use positions to the child
879     auto it = std::lower_bound(usePositions_.begin(), usePositions_.end(), ln);
880     splitChild->usePositions_.insert(splitChild->usePositions_.end(), it, usePositions_.end());
881     usePositions_.erase(it, usePositions_.end());
882 
883     return splitChild;
884 }
885 
SplitAroundUses(ArenaAllocator * alloc)886 void LifeIntervals::SplitAroundUses(ArenaAllocator *alloc)
887 {
888     if (GetUsePositions().empty()) {
889         return;
890     }
891     auto use = *GetUsePositions().begin();
892     if (use > GetBegin() + 1) {
893         auto split = SplitAt(use - 1, alloc);
894         split->SplitAroundUses(alloc);
895     } else if (use < GetEnd() - 1) {
896         auto split = SplitAt(use + 1, alloc);
897         split->SplitAroundUses(alloc);
898     }
899 }
900 
MergeSibling()901 void LifeIntervals::MergeSibling()
902 {
903     ASSERT(sibling_ != nullptr);
904 #ifndef NDEBUG
905     ASSERT(finalized_);
906     ASSERT(sibling_->finalized_);
907     if (!usePositions_.empty() && !sibling_->usePositions_.empty()) {
908         ASSERT(usePositions_.back() <= sibling_->usePositions_.front());
909     }
910 #endif
911     for (auto range : liveRanges_) {
912         sibling_->AppendRange(range);
913     }
914     liveRanges_ = std::move(sibling_->liveRanges_);
915 
916     for (auto &usePos : sibling_->usePositions_) {
917         AddUsePosition(usePos);
918     }
919     sibling_ = nullptr;
920 }
921 
FindSiblingAt(LifeNumber ln)922 LifeIntervals *LifeIntervals::FindSiblingAt(LifeNumber ln)
923 {
924     ASSERT(!IsSplitSibling());
925     for (auto head = this; head != nullptr; head = head->GetSibling()) {
926         if (head->GetBegin() <= ln && ln <= head->GetEnd()) {
927             return head;
928         }
929     }
930     return nullptr;
931 }
932 
Intersects(const LiveRange & range) const933 bool LifeIntervals::Intersects(const LiveRange &range) const
934 {
935     return  // the interval starts within the range
936         (range.GetBegin() <= GetBegin() && GetBegin() <= range.GetEnd()) ||
937         // the interval ends within the range
938         (range.GetBegin() <= GetEnd() && GetEnd() <= range.GetEnd()) ||
939         // the range is fully covered by the interval
940         (GetBegin() <= range.GetBegin() && range.GetEnd() <= GetEnd());
941 }
942 
GetFirstIntersectionWith(const LifeIntervals * other,LifeNumber searchFrom) const943 LifeNumber LifeIntervals::GetFirstIntersectionWith(const LifeIntervals *other, LifeNumber searchFrom) const
944 {
945     for (auto it = GetRanges().rbegin(); it != GetRanges().rend(); it++) {
946         auto range = *it;
947         if (range.GetEnd() <= searchFrom) {
948             continue;
949         }
950         for (auto otherIt = other->GetRanges().rbegin(); otherIt != other->GetRanges().rend(); otherIt++) {
951             auto otherRange = *otherIt;
952             if (otherRange.GetEnd() <= searchFrom) {
953                 continue;
954             }
955             auto rangeBegin = std::max<LifeNumber>(searchFrom, range.GetBegin());
956             auto otherRangeBegin = std::max<LifeNumber>(searchFrom, otherRange.GetBegin());
957             if (rangeBegin <= otherRangeBegin && otherRangeBegin < range.GetEnd()) {
958                 // [range]
959                 //    [other]
960                 return otherRangeBegin;
961                 // NOLINTNEXTLINE(readability-else-after-return)
962             } else if (rangeBegin > otherRangeBegin && rangeBegin < otherRange.GetEnd()) {
963                 //     [range]
964                 // [other]
965                 return rangeBegin;
966             }
967         }
968     }
969     return INVALID_LIFE_NUMBER;
970 }
971 
FindRangeCoveringPosition(LifeNumber ln,LiveRange * dst) const972 bool LifeIntervals::FindRangeCoveringPosition(LifeNumber ln, LiveRange *dst) const
973 {
974     for (auto &range : GetRanges()) {
975         if (range.Contains(ln)) {
976             *dst = range;
977             return true;
978         }
979     }
980 
981     return false;
982 }
983 
GetSpillWeightAt(const LivenessAnalyzer & la,LifeNumber ln)984 static float GetSpillWeightAt(const LivenessAnalyzer &la, LifeNumber ln)
985 {
986     static constexpr float LOOP_MULT = 10.0;
987     auto block = la.GetBlockCoversPoint(ln);
988     return std::pow<float>(LOOP_MULT, block->GetLoop()->GetDepth());
989 }
990 
CalcSpillWeight(const LivenessAnalyzer & la,LifeIntervals * interval)991 float CalcSpillWeight(const LivenessAnalyzer &la, LifeIntervals *interval)
992 {
993     if (interval->IsPhysical()) {
994         return std::numeric_limits<float>::max();
995     }
996 
997     size_t length = interval->GetEnd() - interval->GetBegin();
998     // Interval can't be splitted
999     if (length <= LIFE_NUMBER_GAP) {
1000         return std::numeric_limits<float>::max();
1001     }
1002 
1003     // Constant intervals are the first candidates to spill
1004     if (interval->GetInst()->IsConst()) {
1005         return std::numeric_limits<float>::min();
1006     }
1007 
1008     float useWeight = 0;
1009     if (interval->GetInst()->IsPhi()) {
1010         useWeight += GetSpillWeightAt(la, interval->GetBegin());
1011     }
1012 
1013     for (auto use : interval->GetUsePositions()) {
1014         if (use == interval->GetBegin()) {
1015             useWeight += GetSpillWeightAt(la, use + 1);
1016         } else {
1017             useWeight += GetSpillWeightAt(la, use - 1);
1018         }
1019     }
1020     return useWeight;
1021 }
1022 
1023 }  // namespace ark::compiler
1024